examples/ONNX/convnet_onnx_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eFYFac2yNGez"
   },
   "source": [
    "## Exporting a convnet using ONNX in PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "eFphj-TzLL9E",
    "outputId": "e1ddb033-0b8a-404c-b86e-4cadd91a5b6e"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torchvision.datasets as dsets\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data.sampler import SubsetRandomSampler\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "# Assuming that we are on a CUDA machine, this should print a CUDA device:\n",
    "print(device)\n",
    "import numpy as np\n",
    "VERBOSE = True  # Suppress printing of mini-batch losses"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AKwloNzlPfi8"
   },
   "source": [
    "Note the 2-D input_size.  Read the MNIST dataset and resizing the images to 32x32 pixels with zero padding includes.  They are grayscale already, so the input will be 32x32x1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "JZnh3mfSNV2J"
   },
   "outputs": [],
   "source": [
    "# Hyper Parameters \n",
    "input_size = (32, 32)\n",
    "num_classes = 10\n",
    "num_epochs = 10\n",
    "batch_size_train = 256\n",
    "batch_size_val = 256\n",
    "batch_size_test = 1024\n",
    "learning_rate = 2e-3\n",
    "num_folds = 6  # V-fold cross validation!\n",
    "v = 4  # The filter hyperparameter.  The number of activation maps is dependent.\n",
    "torch.set_printoptions(threshold=1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "IoI3ercORU1T"
   },
   "source": [
    "This initially downloads two datasets, one for training and validation (called train_dataset) and one for test."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 386,
     "referenced_widgets": [
      "0fdc8c6b9c4f44b68c77a37dec02285a",
      "61f07b5ef91945ae90dd4eca99f65415",
      "5da99c5817b14afb81831e9fa3a855b7",
      "d2c08eeb2ed7458d84fa632f2367f3c9",
      "3b2d2661e4774768b12f95ff05553814",
      "87fa6dfbd89948a18a43c315a6f86225",
      "59ea7f1e93e5471baca768dedba52424",
      "10c00e25c61b4865b00615c799ffefca",
      "2666549fa98a48c99af50d21f7da23a3",
      "e193fec346774a058a46d0d21fdfe501",
      "4a1262191e894762a438b887b5fa5b42",
      "f960cf103277412b977bfb0d85acfbc8",
      "0c9fa9ed69e848809a3d558782c4cb2e",
      "828715049fea4da9a618b78ce5603b46",
      "e74ca5b7424d4feea66824539fd647b3",
      "d9a5a6f734ba4e99adee4a67fe0d4911",
      "99dda607f4234e799c531cefe46f9207",
      "c20e973458404df0a082b029c8c99017",
      "1e96c9aeccb94068aaf58133ed3ea184",
      "49a3c8a8337744b6bd9bf681dc9188f6",
      "65cf6717f2494167ba89926fd23dc53b",
      "45b001189dbf4b15bb8671ae4d1511d1",
      "eada2145ba724c8cb13275654e72fc08",
      "1c224c92d2ef455db862661a694086d6",
      "9015a50501d94724880406719282a26f",
      "5fb6c7c904664ddc90a3ec3b25b38dc9",
      "cb3fc51a836149be8dfb149f08c236fb",
      "cbe4c48a27ea4162bc099acec1ea3f7f",
      "e21f7f644cad45ec8fb01183381fcfc3",
      "d4a8816f361f4cb2830717a830bc1e58",
      "e98052315fd64ae1b3aabf424f8fd427",
      "3444f30fa37e463fba4f4919caaabad5"
     ]
    },
    "id": "UjvAEX-WLvzO",
    "outputId": "bc4fb243-86fb-4275-83ba-ebd65b66a9a2"
   },
   "outputs": [],
   "source": [
    "# Load image data and transform images to 32x32x1\n",
    "train_dataset = dsets.MNIST(root='./data',\n",
    "                         train=True,\n",
    "                         transform=transforms.Compose([\n",
    "                             transforms.Resize(input_size),\n",
    "                             transforms.ToTensor()]),\n",
    "                         download=True)\n",
    "test_dataset = dsets.MNIST('./data',\n",
    "                        train=False,\n",
    "                        transform=transforms.Compose([\n",
    "                            transforms.Resize(input_size),\n",
    "                            transforms.ToTensor()]),\n",
    "                        download=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "UMMlJI8hNVBD"
   },
   "outputs": [],
   "source": [
    "# Dataset loaders (handle mini-batching of data) \n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size_train, shuffle=True) \n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size_test, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GXA3Fe_g42M_"
   },
   "source": [
    "A version of the well-known LeNet-5 CNN model architecture."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Y78iY-1lQfdT"
   },
   "outputs": [],
   "source": [
    "class LeNet5(nn.Module):\n",
    "    # A version of LeNet-5.  Note the hyperparameter 'v' (n^v activation maps).\n",
    "    def __init__(self, v=0):\n",
    "        super(LeNet5, self).__init__()\n",
    "        # 1 image input channel, 6 filters, 5x5 kernel\n",
    "        self.convnet = nn.Sequential(\n",
    "            nn.Conv2d(1, 2**v, kernel_size=(5, 5)),  # c1\n",
    "            nn.ReLU(),  # relu1\n",
    "            nn.MaxPool2d(kernel_size=(2, 2), stride=2),  # s2\n",
    "            nn.Conv2d(2**v, 3**v, kernel_size=(5, 5)),  # c3\n",
    "            nn.ReLU(),  # relu3\n",
    "            nn.MaxPool2d(kernel_size=(2, 2), stride=2),  # s4\n",
    "            nn.Conv2d(3**v, 5**v, kernel_size=(5, 5)),  # c5\n",
    "            nn.ReLU(),  # relu5\n",
    "        )\n",
    "\n",
    "        self.fc = nn.Sequential(\n",
    "            nn.Linear(5**v, 84),  # f6\n",
    "            nn.ReLU(),  # relu6\n",
    "            nn.Linear(84, 10),  # f7\n",
    "        )\n",
    "        \n",
    "    def forward(self, input):\n",
    "        convout = self.convnet(input)\n",
    "        convout = convout.view(input.size(0), -1)\n",
    "        output = self.fc(convout)\n",
    "        return output\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YH8eRsX25g21"
   },
   "source": [
    "The following function will train the model for a single epoch and report the training loss."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ku8iA2cdbduQ"
   },
   "outputs": [],
   "source": [
    "def train_one_epoch(epoch_num, verbose=VERBOSE):\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        # Forward pass\n",
    "        optimizer.zero_grad() \n",
    "        outputs = model(images) \n",
    "        loss = criterion(outputs, labels)\n",
    "        # Backward pass\n",
    "        loss.backward()\n",
    "        # Optimize\n",
    "        optimizer.step()\n",
    "        if verbose is True:\n",
    "            if (i + 1) % 100 == 0:\n",
    "                print('Epoch: [% d/% d], Step: [% d/% d], Loss: %.4f'\n",
    "                      % (epoch_num + 1, num_epochs, i + 1,\n",
    "                         len(train_dataset) // batch_size_train, loss.item())) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_uRXmaKj5vst"
   },
   "source": [
    "The following function computes the error for one epoch of data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "o-7_tFN7mbOt"
   },
   "outputs": [],
   "source": [
    "def epoch_error(loader, length, split='validation'):\n",
    "    \"\"\" Computes the error for all data points in a loader.\n",
    "       \n",
    "        Inputs:\n",
    "            loader: Pytorch data loader (object)\n",
    "            length: Number of data points (integer)\n",
    "            split: Name of split, typically 'train', 'test', or 'validation' (string)\n",
    "        \n",
    "        Returns:\n",
    "            error (floating point)\n",
    "    \"\"\"\n",
    "    model.eval()\n",
    "    # Measure the error for the entire loader split.\n",
    "    i = 0\n",
    "    total = 0.\n",
    "    incorrect = 0.\n",
    "    for images, labels in loader:  # One batch at a time!\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "        _, predicted = torch.max(outputs.data, 1) \n",
    "        total += labels.size(0) \n",
    "        incorrect += (predicted != labels).sum()\n",
    "\n",
    "    print(f'Error of the model on the {length} {split} images: {float(incorrect) / total:3.1%}')\n",
    "    return float(incorrect) / total"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "dFrwsRx56I9r"
   },
   "source": [
    "This procedure will initialize the model and run a training loop."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rDHcLUbL5GTy"
   },
   "source": [
    "This is a 10-class classification problem.  Adam is used for optimization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Fs9C8jGYuVSB"
   },
   "outputs": [],
   "source": [
    "def run_training(v):\n",
    "    # Re-initialize model and optimizer!\n",
    "    global model, criterion, optimizer\n",
    "    model = LeNet5(v).to(device)\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "    for epoch in range(num_epochs):\n",
    "        train_one_epoch(epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "UwmihiACh-ge",
    "outputId": "0cf80e9f-47f3-44c5-89a4-b216ec69d712",
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "# Assess the training and test accuracy of LeNet-5\n",
    "run_training(v)\n",
    "test_accuracy = 1. - epoch_error(test_loader, test_dataset, 'test')\n",
    "train_accuracy = 1. - epoch_error(train_loader, train_dataset, 'train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Hoy1LTZc1rwe",
    "outputId": "57907a5e-fcd1-4c60-e2c4-01f9ebbc46cd",
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "# Set the model to inference mode\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "U5xQjnrt2PeS",
    "outputId": "50037019-70ef-4ec3-825c-9f2d12a04578",
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "for images, labels in test_loader:\n",
    "    batch_of_images = images.to(device)  # one batch\n",
    "    print(batch_of_images.shape)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DybDHZQO5AUg",
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "# Input to the model\n",
    "torch_out = model(batch_of_images)\n",
    "\n",
    "# Export the model\n",
    "torch.onnx.export(model,                     # model being run\n",
    "                  batch_of_images,           # model input (or a tuple for multiple inputs)\n",
    "                  \"convnet.onnx\",   # where to save the model (can be a file or file-like object)\n",
    "                  export_params=True,        # store the trained parameter weights inside the model file\n",
    "                  opset_version=10,          # the ONNX version to export the model to\n",
    "                  do_constant_folding=True,  # whether to execute constant folding for optimization\n",
    "                  input_names = ['input'],   # the model's input names\n",
    "                  output_names = ['output'], # the model's output names\n",
    "                  dynamic_axes={'input' : {0 : 'batch_size'},    # variable length axes\n",
    "                                'output' : {0 : 'batch_size'}})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mgYvHP6arXFH"
   },
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ROtxdVvg5q-W",
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "import onnx\n",
    "\n",
    "onnx_model = onnx.load(\"convnet.onnx\")\n",
    "onnx.checker.check_model(onnx_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1btDfFiJ6ns1"
   },
   "source": [
    "Verify that the ONNX runtime and PyTorch models are computing the same values for the network.  Do this by creating an inference session for the model and evaluating it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7z0HNCmQ6xbm",
    "outputId": "c18b84f0-2edf-4c54-c189-c51e4d1d9ec7",
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "import onnxruntime\n",
    "\n",
    "ort_session = onnxruntime.InferenceSession(\"convnet.onnx\")\n",
    "\n",
    "def to_numpy(tensor):\n",
    "    return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()\n",
    "\n",
    "# compute ONNX Runtime output prediction\n",
    "ort_inputs = {ort_session.get_inputs()[0].name: to_numpy(batch_of_images)}\n",
    "ort_outs = ort_session.run(None, ort_inputs)\n",
    "\n",
    "# compare ONNX Runtime and PyTorch results\n",
    "np.testing.assert_allclose(to_numpy(torch_out), ort_outs[0], rtol=1e-02, atol=1e-03)\n",
    "\n",
    "print(\"Exported model has been tested with ONNXRuntime, and the result looks good!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "SOhbJCs_78PJ",
    "outputId": "4a7ffc61-7a4f-44c4-8a3a-e2116fd44edc",
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "!ls -alt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "e7lMKSIN7-nJ",
    "outputId": "1e9115cd-5f83-477f-9f0b-e1361e418adf",
    "scrolled": false,
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "!protoc --decode=onnx.ModelProto onnx.proto < convnet.onnx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "WDpEAsRgksUJ",
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "PG copy of convnet_onnx_example.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "name": "pycharm-c5d14719",
   "language": "python",
   "display_name": "PyCharm (MDF)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0c9fa9ed69e848809a3d558782c4cb2e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "0fdc8c6b9c4f44b68c77a37dec02285a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_5da99c5817b14afb81831e9fa3a855b7",
       "IPY_MODEL_d2c08eeb2ed7458d84fa632f2367f3c9"
      ],
      "layout": "IPY_MODEL_61f07b5ef91945ae90dd4eca99f65415"
     }
    },
    "10c00e25c61b4865b00615c799ffefca": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1c224c92d2ef455db862661a694086d6": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1e96c9aeccb94068aaf58133ed3ea184": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_45b001189dbf4b15bb8671ae4d1511d1",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_65cf6717f2494167ba89926fd23dc53b",
      "value": 1
     }
    },
    "2666549fa98a48c99af50d21f7da23a3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_4a1262191e894762a438b887b5fa5b42",
       "IPY_MODEL_f960cf103277412b977bfb0d85acfbc8"
      ],
      "layout": "IPY_MODEL_e193fec346774a058a46d0d21fdfe501"
     }
    },
    "3444f30fa37e463fba4f4919caaabad5": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3b2d2661e4774768b12f95ff05553814": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "45b001189dbf4b15bb8671ae4d1511d1": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "49a3c8a8337744b6bd9bf681dc9188f6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1c224c92d2ef455db862661a694086d6",
      "placeholder": "​",
      "style": "IPY_MODEL_eada2145ba724c8cb13275654e72fc08",
      "value": " 1654784/? [00:18&lt;00:00, 396209.83it/s]"
     }
    },
    "4a1262191e894762a438b887b5fa5b42": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_828715049fea4da9a618b78ce5603b46",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_0c9fa9ed69e848809a3d558782c4cb2e",
      "value": 1
     }
    },
    "59ea7f1e93e5471baca768dedba52424": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "5da99c5817b14afb81831e9fa3a855b7": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_87fa6dfbd89948a18a43c315a6f86225",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_3b2d2661e4774768b12f95ff05553814",
      "value": 1
     }
    },
    "5fb6c7c904664ddc90a3ec3b25b38dc9": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "61f07b5ef91945ae90dd4eca99f65415": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "65cf6717f2494167ba89926fd23dc53b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "828715049fea4da9a618b78ce5603b46": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "87fa6dfbd89948a18a43c315a6f86225": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9015a50501d94724880406719282a26f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_cb3fc51a836149be8dfb149f08c236fb",
       "IPY_MODEL_cbe4c48a27ea4162bc099acec1ea3f7f"
      ],
      "layout": "IPY_MODEL_5fb6c7c904664ddc90a3ec3b25b38dc9"
     }
    },
    "99dda607f4234e799c531cefe46f9207": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_1e96c9aeccb94068aaf58133ed3ea184",
       "IPY_MODEL_49a3c8a8337744b6bd9bf681dc9188f6"
      ],
      "layout": "IPY_MODEL_c20e973458404df0a082b029c8c99017"
     }
    },
    "c20e973458404df0a082b029c8c99017": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "cb3fc51a836149be8dfb149f08c236fb": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "  0%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d4a8816f361f4cb2830717a830bc1e58",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_e21f7f644cad45ec8fb01183381fcfc3",
      "value": 0
     }
    },
    "cbe4c48a27ea4162bc099acec1ea3f7f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3444f30fa37e463fba4f4919caaabad5",
      "placeholder": "​",
      "style": "IPY_MODEL_e98052315fd64ae1b3aabf424f8fd427",
      "value": " 0/4542 [00:00&lt;?, ?it/s]"
     }
    },
    "d2c08eeb2ed7458d84fa632f2367f3c9": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_10c00e25c61b4865b00615c799ffefca",
      "placeholder": "​",
      "style": "IPY_MODEL_59ea7f1e93e5471baca768dedba52424",
      "value": " 9920512/? [00:20&lt;00:00, 777857.12it/s]"
     }
    },
    "d4a8816f361f4cb2830717a830bc1e58": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d9a5a6f734ba4e99adee4a67fe0d4911": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e193fec346774a058a46d0d21fdfe501": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e21f7f644cad45ec8fb01183381fcfc3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "e74ca5b7424d4feea66824539fd647b3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "e98052315fd64ae1b3aabf424f8fd427": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "eada2145ba724c8cb13275654e72fc08": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "f960cf103277412b977bfb0d85acfbc8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d9a5a6f734ba4e99adee4a67fe0d4911",
      "placeholder": "​",
      "style": "IPY_MODEL_e74ca5b7424d4feea66824539fd647b3",
      "value": " 32768/? [00:00&lt;00:00, 109984.51it/s]"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}