inat_cnn_pretrained.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "inat_cnn_pretrained.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c5f39a65f6454ed8bc7a6649c375e37e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_cec7415f7614405bb020946547319115",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3f856ca0ec4e40fc928d8a81e253af63",
              "IPY_MODEL_9bd13bb1591342d8ad9e29d878568631"
            ]
          }
        },
        "cec7415f7614405bb020946547319115": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3f856ca0ec4e40fc928d8a81e253af63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_886d43b9a1d340fba0d53f269500a3d9",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 32342954,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 32342954,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_02f9e3e9b8fc42f9b69c9638a4c6e3ad"
          }
        },
        "9bd13bb1591342d8ad9e29d878568631": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_81a3e94d613a48d68962a19f71e20366",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 30.8M/30.8M [00:01&lt;00:00, 21.7MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ebf5eec15f2840d5a439433971c808da"
          }
        },
        "886d43b9a1d340fba0d53f269500a3d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "02f9e3e9b8fc42f9b69c9638a4c6e3ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "81a3e94d613a48d68962a19f71e20366": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ebf5eec15f2840d5a439433971c808da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/paddy3696/CNN_Pytorch/blob/main/inat_cnn_pretrained.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PYcsn9UVpEyH",
        "outputId": "f9956e9a-c1b1-4c90-876b-98358f41dc68"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sat Apr 17 20:33:56 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 460.67       Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   44C    P0    28W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DsLHtUM6pJlg"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim \n",
        "from torchvision import datasets, models, transforms\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "import os\n",
        "\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "woHcNgnkpLpQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fc446bcf-a354-442a-82d1-78d9b5359270"
      },
      "source": [
        "%pip install wandb -q\n",
        "\n",
        "# Ignore excessive warnings\n",
        "import logging\n",
        "logging.propagate = False \n",
        "logging.getLogger().setLevel(logging.ERROR)\n",
        "\n",
        "# WandB – Import the wandb library\n",
        "import wandb\n",
        "#wandb.init(project='mnist_classification', entity='paddy3696')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 2.1MB 9.4MB/s \n",
            "\u001b[K     |████████████████████████████████| 102kB 10.7MB/s \n",
            "\u001b[K     |████████████████████████████████| 133kB 46.4MB/s \n",
            "\u001b[K     |████████████████████████████████| 163kB 50.5MB/s \n",
            "\u001b[K     |████████████████████████████████| 71kB 7.9MB/s \n",
            "\u001b[?25h  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for subprocess32 (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6lVpi2dppObF",
        "outputId": "e7eae87a-a9dc-4370-deb1-db9f6bb958de"
      },
      "source": [
        "train_gpu = torch.cuda.is_available()\n",
        "\n",
        "if not train_gpu:\n",
        "    print('CUDA is not available. Training on CPU')\n",
        "else:\n",
        "    print('CUDA is available! Training on GPU')"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CUDA is available! Training on GPU\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ENHB62dRqM2W",
        "outputId": "9dad40b1-c268-4acd-98b2-a3b10474d237"
      },
      "source": [
        "!pip install wget"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting wget\n",
            "  Downloading https://files.pythonhosted.org/packages/47/6a/62e288da7bcda82b935ff0c6cfe542970f04e29c756b0e147251b2fb251f/wget-3.2.zip\n",
            "Building wheels for collected packages: wget\n",
            "  Building wheel for wget (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for wget: filename=wget-3.2-cp37-none-any.whl size=9681 sha256=ade11c972cf8349e0ffd153bb83421acbc1bc4f0008004496690b753065473fc\n",
            "  Stored in directory: /root/.cache/pip/wheels/40/15/30/7d8f7cea2902b4db79e3fea550d7d7b85ecb27ef992b618f3f\n",
            "Successfully built wget\n",
            "Installing collected packages: wget\n",
            "Successfully installed wget-3.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VUI8eRZepQex"
      },
      "source": [
        "import wget\n",
        "wget.download('https://storage.googleapis.com/wandb_datasets/nature_12K.zip')\n",
        "!unzip /content/nature_12K.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TKxgQo7bUN6V"
      },
      "source": [
        "def initialize_model(model_name, num_classes, feature_extract, use_pretrained=True):\n",
        "    model = None\n",
        "    input_size = 0\n",
        "\n",
        "    if model_name == \"resnet\":\n",
        "        model = models.resnet18(pretrained=use_pretrained)\n",
        "        set_parameter_requires_grad(model, feature_extract)\n",
        "        num_ftrs = model.fc.in_features\n",
        "        model.fc = nn.Linear(num_ftrs, num_classes)\n",
        "        input_size = 224\n",
        "\n",
        "    elif model_name == \"alexnet\":\n",
        "        model = models.alexnet(pretrained=use_pretrained)\n",
        "        set_parameter_requires_grad(model, feature_extract)\n",
        "        num_ftrs = model.classifier[6].in_features\n",
        "        model.classifier[6] = nn.Linear(num_ftrs,num_classes)\n",
        "        input_size = 224\n",
        "\n",
        "    elif model_name == \"vgg\":\n",
        "        model = models.vgg11_bn(pretrained=use_pretrained)\n",
        "        set_parameter_requires_grad(model, feature_extract)\n",
        "        num_ftrs = model.classifier[6].in_features\n",
        "        model.classifier[6] = nn.Linear(num_ftrs,num_classes)\n",
        "        input_size = 224\n",
        "\n",
        "    elif model_name == \"squeezenet\":\n",
        "        model = models.squeezenet1_0(pretrained=use_pretrained)\n",
        "        set_parameter_requires_grad(model, feature_extract)\n",
        "        model.classifier[1] = nn.Conv2d(512, num_classes, kernel_size=(1,1), stride=(1,1))\n",
        "        model.num_classes = num_classes\n",
        "        input_size = 224\n",
        "\n",
        "    elif model_name == \"densenet\":\n",
        "        model = models.densenet121(pretrained=use_pretrained)\n",
        "        set_parameter_requires_grad(model, feature_extract)\n",
        "        num_ftrs = model.classifier.in_features\n",
        "        model.classifier = nn.Linear(num_ftrs, num_classes)\n",
        "        input_size = 224\n",
        "\n",
        "    elif model_name == \"inception\":\n",
        "        model = models.inception_v3(pretrained=use_pretrained)\n",
        "        set_parameter_requires_grad(model, feature_extract)\n",
        "        # Handle the auxilary net\n",
        "        num_ftrs = model.AuxLogits.fc.in_features\n",
        "        model.AuxLogits.fc = nn.Linear(num_ftrs, num_classes)\n",
        "        # Handle the primary net\n",
        "        num_ftrs = model.fc.in_features\n",
        "        model.fc = nn.Linear(num_ftrs,num_classes)\n",
        "        input_size = 299\n",
        "\n",
        "    else:\n",
        "        print(\"Invalid model name, exiting...\")\n",
        "        exit()\n",
        "\n",
        "    return model, input_size"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N_RsJQZAuTiY"
      },
      "source": [
        "def set_parameter_requires_grad(model, feature_extracting):\n",
        "    if feature_extracting:\n",
        "        for param in model.parameters():\n",
        "            param.requires_grad = False"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rutMDd0J5Pm_"
      },
      "source": [
        "def fit(model,train_loader,val_loader,optimizer,epoch,criterion, is_inception=False):\n",
        "  acc_epoch = []\n",
        "\n",
        "  for epoch in range(1, epoch+1):\n",
        "    train_loss = 0.0\n",
        "    val_loss = 0.0\n",
        "    class_correct = list(0. for i in range(10))\n",
        "    class_total = list(0. for i in range(10))\n",
        "    print('training started')\n",
        "\n",
        "    model.train()\n",
        "    for data, target in train_loader:\n",
        "      if train_gpu:\n",
        "        data, target = data.cuda(), target.cuda()\n",
        "      \n",
        "      optimizer.zero_grad()\n",
        "      if is_inception:\n",
        "        output, aux_output = model(data)\n",
        "        loss1 = criterion(output, target)\n",
        "        loss2 = criterion(aux_output, target)\n",
        "        loss = loss1 + 0.4*loss2\n",
        "      else:\n",
        "        output = model(data)\n",
        "        loss = criterion(output, target)\n",
        "      \n",
        "      loss.backward()\n",
        "      optimizer.step()\n",
        "      train_loss += loss.item()*data.size(0)\n",
        "\n",
        "    train_loss = train_loss/len(train_loader.sampler)\n",
        "    print('Training over')\n",
        "    model.eval()\n",
        "    for data, target in val_loader:\n",
        "      batch_data_size = data.size(0)\n",
        "      if train_gpu:\n",
        "        data, target = data.cuda(), target.cuda()\n",
        "      output = model(data)\n",
        "      loss = criterion(output, target)\n",
        "      val_loss += loss.item()*data.size(0)\n",
        "\n",
        "      _, pred = torch.max(output, 1)    \n",
        "      correct_tensor = pred.eq(target.data.view_as(pred))\n",
        "      correct = np.squeeze(correct_tensor.numpy()) if not train_gpu else np.squeeze(correct_tensor.cpu().numpy())\n",
        "      for i in range(batch_data_size):\n",
        "        label = target.data[i]\n",
        "        class_correct[label] += correct[i].item()\n",
        "        class_total[label] += 1\n",
        "    \n",
        "    val_loss = val_loss/len(val_loader.sampler)\n",
        "    val_accuracy = 100. * np.sum(class_correct) / np.sum(class_total)\n",
        "    acc_epoch.append(val_accuracy)\n",
        "    print('Epoch: {} \\tTraining Loss: {:.6f} \\tValidation Loss: {:.6f}'.format(epoch, train_loss, val_loss))\n",
        "    print('Val Accuracy (Overall): %.2f%% (%2d/%2d)\\n' % (val_accuracy,np.sum(class_correct), np.sum(class_total)))\n",
        "\n",
        "    wandb.log({\"Train Loss\": train_loss,\"Val Loss\": val_loss,\"Val Accuracy\": val_accuracy})\n",
        "  \n",
        "  acc_best = max(acc_epoch)\n",
        "  wandb.log({'accuracy' : acc_best})\n",
        "  #return loss_train, loss_val"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G4xJBaL_pZq1"
      },
      "source": [
        "sweep_config = {\n",
        "    'method': 'bayes', #grid, random\n",
        "    'metric': {\n",
        "      'name': 'accuracy',\n",
        "      'goal': 'maximize'   \n",
        "    },\n",
        "    'parameters': {\n",
        "        'model_name': {\n",
        "            'values': ['resnet', 'alexnet', 'vgg', 'squeezenet', 'densenet', 'inception']\n",
        "        },\n",
        "        'batch_size': {\n",
        "            'values': [16, 32]\n",
        "        },\n",
        "        'epoch': {\n",
        "            'values': [10,20]\n",
        "        },\n",
        "        'data_aug': {\n",
        "            'values': ['Yes', 'No']\n",
        "        },\n",
        "        'optimizer': {\n",
        "            'values': ['SGD','ADAM'] \n",
        "        },\n",
        "        'lr': {\n",
        "            'values': [0.1,0.01, 0.001] \n",
        "        },\n",
        "    }\n",
        "}"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "llLFio2WpbbO"
      },
      "source": [
        "#sweep_id = wandb.sweep(sweep_config, entity=\"paddy3696\", project=\"cnn_inat\")"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E8v9Btq_pdGM"
      },
      "source": [
        "def train():\n",
        "  # Default values for hyper-parameters we're going to sweep over\n",
        "  config_defaults = {\n",
        "      'model_name': 'inception',\n",
        "      'epoch': 10,\n",
        "      'data_aug': 'Yes',\n",
        "      'optimizer': 'SGD',\n",
        "      'batch_size': 16,\n",
        "      'seed': 9,\n",
        "      'lr': 0.001,\n",
        "  }\n",
        "\n",
        "  # Initialize a new wandb run\n",
        "  wandb.init(config=config_defaults)\n",
        "   \n",
        "  # Config is a variable that holds and saves hyperparameters and inputs\n",
        "  config = wandb.config\n",
        "  \n",
        "  #config.model_name = 'inception'\n",
        "  model_name = config.model_name\n",
        "  num_classes = 10\n",
        "  feature_extract = False \n",
        "\n",
        "  model, input_size = initialize_model(model_name, num_classes, feature_extract, use_pretrained=True)\n",
        "\n",
        "  print(model)\n",
        "  \n",
        "  data_dir = '/content/inaturalist_12K'\n",
        "                              \n",
        "  if config.data_aug == 'Yes':\n",
        "    train_transforms = transforms.Compose([transforms.RandomRotation(30),transforms.RandomResizedCrop(input_size),\n",
        "                                       transforms.RandomHorizontalFlip(),transforms.ToTensor(), \n",
        "                                       transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
        "                                           \n",
        "  elif config.data_aug == 'No':\n",
        "    train_transforms = transforms.Compose([transforms.RandomResizedCrop(input_size), transforms.ToTensor()])\n",
        "\n",
        "  train_data = datasets.ImageFolder(data_dir + '/train',transform=train_transforms)\n",
        "\n",
        "  batch_size = config.batch_size\n",
        "  validation_split = .1\n",
        "  shuffle_dataset = True\n",
        "  random_seed= config.seed\n",
        "\n",
        "  dataset_size = len(train_data)\n",
        "  indices = list(range(dataset_size))\n",
        "  split = int(np.floor(validation_split * dataset_size))\n",
        "  if shuffle_dataset :\n",
        "    np.random.seed(random_seed)\n",
        "    np.random.shuffle(indices)\n",
        "  train_indices, val_indices = indices[split:], indices[:split]\n",
        "\n",
        "  train_sampler = SubsetRandomSampler(train_indices)\n",
        "  valid_sampler = SubsetRandomSampler(val_indices)\n",
        "\n",
        "  train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size,sampler=train_sampler)\n",
        "  val_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size,sampler=valid_sampler)\n",
        "  print('data ready')\n",
        "    \n",
        "\n",
        "  if train_gpu:\n",
        "    model.cuda()\n",
        "\n",
        "  criterion = nn.CrossEntropyLoss()\n",
        "  epoch = config.epoch\n",
        "\n",
        "  params_to_update = model.parameters()\n",
        "  print(\"Params to learn:\")\n",
        "  if feature_extract:\n",
        "    params_to_update = []\n",
        "    for name,param in model.named_parameters():\n",
        "        if param.requires_grad == True:\n",
        "            params_to_update.append(param)\n",
        "            print(\"\\t\",name)\n",
        "  else:\n",
        "    for name,param in model.named_parameters():\n",
        "        if param.requires_grad == True:\n",
        "            print(\"\\t\",name)\n",
        "\n",
        "  if config.optimizer == 'SGD':\n",
        "    optimizer = optim.SGD(params_to_update, lr=config.lr,momentum=0.9)\n",
        "  elif config.optimizer == 'ADAM':\n",
        "    optimizer = optim.Adam(model.parameters(), lr=config.lr)\n",
        "\n",
        "  wandb.run.name = str(config.model_name) + '_bs_' + str(config.batch_size)\n",
        "\n",
        "  fit(model,train_loader,val_loader,optimizer,epoch,criterion,is_inception=(model_name==\"inception\"))"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "c5f39a65f6454ed8bc7a6649c375e37e",
            "cec7415f7614405bb020946547319115",
            "3f856ca0ec4e40fc928d8a81e253af63",
            "9bd13bb1591342d8ad9e29d878568631",
            "886d43b9a1d340fba0d53f269500a3d9",
            "02f9e3e9b8fc42f9b69c9638a4c6e3ad",
            "81a3e94d613a48d68962a19f71e20366",
            "ebf5eec15f2840d5a439433971c808da"
          ]
        },
        "id": "-p2hfQJmpd9R",
        "outputId": "83b99f66-213f-4546-dac4-e9616ffd4598"
      },
      "source": [
        "#wandb.agent(sweep_id, train,count=25)\n",
        "wandb.agent(\"5utort2o\", entity=\"paddy3696\",project=\"cnn_inat\", function =train,count=100)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Agent Starting Run: iceury0y with config:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tbatch_size: 32\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tdata_aug: No\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tepoch: 20\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlr: 0.001\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tmodel_name: densenet\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \toptimizer: SGD\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mpaddy3696\u001b[0m (use `wandb login --relogin` to force relogin)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Tracking run with wandb version 0.10.26<br/>\n",
              "                Syncing run <strong style=\"color:#cdcd00\">peachy-sweep-61</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://wandb.ai/paddy3696/cnn_inat\" target=\"_blank\">https://wandb.ai/paddy3696/cnn_inat</a><br/>\n",
              "                Sweep page: <a href=\"https://wandb.ai/paddy3696/cnn_inat/sweeps/5utort2o\" target=\"_blank\">https://wandb.ai/paddy3696/cnn_inat/sweeps/5utort2o</a><br/>\n",
              "Run page: <a href=\"https://wandb.ai/paddy3696/cnn_inat/runs/iceury0y\" target=\"_blank\">https://wandb.ai/paddy3696/cnn_inat/runs/iceury0y</a><br/>\n",
              "                Run data is saved locally in <code>/content/wandb/run-20210417_203550-iceury0y</code><br/><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading: \"https://download.pytorch.org/models/densenet121-a639ec97.pth\" to /root/.cache/torch/hub/checkpoints/densenet121-a639ec97.pth\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c5f39a65f6454ed8bc7a6649c375e37e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=32342954.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "DenseNet(\n",
            "  (features): Sequential(\n",
            "    (conv0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
            "    (norm0): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "    (relu0): ReLU(inplace=True)\n",
            "    (pool0): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
            "    (denseblock1): _DenseBlock(\n",
            "      (denselayer1): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer2): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(96, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer3): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer4): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(160, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(160, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer5): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(192, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer6): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(224, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(224, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "    )\n",
            "    (transition1): _Transition(\n",
            "      (norm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace=True)\n",
            "      (conv): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
            "    )\n",
            "    (denseblock2): _DenseBlock(\n",
            "      (denselayer1): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer2): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(160, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(160, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer3): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(192, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer4): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(224, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(224, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer5): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer6): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(288, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer7): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(320, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer8): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(352, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(352, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer9): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(384, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer10): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(416, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(416, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer11): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(448, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(448, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer12): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(480, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "    )\n",
            "    (transition2): _Transition(\n",
            "      (norm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace=True)\n",
            "      (conv): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
            "    )\n",
            "    (denseblock3): _DenseBlock(\n",
            "      (denselayer1): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer2): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(288, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer3): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(320, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer4): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(352, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(352, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer5): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(384, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer6): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(416, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(416, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer7): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(448, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(448, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer8): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(480, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer9): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer10): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(544, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(544, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer11): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(576, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer12): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(608, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(608, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer13): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(640, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(640, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer14): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(672, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer15): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(704, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(704, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer16): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(736, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(736, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer17): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(768, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer18): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(800, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(800, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer19): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(832, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(832, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer20): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(864, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(864, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer21): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(896, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(896, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer22): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(928, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(928, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer23): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(960, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(960, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer24): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(992, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(992, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "    )\n",
            "    (transition3): _Transition(\n",
            "      (norm): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace=True)\n",
            "      (conv): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
            "    )\n",
            "    (denseblock4): _DenseBlock(\n",
            "      (denselayer1): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer2): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(544, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(544, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer3): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(576, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer4): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(608, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(608, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer5): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(640, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(640, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer6): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(672, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer7): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(704, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(704, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer8): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(736, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(736, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer9): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(768, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer10): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(800, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(800, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer11): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(832, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(832, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer12): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(864, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(864, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer13): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(896, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(896, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer14): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(928, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(928, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer15): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(960, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(960, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer16): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(992, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace=True)\n",
            "        (conv1): Conv2d(992, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "    )\n",
            "    (norm5): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  )\n",
            "  (classifier): Linear(in_features=1024, out_features=10, bias=True)\n",
            ")\n",
            "data ready\n",
            "Params to learn:\n",
            "\t features.conv0.weight\n",
            "\t features.norm0.weight\n",
            "\t features.norm0.bias\n",
            "\t features.denseblock1.denselayer1.norm1.weight\n",
            "\t features.denseblock1.denselayer1.norm1.bias\n",
            "\t features.denseblock1.denselayer1.conv1.weight\n",
            "\t features.denseblock1.denselayer1.norm2.weight\n",
            "\t features.denseblock1.denselayer1.norm2.bias\n",
            "\t features.denseblock1.denselayer1.conv2.weight\n",
            "\t features.denseblock1.denselayer2.norm1.weight\n",
            "\t features.denseblock1.denselayer2.norm1.bias\n",
            "\t features.denseblock1.denselayer2.conv1.weight\n",
            "\t features.denseblock1.denselayer2.norm2.weight\n",
            "\t features.denseblock1.denselayer2.norm2.bias\n",
            "\t features.denseblock1.denselayer2.conv2.weight\n",
            "\t features.denseblock1.denselayer3.norm1.weight\n",
            "\t features.denseblock1.denselayer3.norm1.bias\n",
            "\t features.denseblock1.denselayer3.conv1.weight\n",
            "\t features.denseblock1.denselayer3.norm2.weight\n",
            "\t features.denseblock1.denselayer3.norm2.bias\n",
            "\t features.denseblock1.denselayer3.conv2.weight\n",
            "\t features.denseblock1.denselayer4.norm1.weight\n",
            "\t features.denseblock1.denselayer4.norm1.bias\n",
            "\t features.denseblock1.denselayer4.conv1.weight\n",
            "\t features.denseblock1.denselayer4.norm2.weight\n",
            "\t features.denseblock1.denselayer4.norm2.bias\n",
            "\t features.denseblock1.denselayer4.conv2.weight\n",
            "\t features.denseblock1.denselayer5.norm1.weight\n",
            "\t features.denseblock1.denselayer5.norm1.bias\n",
            "\t features.denseblock1.denselayer5.conv1.weight\n",
            "\t features.denseblock1.denselayer5.norm2.weight\n",
            "\t features.denseblock1.denselayer5.norm2.bias\n",
            "\t features.denseblock1.denselayer5.conv2.weight\n",
            "\t features.denseblock1.denselayer6.norm1.weight\n",
            "\t features.denseblock1.denselayer6.norm1.bias\n",
            "\t features.denseblock1.denselayer6.conv1.weight\n",
            "\t features.denseblock1.denselayer6.norm2.weight\n",
            "\t features.denseblock1.denselayer6.norm2.bias\n",
            "\t features.denseblock1.denselayer6.conv2.weight\n",
            "\t features.transition1.norm.weight\n",
            "\t features.transition1.norm.bias\n",
            "\t features.transition1.conv.weight\n",
            "\t features.denseblock2.denselayer1.norm1.weight\n",
            "\t features.denseblock2.denselayer1.norm1.bias\n",
            "\t features.denseblock2.denselayer1.conv1.weight\n",
            "\t features.denseblock2.denselayer1.norm2.weight\n",
            "\t features.denseblock2.denselayer1.norm2.bias\n",
            "\t features.denseblock2.denselayer1.conv2.weight\n",
            "\t features.denseblock2.denselayer2.norm1.weight\n",
            "\t features.denseblock2.denselayer2.norm1.bias\n",
            "\t features.denseblock2.denselayer2.conv1.weight\n",
            "\t features.denseblock2.denselayer2.norm2.weight\n",
            "\t features.denseblock2.denselayer2.norm2.bias\n",
            "\t features.denseblock2.denselayer2.conv2.weight\n",
            "\t features.denseblock2.denselayer3.norm1.weight\n",
            "\t features.denseblock2.denselayer3.norm1.bias\n",
            "\t features.denseblock2.denselayer3.conv1.weight\n",
            "\t features.denseblock2.denselayer3.norm2.weight\n",
            "\t features.denseblock2.denselayer3.norm2.bias\n",
            "\t features.denseblock2.denselayer3.conv2.weight\n",
            "\t features.denseblock2.denselayer4.norm1.weight\n",
            "\t features.denseblock2.denselayer4.norm1.bias\n",
            "\t features.denseblock2.denselayer4.conv1.weight\n",
            "\t features.denseblock2.denselayer4.norm2.weight\n",
            "\t features.denseblock2.denselayer4.norm2.bias\n",
            "\t features.denseblock2.denselayer4.conv2.weight\n",
            "\t features.denseblock2.denselayer5.norm1.weight\n",
            "\t features.denseblock2.denselayer5.norm1.bias\n",
            "\t features.denseblock2.denselayer5.conv1.weight\n",
            "\t features.denseblock2.denselayer5.norm2.weight\n",
            "\t features.denseblock2.denselayer5.norm2.bias\n",
            "\t features.denseblock2.denselayer5.conv2.weight\n",
            "\t features.denseblock2.denselayer6.norm1.weight\n",
            "\t features.denseblock2.denselayer6.norm1.bias\n",
            "\t features.denseblock2.denselayer6.conv1.weight\n",
            "\t features.denseblock2.denselayer6.norm2.weight\n",
            "\t features.denseblock2.denselayer6.norm2.bias\n",
            "\t features.denseblock2.denselayer6.conv2.weight\n",
            "\t features.denseblock2.denselayer7.norm1.weight\n",
            "\t features.denseblock2.denselayer7.norm1.bias\n",
            "\t features.denseblock2.denselayer7.conv1.weight\n",
            "\t features.denseblock2.denselayer7.norm2.weight\n",
            "\t features.denseblock2.denselayer7.norm2.bias\n",
            "\t features.denseblock2.denselayer7.conv2.weight\n",
            "\t features.denseblock2.denselayer8.norm1.weight\n",
            "\t features.denseblock2.denselayer8.norm1.bias\n",
            "\t features.denseblock2.denselayer8.conv1.weight\n",
            "\t features.denseblock2.denselayer8.norm2.weight\n",
            "\t features.denseblock2.denselayer8.norm2.bias\n",
            "\t features.denseblock2.denselayer8.conv2.weight\n",
            "\t features.denseblock2.denselayer9.norm1.weight\n",
            "\t features.denseblock2.denselayer9.norm1.bias\n",
            "\t features.denseblock2.denselayer9.conv1.weight\n",
            "\t features.denseblock2.denselayer9.norm2.weight\n",
            "\t features.denseblock2.denselayer9.norm2.bias\n",
            "\t features.denseblock2.denselayer9.conv2.weight\n",
            "\t features.denseblock2.denselayer10.norm1.weight\n",
            "\t features.denseblock2.denselayer10.norm1.bias\n",
            "\t features.denseblock2.denselayer10.conv1.weight\n",
            "\t features.denseblock2.denselayer10.norm2.weight\n",
            "\t features.denseblock2.denselayer10.norm2.bias\n",
            "\t features.denseblock2.denselayer10.conv2.weight\n",
            "\t features.denseblock2.denselayer11.norm1.weight\n",
            "\t features.denseblock2.denselayer11.norm1.bias\n",
            "\t features.denseblock2.denselayer11.conv1.weight\n",
            "\t features.denseblock2.denselayer11.norm2.weight\n",
            "\t features.denseblock2.denselayer11.norm2.bias\n",
            "\t features.denseblock2.denselayer11.conv2.weight\n",
            "\t features.denseblock2.denselayer12.norm1.weight\n",
            "\t features.denseblock2.denselayer12.norm1.bias\n",
            "\t features.denseblock2.denselayer12.conv1.weight\n",
            "\t features.denseblock2.denselayer12.norm2.weight\n",
            "\t features.denseblock2.denselayer12.norm2.bias\n",
            "\t features.denseblock2.denselayer12.conv2.weight\n",
            "\t features.transition2.norm.weight\n",
            "\t features.transition2.norm.bias\n",
            "\t features.transition2.conv.weight\n",
            "\t features.denseblock3.denselayer1.norm1.weight\n",
            "\t features.denseblock3.denselayer1.norm1.bias\n",
            "\t features.denseblock3.denselayer1.conv1.weight\n",
            "\t features.denseblock3.denselayer1.norm2.weight\n",
            "\t features.denseblock3.denselayer1.norm2.bias\n",
            "\t features.denseblock3.denselayer1.conv2.weight\n",
            "\t features.denseblock3.denselayer2.norm1.weight\n",
            "\t features.denseblock3.denselayer2.norm1.bias\n",
            "\t features.denseblock3.denselayer2.conv1.weight\n",
            "\t features.denseblock3.denselayer2.norm2.weight\n",
            "\t features.denseblock3.denselayer2.norm2.bias\n",
            "\t features.denseblock3.denselayer2.conv2.weight\n",
            "\t features.denseblock3.denselayer3.norm1.weight\n",
            "\t features.denseblock3.denselayer3.norm1.bias\n",
            "\t features.denseblock3.denselayer3.conv1.weight\n",
            "\t features.denseblock3.denselayer3.norm2.weight\n",
            "\t features.denseblock3.denselayer3.norm2.bias\n",
            "\t features.denseblock3.denselayer3.conv2.weight\n",
            "\t features.denseblock3.denselayer4.norm1.weight\n",
            "\t features.denseblock3.denselayer4.norm1.bias\n",
            "\t features.denseblock3.denselayer4.conv1.weight\n",
            "\t features.denseblock3.denselayer4.norm2.weight\n",
            "\t features.denseblock3.denselayer4.norm2.bias\n",
            "\t features.denseblock3.denselayer4.conv2.weight\n",
            "\t features.denseblock3.denselayer5.norm1.weight\n",
            "\t features.denseblock3.denselayer5.norm1.bias\n",
            "\t features.denseblock3.denselayer5.conv1.weight\n",
            "\t features.denseblock3.denselayer5.norm2.weight\n",
            "\t features.denseblock3.denselayer5.norm2.bias\n",
            "\t features.denseblock3.denselayer5.conv2.weight\n",
            "\t features.denseblock3.denselayer6.norm1.weight\n",
            "\t features.denseblock3.denselayer6.norm1.bias\n",
            "\t features.denseblock3.denselayer6.conv1.weight\n",
            "\t features.denseblock3.denselayer6.norm2.weight\n",
            "\t features.denseblock3.denselayer6.norm2.bias\n",
            "\t features.denseblock3.denselayer6.conv2.weight\n",
            "\t features.denseblock3.denselayer7.norm1.weight\n",
            "\t features.denseblock3.denselayer7.norm1.bias\n",
            "\t features.denseblock3.denselayer7.conv1.weight\n",
            "\t features.denseblock3.denselayer7.norm2.weight\n",
            "\t features.denseblock3.denselayer7.norm2.bias\n",
            "\t features.denseblock3.denselayer7.conv2.weight\n",
            "\t features.denseblock3.denselayer8.norm1.weight\n",
            "\t features.denseblock3.denselayer8.norm1.bias\n",
            "\t features.denseblock3.denselayer8.conv1.weight\n",
            "\t features.denseblock3.denselayer8.norm2.weight\n",
            "\t features.denseblock3.denselayer8.norm2.bias\n",
            "\t features.denseblock3.denselayer8.conv2.weight\n",
            "\t features.denseblock3.denselayer9.norm1.weight\n",
            "\t features.denseblock3.denselayer9.norm1.bias\n",
            "\t features.denseblock3.denselayer9.conv1.weight\n",
            "\t features.denseblock3.denselayer9.norm2.weight\n",
            "\t features.denseblock3.denselayer9.norm2.bias\n",
            "\t features.denseblock3.denselayer9.conv2.weight\n",
            "\t features.denseblock3.denselayer10.norm1.weight\n",
            "\t features.denseblock3.denselayer10.norm1.bias\n",
            "\t features.denseblock3.denselayer10.conv1.weight\n",
            "\t features.denseblock3.denselayer10.norm2.weight\n",
            "\t features.denseblock3.denselayer10.norm2.bias\n",
            "\t features.denseblock3.denselayer10.conv2.weight\n",
            "\t features.denseblock3.denselayer11.norm1.weight\n",
            "\t features.denseblock3.denselayer11.norm1.bias\n",
            "\t features.denseblock3.denselayer11.conv1.weight\n",
            "\t features.denseblock3.denselayer11.norm2.weight\n",
            "\t features.denseblock3.denselayer11.norm2.bias\n",
            "\t features.denseblock3.denselayer11.conv2.weight\n",
            "\t features.denseblock3.denselayer12.norm1.weight\n",
            "\t features.denseblock3.denselayer12.norm1.bias\n",
            "\t features.denseblock3.denselayer12.conv1.weight\n",
            "\t features.denseblock3.denselayer12.norm2.weight\n",
            "\t features.denseblock3.denselayer12.norm2.bias\n",
            "\t features.denseblock3.denselayer12.conv2.weight\n",
            "\t features.denseblock3.denselayer13.norm1.weight\n",
            "\t features.denseblock3.denselayer13.norm1.bias\n",
            "\t features.denseblock3.denselayer13.conv1.weight\n",
            "\t features.denseblock3.denselayer13.norm2.weight\n",
            "\t features.denseblock3.denselayer13.norm2.bias\n",
            "\t features.denseblock3.denselayer13.conv2.weight\n",
            "\t features.denseblock3.denselayer14.norm1.weight\n",
            "\t features.denseblock3.denselayer14.norm1.bias\n",
            "\t features.denseblock3.denselayer14.conv1.weight\n",
            "\t features.denseblock3.denselayer14.norm2.weight\n",
            "\t features.denseblock3.denselayer14.norm2.bias\n",
            "\t features.denseblock3.denselayer14.conv2.weight\n",
            "\t features.denseblock3.denselayer15.norm1.weight\n",
            "\t features.denseblock3.denselayer15.norm1.bias\n",
            "\t features.denseblock3.denselayer15.conv1.weight\n",
            "\t features.denseblock3.denselayer15.norm2.weight\n",
            "\t features.denseblock3.denselayer15.norm2.bias\n",
            "\t features.denseblock3.denselayer15.conv2.weight\n",
            "\t features.denseblock3.denselayer16.norm1.weight\n",
            "\t features.denseblock3.denselayer16.norm1.bias\n",
            "\t features.denseblock3.denselayer16.conv1.weight\n",
            "\t features.denseblock3.denselayer16.norm2.weight\n",
            "\t features.denseblock3.denselayer16.norm2.bias\n",
            "\t features.denseblock3.denselayer16.conv2.weight\n",
            "\t features.denseblock3.denselayer17.norm1.weight\n",
            "\t features.denseblock3.denselayer17.norm1.bias\n",
            "\t features.denseblock3.denselayer17.conv1.weight\n",
            "\t features.denseblock3.denselayer17.norm2.weight\n",
            "\t features.denseblock3.denselayer17.norm2.bias\n",
            "\t features.denseblock3.denselayer17.conv2.weight\n",
            "\t features.denseblock3.denselayer18.norm1.weight\n",
            "\t features.denseblock3.denselayer18.norm1.bias\n",
            "\t features.denseblock3.denselayer18.conv1.weight\n",
            "\t features.denseblock3.denselayer18.norm2.weight\n",
            "\t features.denseblock3.denselayer18.norm2.bias\n",
            "\t features.denseblock3.denselayer18.conv2.weight\n",
            "\t features.denseblock3.denselayer19.norm1.weight\n",
            "\t features.denseblock3.denselayer19.norm1.bias\n",
            "\t features.denseblock3.denselayer19.conv1.weight\n",
            "\t features.denseblock3.denselayer19.norm2.weight\n",
            "\t features.denseblock3.denselayer19.norm2.bias\n",
            "\t features.denseblock3.denselayer19.conv2.weight\n",
            "\t features.denseblock3.denselayer20.norm1.weight\n",
            "\t features.denseblock3.denselayer20.norm1.bias\n",
            "\t features.denseblock3.denselayer20.conv1.weight\n",
            "\t features.denseblock3.denselayer20.norm2.weight\n",
            "\t features.denseblock3.denselayer20.norm2.bias\n",
            "\t features.denseblock3.denselayer20.conv2.weight\n",
            "\t features.denseblock3.denselayer21.norm1.weight\n",
            "\t features.denseblock3.denselayer21.norm1.bias\n",
            "\t features.denseblock3.denselayer21.conv1.weight\n",
            "\t features.denseblock3.denselayer21.norm2.weight\n",
            "\t features.denseblock3.denselayer21.norm2.bias\n",
            "\t features.denseblock3.denselayer21.conv2.weight\n",
            "\t features.denseblock3.denselayer22.norm1.weight\n",
            "\t features.denseblock3.denselayer22.norm1.bias\n",
            "\t features.denseblock3.denselayer22.conv1.weight\n",
            "\t features.denseblock3.denselayer22.norm2.weight\n",
            "\t features.denseblock3.denselayer22.norm2.bias\n",
            "\t features.denseblock3.denselayer22.conv2.weight\n",
            "\t features.denseblock3.denselayer23.norm1.weight\n",
            "\t features.denseblock3.denselayer23.norm1.bias\n",
            "\t features.denseblock3.denselayer23.conv1.weight\n",
            "\t features.denseblock3.denselayer23.norm2.weight\n",
            "\t features.denseblock3.denselayer23.norm2.bias\n",
            "\t features.denseblock3.denselayer23.conv2.weight\n",
            "\t features.denseblock3.denselayer24.norm1.weight\n",
            "\t features.denseblock3.denselayer24.norm1.bias\n",
            "\t features.denseblock3.denselayer24.conv1.weight\n",
            "\t features.denseblock3.denselayer24.norm2.weight\n",
            "\t features.denseblock3.denselayer24.norm2.bias\n",
            "\t features.denseblock3.denselayer24.conv2.weight\n",
            "\t features.transition3.norm.weight\n",
            "\t features.transition3.norm.bias\n",
            "\t features.transition3.conv.weight\n",
            "\t features.denseblock4.denselayer1.norm1.weight\n",
            "\t features.denseblock4.denselayer1.norm1.bias\n",
            "\t features.denseblock4.denselayer1.conv1.weight\n",
            "\t features.denseblock4.denselayer1.norm2.weight\n",
            "\t features.denseblock4.denselayer1.norm2.bias\n",
            "\t features.denseblock4.denselayer1.conv2.weight\n",
            "\t features.denseblock4.denselayer2.norm1.weight\n",
            "\t features.denseblock4.denselayer2.norm1.bias\n",
            "\t features.denseblock4.denselayer2.conv1.weight\n",
            "\t features.denseblock4.denselayer2.norm2.weight\n",
            "\t features.denseblock4.denselayer2.norm2.bias\n",
            "\t features.denseblock4.denselayer2.conv2.weight\n",
            "\t features.denseblock4.denselayer3.norm1.weight\n",
            "\t features.denseblock4.denselayer3.norm1.bias\n",
            "\t features.denseblock4.denselayer3.conv1.weight\n",
            "\t features.denseblock4.denselayer3.norm2.weight\n",
            "\t features.denseblock4.denselayer3.norm2.bias\n",
            "\t features.denseblock4.denselayer3.conv2.weight\n",
            "\t features.denseblock4.denselayer4.norm1.weight\n",
            "\t features.denseblock4.denselayer4.norm1.bias\n",
            "\t features.denseblock4.denselayer4.conv1.weight\n",
            "\t features.denseblock4.denselayer4.norm2.weight\n",
            "\t features.denseblock4.denselayer4.norm2.bias\n",
            "\t features.denseblock4.denselayer4.conv2.weight\n",
            "\t features.denseblock4.denselayer5.norm1.weight\n",
            "\t features.denseblock4.denselayer5.norm1.bias\n",
            "\t features.denseblock4.denselayer5.conv1.weight\n",
            "\t features.denseblock4.denselayer5.norm2.weight\n",
            "\t features.denseblock4.denselayer5.norm2.bias\n",
            "\t features.denseblock4.denselayer5.conv2.weight\n",
            "\t features.denseblock4.denselayer6.norm1.weight\n",
            "\t features.denseblock4.denselayer6.norm1.bias\n",
            "\t features.denseblock4.denselayer6.conv1.weight\n",
            "\t features.denseblock4.denselayer6.norm2.weight\n",
            "\t features.denseblock4.denselayer6.norm2.bias\n",
            "\t features.denseblock4.denselayer6.conv2.weight\n",
            "\t features.denseblock4.denselayer7.norm1.weight\n",
            "\t features.denseblock4.denselayer7.norm1.bias\n",
            "\t features.denseblock4.denselayer7.conv1.weight\n",
            "\t features.denseblock4.denselayer7.norm2.weight\n",
            "\t features.denseblock4.denselayer7.norm2.bias\n",
            "\t features.denseblock4.denselayer7.conv2.weight\n",
            "\t features.denseblock4.denselayer8.norm1.weight\n",
            "\t features.denseblock4.denselayer8.norm1.bias\n",
            "\t features.denseblock4.denselayer8.conv1.weight\n",
            "\t features.denseblock4.denselayer8.norm2.weight\n",
            "\t features.denseblock4.denselayer8.norm2.bias\n",
            "\t features.denseblock4.denselayer8.conv2.weight\n",
            "\t features.denseblock4.denselayer9.norm1.weight\n",
            "\t features.denseblock4.denselayer9.norm1.bias\n",
            "\t features.denseblock4.denselayer9.conv1.weight\n",
            "\t features.denseblock4.denselayer9.norm2.weight\n",
            "\t features.denseblock4.denselayer9.norm2.bias\n",
            "\t features.denseblock4.denselayer9.conv2.weight\n",
            "\t features.denseblock4.denselayer10.norm1.weight\n",
            "\t features.denseblock4.denselayer10.norm1.bias\n",
            "\t features.denseblock4.denselayer10.conv1.weight\n",
            "\t features.denseblock4.denselayer10.norm2.weight\n",
            "\t features.denseblock4.denselayer10.norm2.bias\n",
            "\t features.denseblock4.denselayer10.conv2.weight\n",
            "\t features.denseblock4.denselayer11.norm1.weight\n",
            "\t features.denseblock4.denselayer11.norm1.bias\n",
            "\t features.denseblock4.denselayer11.conv1.weight\n",
            "\t features.denseblock4.denselayer11.norm2.weight\n",
            "\t features.denseblock4.denselayer11.norm2.bias\n",
            "\t features.denseblock4.denselayer11.conv2.weight\n",
            "\t features.denseblock4.denselayer12.norm1.weight\n",
            "\t features.denseblock4.denselayer12.norm1.bias\n",
            "\t features.denseblock4.denselayer12.conv1.weight\n",
            "\t features.denseblock4.denselayer12.norm2.weight\n",
            "\t features.denseblock4.denselayer12.norm2.bias\n",
            "\t features.denseblock4.denselayer12.conv2.weight\n",
            "\t features.denseblock4.denselayer13.norm1.weight\n",
            "\t features.denseblock4.denselayer13.norm1.bias\n",
            "\t features.denseblock4.denselayer13.conv1.weight\n",
            "\t features.denseblock4.denselayer13.norm2.weight\n",
            "\t features.denseblock4.denselayer13.norm2.bias\n",
            "\t features.denseblock4.denselayer13.conv2.weight\n",
            "\t features.denseblock4.denselayer14.norm1.weight\n",
            "\t features.denseblock4.denselayer14.norm1.bias\n",
            "\t features.denseblock4.denselayer14.conv1.weight\n",
            "\t features.denseblock4.denselayer14.norm2.weight\n",
            "\t features.denseblock4.denselayer14.norm2.bias\n",
            "\t features.denseblock4.denselayer14.conv2.weight\n",
            "\t features.denseblock4.denselayer15.norm1.weight\n",
            "\t features.denseblock4.denselayer15.norm1.bias\n",
            "\t features.denseblock4.denselayer15.conv1.weight\n",
            "\t features.denseblock4.denselayer15.norm2.weight\n",
            "\t features.denseblock4.denselayer15.norm2.bias\n",
            "\t features.denseblock4.denselayer15.conv2.weight\n",
            "\t features.denseblock4.denselayer16.norm1.weight\n",
            "\t features.denseblock4.denselayer16.norm1.bias\n",
            "\t features.denseblock4.denselayer16.conv1.weight\n",
            "\t features.denseblock4.denselayer16.norm2.weight\n",
            "\t features.denseblock4.denselayer16.norm2.bias\n",
            "\t features.denseblock4.denselayer16.conv2.weight\n",
            "\t features.norm5.weight\n",
            "\t features.norm5.bias\n",
            "\t classifier.weight\n",
            "\t classifier.bias\n",
            "training started\n",
            "Training over\n",
            "Epoch: 1 \tTraining Loss: 1.125830 \tValidation Loss: 0.780514\n",
            "Val Accuracy (Overall): 74.87% (748/999)\n",
            "\n",
            "training started\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Ctrl + C detected. Stopping sweep.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<br/>Waiting for W&B process to finish, PID 192<br/>Program failed with code 1.  Press ctrl-c to abort syncing."
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}