SineKAN_MNIST_Demo.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ereinha/SineKAN/blob/main/SineKAN_MNIST_Demo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pb5N8Ayea-QZ"
      },
      "outputs": [],
      "source": [
        "from sine_kan import *\n",
        "\n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "from torch.utils.data import DataLoader\n",
        "from tqdm import tqdm"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_OYd8RAu5fSf"
      },
      "outputs": [],
      "source": [
        "# Load MNIST\n",
        "transform = transforms.Compose(\n",
        "    [transforms.ToTensor(), transforms.Normalize((0.5,), (0.5,))]\n",
        ")\n",
        "trainset = torchvision.datasets.MNIST(\n",
        "    root=\"./data\", train=True, download=True, transform=transform\n",
        ")\n",
        "valset = torchvision.datasets.MNIST(\n",
        "    root=\"./data\", train=False, download=True, transform=transform\n",
        ")\n",
        "train_loader = DataLoader(trainset, batch_size=64, num_workers=2, shuffle=True)\n",
        "val_loader = DataLoader(valset, batch_size=64, num_workers=2, shuffle=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "7z9Q5gv73Ngn",
        "outputId": "54de114b-0e8c-4a1a-ff54-34b6c480d743"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.56it/s, accuracy=0.969, loss=0.0922, lr=0.0003]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1, Val Loss: 0.21242379619603513, Val Accuracy: 0.9362062101910829\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.24it/s, accuracy=0.938, loss=0.238, lr=0.00024]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2, Val Loss: 0.17234048355892775, Val Accuracy: 0.9500398089171974\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.14it/s, accuracy=0.969, loss=0.143, lr=0.000192]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3, Val Loss: 0.13744682892207888, Val Accuracy: 0.9585987261146497\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.87it/s, accuracy=1, loss=0.0331, lr=0.000154]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4, Val Loss: 0.11333793868639976, Val Accuracy: 0.9658638535031847\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.49it/s, accuracy=1, loss=0.026, lr=0.000123]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, Val Loss: 0.09956982904773763, Val Accuracy: 0.9694466560509554\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.61it/s, accuracy=1, loss=0.0109, lr=9.83e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6, Val Loss: 0.10011682110937953, Val Accuracy: 0.9696457006369427\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.14it/s, accuracy=1, loss=0.0603, lr=7.86e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7, Val Loss: 0.09720293554042875, Val Accuracy: 0.96984474522293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.66it/s, accuracy=1, loss=0.0293, lr=6.29e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8, Val Loss: 0.09484676426260993, Val Accuracy: 0.9703423566878981\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.85it/s, accuracy=0.969, loss=0.0369, lr=5.03e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9, Val Loss: 0.09360288912085608, Val Accuracy: 0.9722332802547771\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.07it/s, accuracy=1, loss=0.005, lr=4.03e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10, Val Loss: 0.09481710887100299, Val Accuracy: 0.9710390127388535\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.55it/s, accuracy=1, loss=0.00488, lr=3.22e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 11, Val Loss: 0.09442130025998233, Val Accuracy: 0.9715366242038217\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.56it/s, accuracy=1, loss=0.0137, lr=2.58e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 12, Val Loss: 0.09360623876690259, Val Accuracy: 0.9718351910828026\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.28it/s, accuracy=1, loss=0.00696, lr=2.06e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 13, Val Loss: 0.09400550615907036, Val Accuracy: 0.9723328025477707\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.55it/s, accuracy=1, loss=0.00489, lr=1.65e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 14, Val Loss: 0.09456536616328481, Val Accuracy: 0.972531847133758\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.05it/s, accuracy=1, loss=0.00229, lr=1.32e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 15, Val Loss: 0.09546848204200806, Val Accuracy: 0.9726313694267515\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.41it/s, accuracy=1, loss=0.00105, lr=1.06e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 16, Val Loss: 0.09563083861854223, Val Accuracy: 0.9719347133757962\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.26it/s, accuracy=1, loss=0.00238, lr=8.44e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 17, Val Loss: 0.09533573473117071, Val Accuracy: 0.9730294585987261\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.04it/s, accuracy=1, loss=0.00436, lr=6.76e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 18, Val Loss: 0.09483264959169808, Val Accuracy: 0.9721337579617835\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.76it/s, accuracy=1, loss=0.00219, lr=5.4e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 19, Val Loss: 0.09708643742571184, Val Accuracy: 0.9728304140127388\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.31it/s, accuracy=1, loss=0.00535, lr=4.32e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 20, Val Loss: 0.09642141046292434, Val Accuracy: 0.9723328025477707\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.32it/s, accuracy=1, loss=0.00354, lr=3.46e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 21, Val Loss: 0.09660260053722271, Val Accuracy: 0.9721337579617835\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.19it/s, accuracy=1, loss=0.00456, lr=2.77e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 22, Val Loss: 0.09772401905497559, Val Accuracy: 0.972531847133758\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.00it/s, accuracy=1, loss=0.00137, lr=2.21e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 23, Val Loss: 0.09678329293666908, Val Accuracy: 0.972531847133758\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.40it/s, accuracy=1, loss=0.000584, lr=1.77e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 24, Val Loss: 0.09699868467436851, Val Accuracy: 0.9727308917197452\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.45it/s, accuracy=1, loss=0.0023, lr=1.42e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 25, Val Loss: 0.09712441040626185, Val Accuracy: 0.9723328025477707\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.59it/s, accuracy=1, loss=0.0109, lr=1.13e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 26, Val Loss: 0.0971162706177632, Val Accuracy: 0.9726313694267515\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.55it/s, accuracy=1, loss=0.0032, lr=9.07e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 27, Val Loss: 0.09749484585099824, Val Accuracy: 0.9726313694267515\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.45it/s, accuracy=1, loss=0.00381, lr=7.25e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 28, Val Loss: 0.0972660922371763, Val Accuracy: 0.9724323248407644\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.34it/s, accuracy=1, loss=0.000317, lr=5.8e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 29, Val Loss: 0.0973552830675463, Val Accuracy: 0.9724323248407644\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.01it/s, accuracy=1, loss=0.00183, lr=4.64e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 30, Val Loss: 0.09731142411008847, Val Accuracy: 0.972531847133758\n",
            "LR: 0.0003 Gamma: 0.8 Hdim: 64 Best Accuracy: 0.9730294585987261\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.38it/s, accuracy=0.906, loss=0.214, lr=0.0003]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1, Val Loss: 0.2039622921188166, Val Accuracy: 0.9422770700636943\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.02it/s, accuracy=0.938, loss=0.0918, lr=0.00024]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2, Val Loss: 0.12352845700647516, Val Accuracy: 0.9620820063694268\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.51it/s, accuracy=0.938, loss=0.161, lr=0.000192]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3, Val Loss: 0.1084155322313546, Val Accuracy: 0.9643710191082803\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.25it/s, accuracy=1, loss=0.00926, lr=0.000154]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4, Val Loss: 0.09235044620533706, Val Accuracy: 0.9709394904458599\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.52it/s, accuracy=1, loss=0.0129, lr=0.000123]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, Val Loss: 0.09216597514445034, Val Accuracy: 0.9713375796178344\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.14it/s, accuracy=1, loss=0.00566, lr=9.83e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6, Val Loss: 0.07931439165898323, Val Accuracy: 0.9750199044585988\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.73it/s, accuracy=1, loss=0.0156, lr=7.86e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7, Val Loss: 0.07809310341032354, Val Accuracy: 0.9759156050955414\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.90it/s, accuracy=1, loss=0.00364, lr=6.29e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8, Val Loss: 0.08026681835076344, Val Accuracy: 0.9757165605095541\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.91it/s, accuracy=1, loss=0.00115, lr=5.03e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9, Val Loss: 0.07915412135188472, Val Accuracy: 0.9754179936305732\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.61it/s, accuracy=1, loss=0.00224, lr=4.03e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10, Val Loss: 0.07933925168374557, Val Accuracy: 0.976015127388535\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.87it/s, accuracy=1, loss=0.00121, lr=3.22e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 11, Val Loss: 0.07749520425396864, Val Accuracy: 0.9770103503184714\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.01it/s, accuracy=1, loss=0.00359, lr=2.58e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 12, Val Loss: 0.07734931905025016, Val Accuracy: 0.9775079617834395\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.39it/s, accuracy=1, loss=0.000624, lr=2.06e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 13, Val Loss: 0.07959290079728075, Val Accuracy: 0.9774084394904459\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.42it/s, accuracy=1, loss=0.00241, lr=1.65e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 14, Val Loss: 0.0812333429768297, Val Accuracy: 0.9765127388535032\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.46it/s, accuracy=1, loss=0.000756, lr=1.32e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 15, Val Loss: 0.08247349806425187, Val Accuracy: 0.9773089171974523\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.86it/s, accuracy=1, loss=0.000871, lr=1.06e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 16, Val Loss: 0.08147913535863067, Val Accuracy: 0.9772093949044586\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.07it/s, accuracy=1, loss=0.000475, lr=8.44e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 17, Val Loss: 0.08132026313646197, Val Accuracy: 0.9777070063694268\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.45it/s, accuracy=1, loss=0.00053, lr=6.76e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 18, Val Loss: 0.08193146803132897, Val Accuracy: 0.9775079617834395\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.59it/s, accuracy=1, loss=0.000816, lr=5.4e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 19, Val Loss: 0.0834959355546987, Val Accuracy: 0.9772093949044586\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.54it/s, accuracy=1, loss=0.000538, lr=4.32e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 20, Val Loss: 0.0826277518857936, Val Accuracy: 0.9777070063694268\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.79it/s, accuracy=1, loss=0.000247, lr=3.46e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 21, Val Loss: 0.08293034021866982, Val Accuracy: 0.9773089171974523\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.27it/s, accuracy=1, loss=0.000326, lr=2.77e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 22, Val Loss: 0.08310823540747024, Val Accuracy: 0.9774084394904459\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.67it/s, accuracy=1, loss=0.000694, lr=2.21e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 23, Val Loss: 0.0836875414852677, Val Accuracy: 0.9774084394904459\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.92it/s, accuracy=1, loss=0.000441, lr=1.77e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 24, Val Loss: 0.08375426088161568, Val Accuracy: 0.9774084394904459\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.30it/s, accuracy=1, loss=0.000272, lr=1.42e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 25, Val Loss: 0.08395082133682509, Val Accuracy: 0.9773089171974523\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.82it/s, accuracy=1, loss=0.000293, lr=1.13e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 26, Val Loss: 0.08402066259568064, Val Accuracy: 0.9776074840764332\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.82it/s, accuracy=1, loss=0.000161, lr=9.07e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 27, Val Loss: 0.08402630906480545, Val Accuracy: 0.9774084394904459\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.85it/s, accuracy=1, loss=0.000869, lr=7.25e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 28, Val Loss: 0.08417512387783573, Val Accuracy: 0.9774084394904459\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.33it/s, accuracy=1, loss=0.000636, lr=5.8e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 29, Val Loss: 0.08420315355955041, Val Accuracy: 0.9775079617834395\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.99it/s, accuracy=1, loss=0.000438, lr=4.64e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 30, Val Loss: 0.08418731019465106, Val Accuracy: 0.9775079617834395\n",
            "LR: 0.0003 Gamma: 0.8 Hdim: 128 Best Accuracy: 0.9777070063694268\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.29it/s, accuracy=1, loss=0.0392, lr=0.0003]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1, Val Loss: 0.16471018213600774, Val Accuracy: 0.9527269108280255\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.21it/s, accuracy=0.875, loss=0.386, lr=0.00024]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2, Val Loss: 0.12116482815924723, Val Accuracy: 0.9626791401273885\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.32it/s, accuracy=1, loss=0.0398, lr=0.000192]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3, Val Loss: 0.10116630336447696, Val Accuracy: 0.9669585987261147\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.90it/s, accuracy=0.969, loss=0.0505, lr=0.000154]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4, Val Loss: 0.09222686409380786, Val Accuracy: 0.9699442675159236\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.13it/s, accuracy=1, loss=0.00276, lr=0.000123]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, Val Loss: 0.0743863805915321, Val Accuracy: 0.9764132165605095\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.09it/s, accuracy=1, loss=0.0174, lr=9.83e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6, Val Loss: 0.07101520900323521, Val Accuracy: 0.9790007961783439\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.16it/s, accuracy=1, loss=0.00117, lr=7.86e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7, Val Loss: 0.0686128812140057, Val Accuracy: 0.9786027070063694\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.81it/s, accuracy=1, loss=0.00161, lr=6.29e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8, Val Loss: 0.06804558684207859, Val Accuracy: 0.979796974522293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.94it/s, accuracy=1, loss=0.000453, lr=5.03e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9, Val Loss: 0.06683609943665594, Val Accuracy: 0.979796974522293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.63it/s, accuracy=1, loss=0.000637, lr=4.03e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10, Val Loss: 0.07164423754513835, Val Accuracy: 0.9794984076433121\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.79it/s, accuracy=1, loss=0.00288, lr=3.22e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 11, Val Loss: 0.06885640565621201, Val Accuracy: 0.9788017515923567\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.44it/s, accuracy=1, loss=0.000664, lr=2.58e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 12, Val Loss: 0.06825677690413746, Val Accuracy: 0.9804936305732485\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.79it/s, accuracy=1, loss=0.00154, lr=2.06e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 13, Val Loss: 0.06945997688850346, Val Accuracy: 0.979796974522293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.89it/s, accuracy=1, loss=4.94e-5, lr=1.65e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 14, Val Loss: 0.06771141214370233, Val Accuracy: 0.9800955414012739\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.60it/s, accuracy=1, loss=0.000137, lr=1.32e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 15, Val Loss: 0.07027644031889504, Val Accuracy: 0.9801950636942676\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.48it/s, accuracy=1, loss=0.00019, lr=1.06e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 16, Val Loss: 0.0694659169807747, Val Accuracy: 0.9803941082802548\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.35it/s, accuracy=1, loss=0.000492, lr=8.44e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 17, Val Loss: 0.07038373093013532, Val Accuracy: 0.9801950636942676\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.21it/s, accuracy=1, loss=0.000773, lr=6.76e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 18, Val Loss: 0.07094030952960241, Val Accuracy: 0.979796974522293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.51it/s, accuracy=1, loss=7.97e-5, lr=5.4e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 19, Val Loss: 0.07042696972190202, Val Accuracy: 0.979796974522293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.25it/s, accuracy=1, loss=0.000393, lr=4.32e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 20, Val Loss: 0.0711350089873998, Val Accuracy: 0.9799960191082803\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.11it/s, accuracy=1, loss=0.000336, lr=3.46e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 21, Val Loss: 0.0712487457694147, Val Accuracy: 0.9803941082802548\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.55it/s, accuracy=1, loss=0.000178, lr=2.77e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 22, Val Loss: 0.07233508868759649, Val Accuracy: 0.9796974522292994\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.99it/s, accuracy=1, loss=0.000155, lr=2.21e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 23, Val Loss: 0.0719130766572737, Val Accuracy: 0.9798964968152867\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.53it/s, accuracy=1, loss=7.06e-5, lr=1.77e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 24, Val Loss: 0.07232012049186393, Val Accuracy: 0.9795979299363057\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.23it/s, accuracy=1, loss=9.21e-5, lr=1.42e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 25, Val Loss: 0.07241453965781411, Val Accuracy: 0.979796974522293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.27it/s, accuracy=1, loss=6.99e-5, lr=1.13e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 26, Val Loss: 0.072295983366005, Val Accuracy: 0.979796974522293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.30it/s, accuracy=1, loss=0.000361, lr=9.07e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 27, Val Loss: 0.07245994138554723, Val Accuracy: 0.979796974522293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.23it/s, accuracy=1, loss=9.63e-5, lr=7.25e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 28, Val Loss: 0.0727691285845409, Val Accuracy: 0.9796974522292994\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.10it/s, accuracy=1, loss=0.000327, lr=5.8e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 29, Val Loss: 0.07274005139501384, Val Accuracy: 0.9796974522292994\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.80it/s, accuracy=1, loss=0.000224, lr=4.64e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 30, Val Loss: 0.07263387687999275, Val Accuracy: 0.9796974522292994\n",
            "LR: 0.0003 Gamma: 0.8 Hdim: 256 Best Accuracy: 0.9804936305732485\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.32it/s, accuracy=0.969, loss=0.0922, lr=0.0003]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1, Val Loss: 0.21242379619603513, Val Accuracy: 0.9362062101910829\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.08it/s, accuracy=0.938, loss=0.237, lr=0.00027]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2, Val Loss: 0.1738557616058903, Val Accuracy: 0.9488455414012739\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.33it/s, accuracy=0.969, loss=0.152, lr=0.000243]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3, Val Loss: 0.1369738911788697, Val Accuracy: 0.9584992038216561\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.24it/s, accuracy=1, loss=0.0257, lr=0.000219]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4, Val Loss: 0.10879912294423694, Val Accuracy: 0.9658638535031847\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.03it/s, accuracy=1, loss=0.016, lr=0.000197]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, Val Loss: 0.0998964802555556, Val Accuracy: 0.9693471337579618\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.28it/s, accuracy=1, loss=0.0105, lr=0.000177]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6, Val Loss: 0.10114622348622912, Val Accuracy: 0.9694466560509554\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.04it/s, accuracy=1, loss=0.0458, lr=0.000159]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7, Val Loss: 0.10004394527029388, Val Accuracy: 0.9692476114649682\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.55it/s, accuracy=1, loss=0.0127, lr=0.000143]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8, Val Loss: 0.10263982507191483, Val Accuracy: 0.9681528662420382\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.66it/s, accuracy=0.969, loss=0.059, lr=0.000129]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9, Val Loss: 0.10853677761153716, Val Accuracy: 0.9669585987261147\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.68it/s, accuracy=1, loss=0.00356, lr=0.000116]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10, Val Loss: 0.10298777253176628, Val Accuracy: 0.9696457006369427\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.96it/s, accuracy=1, loss=0.00234, lr=0.000105]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 11, Val Loss: 0.09931600652266666, Val Accuracy: 0.9719347133757962\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.07it/s, accuracy=1, loss=0.0237, lr=9.41e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 12, Val Loss: 0.1027903118552021, Val Accuracy: 0.9701433121019108\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.27it/s, accuracy=1, loss=0.000948, lr=8.47e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 13, Val Loss: 0.09904193292440702, Val Accuracy: 0.9721337579617835\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.53it/s, accuracy=1, loss=0.00141, lr=7.63e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 14, Val Loss: 0.10204604699563811, Val Accuracy: 0.9726313694267515\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.13it/s, accuracy=1, loss=0.00212, lr=6.86e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 15, Val Loss: 0.11387269527844252, Val Accuracy: 0.9690485668789809\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.72it/s, accuracy=1, loss=0.000472, lr=6.18e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 16, Val Loss: 0.10276446662843737, Val Accuracy: 0.9726313694267515\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.29it/s, accuracy=1, loss=0.00103, lr=5.56e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 17, Val Loss: 0.10210566523449757, Val Accuracy: 0.9737261146496815\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.85it/s, accuracy=1, loss=0.000708, lr=5e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 18, Val Loss: 0.1030298106688513, Val Accuracy: 0.9729299363057324\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.77it/s, accuracy=1, loss=0.000412, lr=4.5e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 19, Val Loss: 0.10727652960732212, Val Accuracy: 0.973328025477707\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.01it/s, accuracy=1, loss=0.000898, lr=4.05e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 20, Val Loss: 0.10945995187289641, Val Accuracy: 0.972531847133758\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.81it/s, accuracy=1, loss=0.000469, lr=3.65e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 21, Val Loss: 0.11105900724615607, Val Accuracy: 0.9722332802547771\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.35it/s, accuracy=1, loss=0.000655, lr=3.28e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 22, Val Loss: 0.1140304070314953, Val Accuracy: 0.972531847133758\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 46.97it/s, accuracy=1, loss=0.000155, lr=2.95e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 23, Val Loss: 0.11360263955808987, Val Accuracy: 0.9734275477707006\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.80it/s, accuracy=1, loss=5.29e-5, lr=2.66e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 24, Val Loss: 0.11399504847515786, Val Accuracy: 0.9727308917197452\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.69it/s, accuracy=1, loss=9.98e-5, lr=2.39e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 25, Val Loss: 0.1166743240382495, Val Accuracy: 0.9727308917197452\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.74it/s, accuracy=1, loss=0.00051, lr=2.15e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 26, Val Loss: 0.11776413911066973, Val Accuracy: 0.9732285031847133\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.86it/s, accuracy=1, loss=0.000131, lr=1.94e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 27, Val Loss: 0.11880176255582375, Val Accuracy: 0.9726313694267515\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.86it/s, accuracy=1, loss=8.3e-5, lr=1.74e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 28, Val Loss: 0.12004407612722928, Val Accuracy: 0.9730294585987261\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.41it/s, accuracy=1, loss=9.13e-6, lr=1.57e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 29, Val Loss: 0.1210802539070622, Val Accuracy: 0.973328025477707\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.26it/s, accuracy=1, loss=4.59e-5, lr=1.41e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 30, Val Loss: 0.12243362093643448, Val Accuracy: 0.9721337579617835\n",
            "LR: 0.0003 Gamma: 0.9 Hdim: 64 Best Accuracy: 0.9737261146496815\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.34it/s, accuracy=0.906, loss=0.214, lr=0.0003]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1, Val Loss: 0.2039622921188166, Val Accuracy: 0.9422770700636943\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.27it/s, accuracy=0.969, loss=0.0992, lr=0.00027]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2, Val Loss: 0.1307590758772033, Val Accuracy: 0.9592953821656051\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.53it/s, accuracy=0.938, loss=0.163, lr=0.000243]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3, Val Loss: 0.1111108220336591, Val Accuracy: 0.9648686305732485\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.32it/s, accuracy=1, loss=0.0275, lr=0.000219]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4, Val Loss: 0.10274220236568196, Val Accuracy: 0.9671576433121019\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.65it/s, accuracy=1, loss=0.0188, lr=0.000197]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, Val Loss: 0.10456022050507532, Val Accuracy: 0.9665605095541401\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.71it/s, accuracy=1, loss=0.00725, lr=0.000177]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6, Val Loss: 0.0893607784419841, Val Accuracy: 0.9727308917197452\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.26it/s, accuracy=1, loss=0.0195, lr=0.000159]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7, Val Loss: 0.08576193410033066, Val Accuracy: 0.9737261146496815\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.29it/s, accuracy=1, loss=0.0102, lr=0.000143]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8, Val Loss: 0.08693787118084106, Val Accuracy: 0.9739251592356688\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.79it/s, accuracy=1, loss=0.00337, lr=0.000129]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9, Val Loss: 0.09332404992021145, Val Accuracy: 0.972531847133758\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.27it/s, accuracy=1, loss=0.00101, lr=0.000116]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10, Val Loss: 0.08186101354770987, Val Accuracy: 0.9765127388535032\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.26it/s, accuracy=1, loss=0.00142, lr=0.000105]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 11, Val Loss: 0.08606895735531554, Val Accuracy: 0.9751194267515924\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.92it/s, accuracy=1, loss=0.00327, lr=9.41e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 12, Val Loss: 0.08235289113205799, Val Accuracy: 0.9780055732484076\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.76it/s, accuracy=1, loss=0.000355, lr=8.47e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 13, Val Loss: 0.08097155100075012, Val Accuracy: 0.9786027070063694\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.37it/s, accuracy=1, loss=0.000984, lr=7.63e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 14, Val Loss: 0.08279497241114392, Val Accuracy: 0.9783041401273885\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.07it/s, accuracy=1, loss=0.000393, lr=6.86e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 15, Val Loss: 0.08526106219072288, Val Accuracy: 0.977906050955414\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.73it/s, accuracy=1, loss=9e-5, lr=6.18e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 16, Val Loss: 0.08471209791186819, Val Accuracy: 0.978702229299363\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.85it/s, accuracy=1, loss=0.000295, lr=5.56e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 17, Val Loss: 0.08538869117429693, Val Accuracy: 0.9776074840764332\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.22it/s, accuracy=1, loss=0.000112, lr=5e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 18, Val Loss: 0.0869361197029251, Val Accuracy: 0.977906050955414\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.31it/s, accuracy=1, loss=0.00019, lr=4.5e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 19, Val Loss: 0.08969325105135216, Val Accuracy: 0.9777070063694268\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.19it/s, accuracy=1, loss=0.000131, lr=4.05e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 20, Val Loss: 0.08879337150380655, Val Accuracy: 0.9783041401273885\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.35it/s, accuracy=1, loss=4.78e-5, lr=3.65e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 21, Val Loss: 0.0906782049277003, Val Accuracy: 0.9775079617834395\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.22it/s, accuracy=1, loss=5.46e-5, lr=3.28e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 22, Val Loss: 0.09186842004820323, Val Accuracy: 0.9785031847133758\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.56it/s, accuracy=1, loss=6.27e-5, lr=2.95e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 23, Val Loss: 0.09360769132327677, Val Accuracy: 0.9789012738853503\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.07it/s, accuracy=1, loss=4.67e-5, lr=2.66e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 24, Val Loss: 0.09622574574900704, Val Accuracy: 0.9786027070063694\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.79it/s, accuracy=1, loss=2.28e-5, lr=2.39e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 25, Val Loss: 0.09623206524740198, Val Accuracy: 0.9782046178343949\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.88it/s, accuracy=1, loss=1.44e-5, lr=2.15e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 26, Val Loss: 0.09901059986447099, Val Accuracy: 0.9780055732484076\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.68it/s, accuracy=1, loss=7.35e-6, lr=1.94e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 27, Val Loss: 0.10043074637891493, Val Accuracy: 0.9778065286624203\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.43it/s, accuracy=1, loss=2.47e-5, lr=1.74e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 28, Val Loss: 0.10302471094988462, Val Accuracy: 0.9778065286624203\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.94it/s, accuracy=1, loss=9.82e-6, lr=1.57e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 29, Val Loss: 0.10411269281212607, Val Accuracy: 0.9781050955414012\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.79it/s, accuracy=1, loss=8.11e-6, lr=1.41e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 30, Val Loss: 0.10565648086569845, Val Accuracy: 0.9773089171974523\n",
            "LR: 0.0003 Gamma: 0.9 Hdim: 128 Best Accuracy: 0.9789012738853503\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.69it/s, accuracy=1, loss=0.0392, lr=0.0003]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1, Val Loss: 0.16471018213600774, Val Accuracy: 0.9527269108280255\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.96it/s, accuracy=0.906, loss=0.352, lr=0.00027]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2, Val Loss: 0.12285312816684556, Val Accuracy: 0.9626791401273885\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.38it/s, accuracy=1, loss=0.0388, lr=0.000243]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3, Val Loss: 0.09845568498175378, Val Accuracy: 0.96984474522293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.63it/s, accuracy=0.969, loss=0.074, lr=0.000219]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4, Val Loss: 0.10747340274826404, Val Accuracy: 0.9654657643312102\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.43it/s, accuracy=1, loss=0.0163, lr=0.000197]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, Val Loss: 0.0817872400589845, Val Accuracy: 0.9755175159235668\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.19it/s, accuracy=1, loss=0.0153, lr=0.000177]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6, Val Loss: 0.07925048755056589, Val Accuracy: 0.9767117834394905\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.25it/s, accuracy=1, loss=0.00066, lr=0.000159]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7, Val Loss: 0.07694087669527333, Val Accuracy: 0.9772093949044586\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.26it/s, accuracy=1, loss=0.00131, lr=0.000143]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8, Val Loss: 0.08311416937224975, Val Accuracy: 0.9762141719745223\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.26it/s, accuracy=1, loss=0.0131, lr=0.000129]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9, Val Loss: 0.10246736460619689, Val Accuracy: 0.9685509554140127\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.10it/s, accuracy=1, loss=0.000343, lr=0.000116]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10, Val Loss: 0.08485364867356922, Val Accuracy: 0.977109872611465\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.50it/s, accuracy=1, loss=0.00119, lr=0.000105]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 11, Val Loss: 0.07360146786829035, Val Accuracy: 0.9799960191082803\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.23it/s, accuracy=1, loss=0.000222, lr=9.41e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 12, Val Loss: 0.07130190466578958, Val Accuracy: 0.9798964968152867\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.80it/s, accuracy=1, loss=0.0111, lr=8.47e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 13, Val Loss: 0.07933866384200593, Val Accuracy: 0.9769108280254777\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.84it/s, accuracy=1, loss=3.62e-5, lr=7.63e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 14, Val Loss: 0.07317673625466833, Val Accuracy: 0.9796974522292994\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.74it/s, accuracy=1, loss=4.76e-5, lr=6.86e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 15, Val Loss: 0.07323052724814767, Val Accuracy: 0.9798964968152867\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.03it/s, accuracy=1, loss=0.000176, lr=6.18e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 16, Val Loss: 0.0716191473816455, Val Accuracy: 0.9806926751592356\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.80it/s, accuracy=1, loss=0.000432, lr=5.56e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 17, Val Loss: 0.07324867912111484, Val Accuracy: 0.9799960191082803\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.59it/s, accuracy=1, loss=0.000482, lr=5e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 18, Val Loss: 0.07419233228493385, Val Accuracy: 0.9807921974522293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.75it/s, accuracy=1, loss=0.000161, lr=4.5e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 19, Val Loss: 0.07085067796125856, Val Accuracy: 0.9808917197452229\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.74it/s, accuracy=1, loss=0.000268, lr=4.05e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 20, Val Loss: 0.07273320861962898, Val Accuracy: 0.9810907643312102\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.45it/s, accuracy=1, loss=0.000164, lr=3.65e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 21, Val Loss: 0.07327132297988538, Val Accuracy: 0.980593152866242\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.08it/s, accuracy=1, loss=9.68e-5, lr=3.28e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 22, Val Loss: 0.07417852014123598, Val Accuracy: 0.9808917197452229\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.73it/s, accuracy=1, loss=0.000113, lr=2.95e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 23, Val Loss: 0.07389430299422434, Val Accuracy: 0.9812898089171974\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.89it/s, accuracy=1, loss=5.95e-5, lr=2.66e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 24, Val Loss: 0.07467686537025384, Val Accuracy: 0.980593152866242\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.92it/s, accuracy=1, loss=4.18e-5, lr=2.39e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 25, Val Loss: 0.0753023124710938, Val Accuracy: 0.9815883757961783\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.10it/s, accuracy=1, loss=1.87e-5, lr=2.15e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 26, Val Loss: 0.0761238395397765, Val Accuracy: 0.980593152866242\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.84it/s, accuracy=1, loss=0.000128, lr=1.94e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 27, Val Loss: 0.07686795543632768, Val Accuracy: 0.9810907643312102\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.08it/s, accuracy=1, loss=3.93e-5, lr=1.74e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 28, Val Loss: 0.0786214491741156, Val Accuracy: 0.9806926751592356\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.32it/s, accuracy=1, loss=5.03e-5, lr=1.57e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 29, Val Loss: 0.07903076196528955, Val Accuracy: 0.9806926751592356\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.12it/s, accuracy=1, loss=4.38e-5, lr=1.41e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 30, Val Loss: 0.07852090576356983, Val Accuracy: 0.9810907643312102\n",
            "LR: 0.0003 Gamma: 0.9 Hdim: 256 Best Accuracy: 0.9815883757961783\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.61it/s, accuracy=1, loss=0.0678, lr=0.0004]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1, Val Loss: 0.19725431118658773, Val Accuracy: 0.9415804140127388\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.33it/s, accuracy=0.938, loss=0.19, lr=0.00032]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2, Val Loss: 0.16660313830847384, Val Accuracy: 0.9484474522292994\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.92it/s, accuracy=0.969, loss=0.141, lr=0.000256]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3, Val Loss: 0.1383658179295528, Val Accuracy: 0.9538216560509554\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.69it/s, accuracy=1, loss=0.0325, lr=0.000205]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4, Val Loss: 0.10713015987825503, Val Accuracy: 0.9661624203821656\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.99it/s, accuracy=1, loss=0.0224, lr=0.000164]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, Val Loss: 0.09784169134514845, Val Accuracy: 0.96984474522293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.80it/s, accuracy=1, loss=0.00492, lr=0.000131]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6, Val Loss: 0.09761385792168599, Val Accuracy: 0.9708399681528662\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.23it/s, accuracy=1, loss=0.0214, lr=0.000105]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7, Val Loss: 0.09257911612720164, Val Accuracy: 0.9727308917197452\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.67it/s, accuracy=1, loss=0.00742, lr=8.39e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8, Val Loss: 0.0918645332675838, Val Accuracy: 0.971437101910828\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.70it/s, accuracy=1, loss=0.018, lr=6.71e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9, Val Loss: 0.0933313427844632, Val Accuracy: 0.9732285031847133\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.07it/s, accuracy=1, loss=0.00392, lr=5.37e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10, Val Loss: 0.0930917151242637, Val Accuracy: 0.9739251592356688\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.36it/s, accuracy=1, loss=0.00266, lr=4.29e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 11, Val Loss: 0.09302061549728034, Val Accuracy: 0.9722332802547771\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.33it/s, accuracy=1, loss=0.00785, lr=3.44e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 12, Val Loss: 0.09182939219285191, Val Accuracy: 0.9735270700636943\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.22it/s, accuracy=1, loss=0.00391, lr=2.75e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 13, Val Loss: 0.09484100429307644, Val Accuracy: 0.9724323248407644\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.92it/s, accuracy=1, loss=0.00317, lr=2.2e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 14, Val Loss: 0.09417811687449479, Val Accuracy: 0.972531847133758\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.90it/s, accuracy=1, loss=0.00118, lr=1.76e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 15, Val Loss: 0.09597670314090632, Val Accuracy: 0.9727308917197452\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.99it/s, accuracy=1, loss=0.00112, lr=1.41e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 16, Val Loss: 0.0951487971221126, Val Accuracy: 0.9731289808917197\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.26it/s, accuracy=1, loss=0.0035, lr=1.13e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 17, Val Loss: 0.0953536253685221, Val Accuracy: 0.973328025477707\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.42it/s, accuracy=1, loss=0.00153, lr=9.01e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 18, Val Loss: 0.0950373403254515, Val Accuracy: 0.9736265923566879\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.51it/s, accuracy=1, loss=0.00214, lr=7.21e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 19, Val Loss: 0.09717631226440815, Val Accuracy: 0.9730294585987261\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.38it/s, accuracy=1, loss=0.00284, lr=5.76e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 20, Val Loss: 0.09709021062162199, Val Accuracy: 0.9730294585987261\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.98it/s, accuracy=1, loss=0.00248, lr=4.61e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 21, Val Loss: 0.09695101388382346, Val Accuracy: 0.9730294585987261\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.11it/s, accuracy=1, loss=0.00247, lr=3.69e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 22, Val Loss: 0.09773216684688858, Val Accuracy: 0.9729299363057324\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.08it/s, accuracy=1, loss=0.000533, lr=2.95e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 23, Val Loss: 0.09721937070372307, Val Accuracy: 0.9734275477707006\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.43it/s, accuracy=1, loss=0.000461, lr=2.36e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 24, Val Loss: 0.09765914737873127, Val Accuracy: 0.9732285031847133\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.20it/s, accuracy=1, loss=0.00107, lr=1.89e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 25, Val Loss: 0.09772177176965224, Val Accuracy: 0.9731289808917197\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.55it/s, accuracy=1, loss=0.00392, lr=1.51e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 26, Val Loss: 0.09766424810408886, Val Accuracy: 0.9729299363057324\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.45it/s, accuracy=1, loss=0.00126, lr=1.21e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 27, Val Loss: 0.09812338375555717, Val Accuracy: 0.9729299363057324\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.17it/s, accuracy=1, loss=0.001, lr=9.67e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 28, Val Loss: 0.09795121750591829, Val Accuracy: 0.9729299363057324\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.33it/s, accuracy=1, loss=0.000162, lr=7.74e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 29, Val Loss: 0.09792875765497171, Val Accuracy: 0.9730294585987261\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.66it/s, accuracy=1, loss=0.000503, lr=6.19e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 30, Val Loss: 0.09805970445978247, Val Accuracy: 0.9730294585987261\n",
            "LR: 0.0004 Gamma: 0.8 Hdim: 64 Best Accuracy: 0.9739251592356688\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.09it/s, accuracy=0.906, loss=0.24, lr=0.0004]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1, Val Loss: 0.20275159292872164, Val Accuracy: 0.9435708598726115\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.78it/s, accuracy=0.938, loss=0.139, lr=0.00032]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2, Val Loss: 0.13563353877372233, Val Accuracy: 0.9598925159235668\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.70it/s, accuracy=0.969, loss=0.106, lr=0.000256]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3, Val Loss: 0.10851508905372588, Val Accuracy: 0.9671576433121019\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.87it/s, accuracy=1, loss=0.00893, lr=0.000205]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4, Val Loss: 0.09213124637107904, Val Accuracy: 0.9718351910828026\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.58it/s, accuracy=1, loss=0.0123, lr=0.000164]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, Val Loss: 0.092034742983939, Val Accuracy: 0.9721337579617835\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.51it/s, accuracy=1, loss=0.00192, lr=0.000131]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6, Val Loss: 0.08882025063295913, Val Accuracy: 0.9735270700636943\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.55it/s, accuracy=1, loss=0.00914, lr=0.000105]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7, Val Loss: 0.07704540160068446, Val Accuracy: 0.9772093949044586\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.57it/s, accuracy=1, loss=0.00166, lr=8.39e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8, Val Loss: 0.07907696739389712, Val Accuracy: 0.9764132165605095\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.09it/s, accuracy=1, loss=0.000323, lr=6.71e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9, Val Loss: 0.07766282245869267, Val Accuracy: 0.9776074840764332\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.19it/s, accuracy=1, loss=0.00306, lr=5.37e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10, Val Loss: 0.07780515086818297, Val Accuracy: 0.977906050955414\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.30it/s, accuracy=1, loss=0.00116, lr=4.29e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 11, Val Loss: 0.07967773055772259, Val Accuracy: 0.9781050955414012\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.99it/s, accuracy=1, loss=0.00252, lr=3.44e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 12, Val Loss: 0.0795639719790589, Val Accuracy: 0.9776074840764332\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.91it/s, accuracy=1, loss=0.00035, lr=2.75e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 13, Val Loss: 0.08002508312819472, Val Accuracy: 0.9784036624203821\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.42it/s, accuracy=1, loss=0.0016, lr=2.2e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 14, Val Loss: 0.08152372163868868, Val Accuracy: 0.9783041401273885\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.23it/s, accuracy=1, loss=0.000454, lr=1.76e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 15, Val Loss: 0.08311937773486865, Val Accuracy: 0.9777070063694268\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.47it/s, accuracy=1, loss=0.000375, lr=1.41e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 16, Val Loss: 0.08176963093029563, Val Accuracy: 0.977906050955414\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.93it/s, accuracy=1, loss=0.000178, lr=1.13e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 17, Val Loss: 0.08236415773734833, Val Accuracy: 0.9781050955414012\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.81it/s, accuracy=1, loss=0.000479, lr=9.01e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 18, Val Loss: 0.08345970529381604, Val Accuracy: 0.9782046178343949\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.68it/s, accuracy=1, loss=0.00062, lr=7.21e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 19, Val Loss: 0.08496979389354811, Val Accuracy: 0.9783041401273885\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.99it/s, accuracy=1, loss=0.000271, lr=5.76e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 20, Val Loss: 0.0844229854749721, Val Accuracy: 0.9782046178343949\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.16it/s, accuracy=1, loss=0.000157, lr=4.61e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 21, Val Loss: 0.08472629088501821, Val Accuracy: 0.9784036624203821\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.68it/s, accuracy=1, loss=0.000119, lr=3.69e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 22, Val Loss: 0.08476485933038107, Val Accuracy: 0.9785031847133758\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.85it/s, accuracy=1, loss=0.000425, lr=2.95e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 23, Val Loss: 0.08559241194262732, Val Accuracy: 0.9781050955414012\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.10it/s, accuracy=1, loss=0.000264, lr=2.36e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 24, Val Loss: 0.08562439445101842, Val Accuracy: 0.9780055732484076\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.30it/s, accuracy=1, loss=0.000187, lr=1.89e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 25, Val Loss: 0.08597187649311425, Val Accuracy: 0.9781050955414012\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 40.94it/s, accuracy=1, loss=0.000104, lr=1.51e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 26, Val Loss: 0.08587202335905487, Val Accuracy: 0.977906050955414\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.97it/s, accuracy=1, loss=3.95e-5, lr=1.21e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 27, Val Loss: 0.08588361378174254, Val Accuracy: 0.9776074840764332\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 40.84it/s, accuracy=1, loss=0.000586, lr=9.67e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 28, Val Loss: 0.08612286774237514, Val Accuracy: 0.9780055732484076\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.49it/s, accuracy=1, loss=0.000311, lr=7.74e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 29, Val Loss: 0.08611973887192993, Val Accuracy: 0.9781050955414012\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.36it/s, accuracy=1, loss=0.000183, lr=6.19e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 30, Val Loss: 0.0861475913214664, Val Accuracy: 0.977906050955414\n",
            "LR: 0.0004 Gamma: 0.8 Hdim: 128 Best Accuracy: 0.9785031847133758\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.78it/s, accuracy=1, loss=0.0692, lr=0.0004]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1, Val Loss: 0.16268330219848331, Val Accuracy: 0.9529259554140127\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.31it/s, accuracy=0.938, loss=0.332, lr=0.00032]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2, Val Loss: 0.1287411227125295, Val Accuracy: 0.9615843949044586\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.20it/s, accuracy=1, loss=0.0238, lr=0.000256]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3, Val Loss: 0.09555978241382511, Val Accuracy: 0.9702428343949044\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.34it/s, accuracy=1, loss=0.0321, lr=0.000205]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4, Val Loss: 0.09014713520163398, Val Accuracy: 0.9705414012738853\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.97it/s, accuracy=1, loss=0.0052, lr=0.000164]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, Val Loss: 0.07925123760623526, Val Accuracy: 0.9766122611464968\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.12it/s, accuracy=1, loss=0.0182, lr=0.000131]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6, Val Loss: 0.0823169757307479, Val Accuracy: 0.9749203821656051\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.80it/s, accuracy=1, loss=0.00104, lr=0.000105]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7, Val Loss: 0.07404835354109483, Val Accuracy: 0.9778065286624203\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.77it/s, accuracy=1, loss=0.00186, lr=8.39e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8, Val Loss: 0.07570973285138355, Val Accuracy: 0.9781050955414012\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.32it/s, accuracy=1, loss=0.000216, lr=6.71e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9, Val Loss: 0.07106834768297156, Val Accuracy: 0.9799960191082803\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.67it/s, accuracy=1, loss=0.000697, lr=5.37e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10, Val Loss: 0.0740012628362055, Val Accuracy: 0.979796974522293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.32it/s, accuracy=1, loss=0.00447, lr=4.29e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 11, Val Loss: 0.07270031930113925, Val Accuracy: 0.9796974522292994\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.15it/s, accuracy=1, loss=0.000454, lr=3.44e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 12, Val Loss: 0.07450439892798119, Val Accuracy: 0.9795979299363057\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.32it/s, accuracy=1, loss=0.0011, lr=2.75e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 13, Val Loss: 0.07592495395671243, Val Accuracy: 0.9794984076433121\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.38it/s, accuracy=1, loss=5.98e-5, lr=2.2e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 14, Val Loss: 0.07524816155270543, Val Accuracy: 0.9794984076433121\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.81it/s, accuracy=1, loss=3.97e-5, lr=1.76e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 15, Val Loss: 0.07696167450798976, Val Accuracy: 0.979796974522293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 40.92it/s, accuracy=1, loss=0.000263, lr=1.41e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 16, Val Loss: 0.07635268961448702, Val Accuracy: 0.9806926751592356\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.95it/s, accuracy=1, loss=0.000283, lr=1.13e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 17, Val Loss: 0.07744945313746197, Val Accuracy: 0.9795979299363057\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.32it/s, accuracy=1, loss=0.000548, lr=9.01e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 18, Val Loss: 0.0777232088557461, Val Accuracy: 0.9803941082802548\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.10it/s, accuracy=1, loss=7.92e-5, lr=7.21e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 19, Val Loss: 0.07829364135196197, Val Accuracy: 0.9803941082802548\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 40.99it/s, accuracy=1, loss=0.000204, lr=5.76e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 20, Val Loss: 0.07911044827998666, Val Accuracy: 0.9799960191082803\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.27it/s, accuracy=1, loss=0.000163, lr=4.61e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 21, Val Loss: 0.0790494765206905, Val Accuracy: 0.9804936305732485\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.23it/s, accuracy=1, loss=0.000134, lr=3.69e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 22, Val Loss: 0.08005832356172163, Val Accuracy: 0.9802945859872612\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.44it/s, accuracy=1, loss=7.92e-5, lr=2.95e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 23, Val Loss: 0.0796989527467321, Val Accuracy: 0.9802945859872612\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.62it/s, accuracy=1, loss=3.71e-5, lr=2.36e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 24, Val Loss: 0.08029239156710728, Val Accuracy: 0.9801950636942676\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.02it/s, accuracy=1, loss=3.47e-5, lr=1.89e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 25, Val Loss: 0.08027065497207445, Val Accuracy: 0.9804936305732485\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.37it/s, accuracy=1, loss=4.1e-5, lr=1.51e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 26, Val Loss: 0.0805483429258016, Val Accuracy: 0.9802945859872612\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.14it/s, accuracy=1, loss=0.000181, lr=1.21e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 27, Val Loss: 0.0806374167524534, Val Accuracy: 0.9800955414012739\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.13it/s, accuracy=1, loss=8.03e-5, lr=9.67e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 28, Val Loss: 0.08096183790259484, Val Accuracy: 0.9801950636942676\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.12it/s, accuracy=1, loss=0.000179, lr=7.74e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 29, Val Loss: 0.08095589866335035, Val Accuracy: 0.9802945859872612\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.70it/s, accuracy=1, loss=0.000106, lr=6.19e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 30, Val Loss: 0.08085988526821658, Val Accuracy: 0.9804936305732485\n",
            "LR: 0.0004 Gamma: 0.8 Hdim: 256 Best Accuracy: 0.9806926751592356\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.24it/s, accuracy=1, loss=0.0678, lr=0.0004]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1, Val Loss: 0.19725431118658773, Val Accuracy: 0.9415804140127388\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.42it/s, accuracy=0.938, loss=0.214, lr=0.00036]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2, Val Loss: 0.16720716872365232, Val Accuracy: 0.9478503184713376\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.55it/s, accuracy=0.938, loss=0.164, lr=0.000324]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3, Val Loss: 0.1446358206842071, Val Accuracy: 0.9522292993630573\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.09it/s, accuracy=1, loss=0.0217, lr=0.000292]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4, Val Loss: 0.11202807595158458, Val Accuracy: 0.9650676751592356\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.53it/s, accuracy=1, loss=0.0171, lr=0.000262]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, Val Loss: 0.09896334884562204, Val Accuracy: 0.9692476114649682\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.38it/s, accuracy=1, loss=0.00264, lr=0.000236]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6, Val Loss: 0.09940300237435246, Val Accuracy: 0.9707404458598726\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.24it/s, accuracy=1, loss=0.014, lr=0.000213]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7, Val Loss: 0.1019423350124178, Val Accuracy: 0.96875\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.94it/s, accuracy=1, loss=0.00646, lr=0.000191]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8, Val Loss: 0.09589939816580725, Val Accuracy: 0.9709394904458599\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.20it/s, accuracy=1, loss=0.029, lr=0.000172]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9, Val Loss: 0.10795281166570397, Val Accuracy: 0.9677547770700637\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.26it/s, accuracy=1, loss=0.0051, lr=0.000155]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10, Val Loss: 0.10181578172420472, Val Accuracy: 0.9734275477707006\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.43it/s, accuracy=1, loss=0.00429, lr=0.000139]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 11, Val Loss: 0.11272754185488425, Val Accuracy: 0.9672571656050956\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.48it/s, accuracy=1, loss=0.00954, lr=0.000126]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 12, Val Loss: 0.0979401887973135, Val Accuracy: 0.9738256369426752\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.52it/s, accuracy=1, loss=0.00145, lr=0.000113]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 13, Val Loss: 0.09883755047427535, Val Accuracy: 0.9729299363057324\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.30it/s, accuracy=1, loss=0.00496, lr=0.000102]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 14, Val Loss: 0.1239667671149045, Val Accuracy: 0.9651671974522293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.86it/s, accuracy=1, loss=0.000803, lr=9.15e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 15, Val Loss: 0.10189435796316276, Val Accuracy: 0.9729299363057324\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.67it/s, accuracy=1, loss=0.000403, lr=8.24e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 16, Val Loss: 0.10081551412346758, Val Accuracy: 0.9739251592356688\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.30it/s, accuracy=1, loss=0.00102, lr=7.41e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 17, Val Loss: 0.10154626253024096, Val Accuracy: 0.9737261146496815\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.72it/s, accuracy=1, loss=0.00037, lr=6.67e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 18, Val Loss: 0.1014739672379625, Val Accuracy: 0.974422770700637\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.10it/s, accuracy=1, loss=0.000844, lr=6e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 19, Val Loss: 0.10789653310747647, Val Accuracy: 0.972531847133758\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.38it/s, accuracy=1, loss=0.000765, lr=5.4e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 20, Val Loss: 0.10665208932274996, Val Accuracy: 0.9736265923566879\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.54it/s, accuracy=1, loss=0.000691, lr=4.86e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 21, Val Loss: 0.11187761933249143, Val Accuracy: 0.9726313694267515\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.65it/s, accuracy=1, loss=0.000445, lr=4.38e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 22, Val Loss: 0.11318333379462285, Val Accuracy: 0.974422770700637\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.26it/s, accuracy=1, loss=6.05e-5, lr=3.94e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 23, Val Loss: 0.11192958817741204, Val Accuracy: 0.9742237261146497\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.58it/s, accuracy=1, loss=2.53e-5, lr=3.55e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 24, Val Loss: 0.11499168326072959, Val Accuracy: 0.9734275477707006\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.78it/s, accuracy=1, loss=0.00011, lr=3.19e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 25, Val Loss: 0.11661773523890408, Val Accuracy: 0.9735270700636943\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.48it/s, accuracy=1, loss=0.000245, lr=2.87e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 26, Val Loss: 0.11763518583895338, Val Accuracy: 0.9739251592356688\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.28it/s, accuracy=1, loss=8.16e-5, lr=2.58e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 27, Val Loss: 0.12114917733074189, Val Accuracy: 0.9728304140127388\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.00it/s, accuracy=1, loss=4.35e-5, lr=2.33e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 28, Val Loss: 0.12153779381943368, Val Accuracy: 0.9724323248407644\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.59it/s, accuracy=1, loss=3.51e-6, lr=2.09e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 29, Val Loss: 0.1227940407355657, Val Accuracy: 0.9729299363057324\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.79it/s, accuracy=1, loss=1.37e-5, lr=1.88e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 30, Val Loss: 0.12476965792779522, Val Accuracy: 0.9735270700636943\n",
            "LR: 0.0004 Gamma: 0.9 Hdim: 64 Best Accuracy: 0.974422770700637\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.97it/s, accuracy=0.906, loss=0.24, lr=0.0004]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1, Val Loss: 0.20275159292872164, Val Accuracy: 0.9435708598726115\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.85it/s, accuracy=0.938, loss=0.126, lr=0.00036]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2, Val Loss: 0.1463229445544825, Val Accuracy: 0.9552149681528662\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.98it/s, accuracy=0.938, loss=0.178, lr=0.000324]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3, Val Loss: 0.11069279943687758, Val Accuracy: 0.9657643312101911\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 40.95it/s, accuracy=1, loss=0.0166, lr=0.000292]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4, Val Loss: 0.10441480461021613, Val Accuracy: 0.9677547770700637\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.96it/s, accuracy=1, loss=0.00754, lr=0.000262]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, Val Loss: 0.1053445137563387, Val Accuracy: 0.9667595541401274\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.95it/s, accuracy=1, loss=0.00277, lr=0.000236]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6, Val Loss: 0.10197870175900667, Val Accuracy: 0.9700437898089171\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 39.06it/s, accuracy=0.969, loss=0.0334, lr=0.000213]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7, Val Loss: 0.08727720013672632, Val Accuracy: 0.9737261146496815\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.92it/s, accuracy=1, loss=0.00707, lr=0.000191]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8, Val Loss: 0.1137358055299498, Val Accuracy: 0.9661624203821656\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.07it/s, accuracy=1, loss=0.00135, lr=0.000172]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9, Val Loss: 0.08183455841692634, Val Accuracy: 0.9763136942675159\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.02it/s, accuracy=1, loss=0.00486, lr=0.000155]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10, Val Loss: 0.09894957477753318, Val Accuracy: 0.9710390127388535\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.22it/s, accuracy=1, loss=0.000564, lr=0.000139]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 11, Val Loss: 0.08343163395766401, Val Accuracy: 0.9778065286624203\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.86it/s, accuracy=1, loss=0.00351, lr=0.000126]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 12, Val Loss: 0.1033084763077371, Val Accuracy: 0.9716361464968153\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.30it/s, accuracy=1, loss=0.000253, lr=0.000113]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 13, Val Loss: 0.08533285619788984, Val Accuracy: 0.978702229299363\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 40.90it/s, accuracy=1, loss=0.000431, lr=0.000102]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 14, Val Loss: 0.0835895626488703, Val Accuracy: 0.978702229299363\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.80it/s, accuracy=1, loss=0.000219, lr=9.15e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 15, Val Loss: 0.08573843858652294, Val Accuracy: 0.9780055732484076\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.51it/s, accuracy=1, loss=0.000172, lr=8.24e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 16, Val Loss: 0.08511882480485554, Val Accuracy: 0.9782046178343949\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.34it/s, accuracy=1, loss=7.57e-5, lr=7.41e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 17, Val Loss: 0.0854239272235489, Val Accuracy: 0.978702229299363\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.17it/s, accuracy=1, loss=0.000106, lr=6.67e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 18, Val Loss: 0.08740463810466478, Val Accuracy: 0.9790007961783439\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.56it/s, accuracy=1, loss=0.000242, lr=6e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 19, Val Loss: 0.08891685942176676, Val Accuracy: 0.9786027070063694\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.14it/s, accuracy=1, loss=8.27e-5, lr=5.4e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 20, Val Loss: 0.0901591005230602, Val Accuracy: 0.9789012738853503\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.52it/s, accuracy=1, loss=6.26e-5, lr=4.86e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 21, Val Loss: 0.0925346323496397, Val Accuracy: 0.9785031847133758\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.60it/s, accuracy=1, loss=2.63e-5, lr=4.38e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 22, Val Loss: 0.09176239831891757, Val Accuracy: 0.9782046178343949\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 40.98it/s, accuracy=1, loss=0.000112, lr=3.94e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 23, Val Loss: 0.09412358308321446, Val Accuracy: 0.9780055732484076\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.14it/s, accuracy=1, loss=3.22e-5, lr=3.55e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 24, Val Loss: 0.09592087462899823, Val Accuracy: 0.9776074840764332\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.64it/s, accuracy=1, loss=2.47e-5, lr=3.19e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 25, Val Loss: 0.09750285933511137, Val Accuracy: 0.9784036624203821\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.99it/s, accuracy=1, loss=1.41e-5, lr=2.87e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 26, Val Loss: 0.1003271579244961, Val Accuracy: 0.9777070063694268\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.04it/s, accuracy=1, loss=3.08e-6, lr=2.58e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 27, Val Loss: 0.10003876766503415, Val Accuracy: 0.9784036624203821\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.55it/s, accuracy=1, loss=2.19e-5, lr=2.33e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 28, Val Loss: 0.1035323693710159, Val Accuracy: 0.9777070063694268\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.77it/s, accuracy=1, loss=1.51e-5, lr=2.09e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 29, Val Loss: 0.10295483155864181, Val Accuracy: 0.9784036624203821\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.41it/s, accuracy=1, loss=7.57e-6, lr=1.88e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 30, Val Loss: 0.10641787334107232, Val Accuracy: 0.9777070063694268\n",
            "LR: 0.0004 Gamma: 0.9 Hdim: 128 Best Accuracy: 0.9790007961783439\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.86it/s, accuracy=1, loss=0.0692, lr=0.0004]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1, Val Loss: 0.16268330219848331, Val Accuracy: 0.9529259554140127\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.47it/s, accuracy=0.906, loss=0.275, lr=0.00036]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2, Val Loss: 0.14255919809020154, Val Accuracy: 0.9580015923566879\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.88it/s, accuracy=1, loss=0.0132, lr=0.000324]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3, Val Loss: 0.10198302628846163, Val Accuracy: 0.9684514331210191\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.14it/s, accuracy=1, loss=0.0249, lr=0.000292]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4, Val Loss: 0.09559300348049943, Val Accuracy: 0.9710390127388535\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.92it/s, accuracy=1, loss=0.00579, lr=0.000262]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, Val Loss: 0.09191350254436338, Val Accuracy: 0.9723328025477707\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.24it/s, accuracy=0.969, loss=0.0676, lr=0.000236]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6, Val Loss: 0.09226651151926304, Val Accuracy: 0.9713375796178344\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 40.97it/s, accuracy=1, loss=0.00222, lr=0.000213]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7, Val Loss: 0.09061818127865642, Val Accuracy: 0.9742237261146497\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.63it/s, accuracy=1, loss=0.000201, lr=0.000191]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8, Val Loss: 0.07371367068296254, Val Accuracy: 0.9799960191082803\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 40.99it/s, accuracy=1, loss=0.00253, lr=0.000172]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9, Val Loss: 0.08511863183317056, Val Accuracy: 0.9767117834394905\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.24it/s, accuracy=1, loss=0.00893, lr=0.000155]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10, Val Loss: 0.0765442930585453, Val Accuracy: 0.9795979299363057\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.12it/s, accuracy=1, loss=0.00221, lr=0.000139]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 11, Val Loss: 0.07330246285608641, Val Accuracy: 0.980593152866242\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.78it/s, accuracy=1, loss=0.000173, lr=0.000126]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 12, Val Loss: 0.07304841110169563, Val Accuracy: 0.9802945859872612\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.35it/s, accuracy=1, loss=0.000441, lr=0.000113]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 13, Val Loss: 0.07337076902374187, Val Accuracy: 0.9813893312101911\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.90it/s, accuracy=1, loss=4.41e-5, lr=0.000102]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 14, Val Loss: 0.07428832331727393, Val Accuracy: 0.9815883757961783\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.12it/s, accuracy=1, loss=1.34e-5, lr=9.15e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 15, Val Loss: 0.07602980589387899, Val Accuracy: 0.9807921974522293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.47it/s, accuracy=1, loss=9.73e-5, lr=8.24e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 16, Val Loss: 0.07402158349321863, Val Accuracy: 0.9822850318471338\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.31it/s, accuracy=1, loss=3.76e-5, lr=7.41e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 17, Val Loss: 0.07623745155496442, Val Accuracy: 0.981687898089172\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.52it/s, accuracy=1, loss=0.000195, lr=6.67e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 18, Val Loss: 0.080273074994145, Val Accuracy: 0.9812898089171974\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.40it/s, accuracy=1, loss=3.32e-5, lr=6e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 19, Val Loss: 0.07944238454232401, Val Accuracy: 0.9814888535031847\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.73it/s, accuracy=1, loss=0.000105, lr=5.4e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 20, Val Loss: 0.07933777327949522, Val Accuracy: 0.9811902866242038\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.79it/s, accuracy=1, loss=3.7e-5, lr=4.86e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 21, Val Loss: 0.07888760418271488, Val Accuracy: 0.9809912420382165\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.05it/s, accuracy=1, loss=0.000158, lr=4.38e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 22, Val Loss: 0.07913396816734562, Val Accuracy: 0.9813893312101911\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.19it/s, accuracy=1, loss=5.25e-5, lr=3.94e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 23, Val Loss: 0.07900293199113646, Val Accuracy: 0.9810907643312102\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 40.82it/s, accuracy=1, loss=1.63e-5, lr=3.55e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 24, Val Loss: 0.07992233224483568, Val Accuracy: 0.9812898089171974\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.84it/s, accuracy=1, loss=1.53e-5, lr=3.19e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 25, Val Loss: 0.07873199070985457, Val Accuracy: 0.9808917197452229\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.87it/s, accuracy=1, loss=2.81e-5, lr=2.87e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 26, Val Loss: 0.07968120776516646, Val Accuracy: 0.981687898089172\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.97it/s, accuracy=1, loss=7.61e-5, lr=2.58e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 27, Val Loss: 0.08057059961648945, Val Accuracy: 0.981687898089172\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.67it/s, accuracy=1, loss=2.9e-5, lr=2.33e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 28, Val Loss: 0.08352149130179046, Val Accuracy: 0.9808917197452229\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.54it/s, accuracy=1, loss=2.88e-5, lr=2.09e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 29, Val Loss: 0.08322582516319092, Val Accuracy: 0.981687898089172\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.44it/s, accuracy=1, loss=1.91e-5, lr=1.88e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 30, Val Loss: 0.08315801701411195, Val Accuracy: 0.9815883757961783\n",
            "LR: 0.0004 Gamma: 0.9 Hdim: 256 Best Accuracy: 0.9822850318471338\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.16it/s, accuracy=1, loss=0.0808, lr=0.0005]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1, Val Loss: 0.18932842485796494, Val Accuracy: 0.9460589171974523\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.37it/s, accuracy=0.938, loss=0.211, lr=0.0004]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2, Val Loss: 0.16483362689970216, Val Accuracy: 0.9508359872611465\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.04it/s, accuracy=0.969, loss=0.104, lr=0.00032]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3, Val Loss: 0.1257071957959064, Val Accuracy: 0.9599920382165605\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.70it/s, accuracy=1, loss=0.0266, lr=0.000256]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4, Val Loss: 0.10592672501462304, Val Accuracy: 0.9683519108280255\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 40.98it/s, accuracy=1, loss=0.021, lr=0.000205]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, Val Loss: 0.0934717322382364, Val Accuracy: 0.9708399681528662\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.85it/s, accuracy=1, loss=0.0134, lr=0.000164]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6, Val Loss: 0.09828947285456423, Val Accuracy: 0.9711385350318471\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 40.85it/s, accuracy=1, loss=0.0285, lr=0.000131]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7, Val Loss: 0.08923759354203555, Val Accuracy: 0.9724323248407644\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.85it/s, accuracy=1, loss=0.0121, lr=0.000105]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8, Val Loss: 0.0925193794095217, Val Accuracy: 0.9720342356687898\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.67it/s, accuracy=1, loss=0.011, lr=8.39e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9, Val Loss: 0.08966825073554327, Val Accuracy: 0.9739251592356688\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.36it/s, accuracy=1, loss=0.00294, lr=6.71e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10, Val Loss: 0.09131521703924718, Val Accuracy: 0.9728304140127388\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.11it/s, accuracy=1, loss=0.00301, lr=5.37e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 11, Val Loss: 0.08984029342101184, Val Accuracy: 0.9747213375796179\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.58it/s, accuracy=1, loss=0.00715, lr=4.29e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 12, Val Loss: 0.09016155743427207, Val Accuracy: 0.9746218152866242\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.64it/s, accuracy=1, loss=0.00241, lr=3.44e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 13, Val Loss: 0.09098299945051934, Val Accuracy: 0.9751194267515924\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 40.85it/s, accuracy=1, loss=0.00156, lr=2.75e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 14, Val Loss: 0.09300558659441585, Val Accuracy: 0.9751194267515924\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.44it/s, accuracy=1, loss=0.00113, lr=2.2e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 15, Val Loss: 0.09358492653288064, Val Accuracy: 0.9738256369426752\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.08it/s, accuracy=1, loss=0.000735, lr=1.76e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 16, Val Loss: 0.09323546932442973, Val Accuracy: 0.9746218152866242\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.87it/s, accuracy=1, loss=0.0012, lr=1.41e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 17, Val Loss: 0.09380331328681162, Val Accuracy: 0.975218949044586\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.75it/s, accuracy=1, loss=0.000961, lr=1.13e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 18, Val Loss: 0.09314227727705297, Val Accuracy: 0.9751194267515924\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.44it/s, accuracy=1, loss=0.000979, lr=9.01e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 19, Val Loss: 0.09478734926189275, Val Accuracy: 0.9749203821656051\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.65it/s, accuracy=1, loss=0.00217, lr=7.21e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 20, Val Loss: 0.09537315888162669, Val Accuracy: 0.9746218152866242\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.55it/s, accuracy=1, loss=0.00151, lr=5.76e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 21, Val Loss: 0.09521075491726719, Val Accuracy: 0.9750199044585988\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.33it/s, accuracy=1, loss=0.00137, lr=4.61e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 22, Val Loss: 0.09624930746717054, Val Accuracy: 0.974422770700637\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.20it/s, accuracy=1, loss=0.000245, lr=3.69e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 23, Val Loss: 0.09593527703904688, Val Accuracy: 0.9747213375796179\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.24it/s, accuracy=1, loss=0.000377, lr=2.95e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 24, Val Loss: 0.09631566353771191, Val Accuracy: 0.9748208598726115\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.80it/s, accuracy=1, loss=0.00103, lr=2.36e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 25, Val Loss: 0.09596920695161791, Val Accuracy: 0.9750199044585988\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.83it/s, accuracy=1, loss=0.00314, lr=1.89e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 26, Val Loss: 0.09628805295402912, Val Accuracy: 0.9746218152866242\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.59it/s, accuracy=1, loss=0.00063, lr=1.51e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 27, Val Loss: 0.09655015362558672, Val Accuracy: 0.9748208598726115\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.04it/s, accuracy=1, loss=0.000696, lr=1.21e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 28, Val Loss: 0.09652122115029826, Val Accuracy: 0.9747213375796179\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.41it/s, accuracy=1, loss=7.72e-5, lr=9.67e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 29, Val Loss: 0.09655409106881342, Val Accuracy: 0.9748208598726115\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.09it/s, accuracy=1, loss=0.000463, lr=7.74e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 30, Val Loss: 0.09663597018775102, Val Accuracy: 0.9747213375796179\n",
            "LR: 0.0005 Gamma: 0.8 Hdim: 64 Best Accuracy: 0.975218949044586\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.56it/s, accuracy=0.875, loss=0.291, lr=0.0005]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1, Val Loss: 0.2139858936736728, Val Accuracy: 0.9353105095541401\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.26it/s, accuracy=0.938, loss=0.126, lr=0.0004]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2, Val Loss: 0.1336278582323983, Val Accuracy: 0.9592953821656051\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.73it/s, accuracy=0.969, loss=0.155, lr=0.00032]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3, Val Loss: 0.10970992636592512, Val Accuracy: 0.9674562101910829\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.81it/s, accuracy=1, loss=0.0097, lr=0.000256]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4, Val Loss: 0.10366318500556265, Val Accuracy: 0.9680533439490446\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.77it/s, accuracy=1, loss=0.0173, lr=0.000205]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, Val Loss: 0.09729109084953637, Val Accuracy: 0.9689490445859873\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 37.85it/s, accuracy=1, loss=0.00636, lr=0.000164]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6, Val Loss: 0.08529548129125403, Val Accuracy: 0.9737261146496815\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.47it/s, accuracy=1, loss=0.0074, lr=0.000131]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7, Val Loss: 0.08434684080821403, Val Accuracy: 0.9741242038216561\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 37.97it/s, accuracy=1, loss=0.00151, lr=0.000105]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8, Val Loss: 0.08316602219913116, Val Accuracy: 0.9761146496815286\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.53it/s, accuracy=1, loss=0.000436, lr=8.39e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9, Val Loss: 0.08042649160301049, Val Accuracy: 0.9756170382165605\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.58it/s, accuracy=1, loss=0.00207, lr=6.71e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10, Val Loss: 0.08047001255501157, Val Accuracy: 0.9768113057324841\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.41it/s, accuracy=1, loss=0.000378, lr=5.37e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 11, Val Loss: 0.0805553606761078, Val Accuracy: 0.9766122611464968\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.37it/s, accuracy=1, loss=0.00145, lr=4.29e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 12, Val Loss: 0.08076492110211321, Val Accuracy: 0.9773089171974523\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.58it/s, accuracy=1, loss=0.000249, lr=3.44e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 13, Val Loss: 0.0813007780700373, Val Accuracy: 0.977906050955414\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.01it/s, accuracy=1, loss=0.00128, lr=2.75e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 14, Val Loss: 0.0844051024665115, Val Accuracy: 0.9768113057324841\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.12it/s, accuracy=1, loss=0.00055, lr=2.2e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 15, Val Loss: 0.08451820821737725, Val Accuracy: 0.9772093949044586\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.19it/s, accuracy=1, loss=0.000268, lr=1.76e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 16, Val Loss: 0.08506566414272494, Val Accuracy: 0.9774084394904459\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 37.84it/s, accuracy=1, loss=0.000137, lr=1.41e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 17, Val Loss: 0.08496257525233059, Val Accuracy: 0.9773089171974523\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.36it/s, accuracy=1, loss=0.000339, lr=1.13e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 18, Val Loss: 0.08606766256317332, Val Accuracy: 0.9772093949044586\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.47it/s, accuracy=1, loss=0.000277, lr=9.01e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 19, Val Loss: 0.08744477226894554, Val Accuracy: 0.9783041401273885\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.69it/s, accuracy=1, loss=0.000195, lr=7.21e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 20, Val Loss: 0.08672192773807982, Val Accuracy: 0.9773089171974523\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.82it/s, accuracy=1, loss=9.2e-5, lr=5.76e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 21, Val Loss: 0.08729226837634023, Val Accuracy: 0.9777070063694268\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.91it/s, accuracy=1, loss=0.000153, lr=4.61e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 22, Val Loss: 0.08710172044856376, Val Accuracy: 0.9780055732484076\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 40.97it/s, accuracy=1, loss=0.000405, lr=3.69e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 23, Val Loss: 0.0884607777609058, Val Accuracy: 0.9778065286624203\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.64it/s, accuracy=1, loss=0.000157, lr=2.95e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 24, Val Loss: 0.08862998631866938, Val Accuracy: 0.9776074840764332\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.07it/s, accuracy=1, loss=8.72e-5, lr=2.36e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 25, Val Loss: 0.08911056585785665, Val Accuracy: 0.9776074840764332\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.33it/s, accuracy=1, loss=0.000124, lr=1.89e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 26, Val Loss: 0.08901792963566277, Val Accuracy: 0.9775079617834395\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.66it/s, accuracy=1, loss=4.43e-5, lr=1.51e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 27, Val Loss: 0.08904569859970081, Val Accuracy: 0.9778065286624203\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 39.00it/s, accuracy=1, loss=0.000395, lr=1.21e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 28, Val Loss: 0.08930916533999932, Val Accuracy: 0.9776074840764332\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.39it/s, accuracy=1, loss=0.000189, lr=9.67e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 29, Val Loss: 0.0892940822154445, Val Accuracy: 0.977906050955414\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 37.58it/s, accuracy=1, loss=0.000126, lr=7.74e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 30, Val Loss: 0.08939059804384933, Val Accuracy: 0.9775079617834395\n",
            "LR: 0.0005 Gamma: 0.8 Hdim: 128 Best Accuracy: 0.9783041401273885\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.08it/s, accuracy=1, loss=0.0979, lr=0.0005]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1, Val Loss: 0.19374353556069218, Val Accuracy: 0.9461584394904459\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.46it/s, accuracy=0.906, loss=0.294, lr=0.0004]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2, Val Loss: 0.14083883819870271, Val Accuracy: 0.9551154458598726\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.49it/s, accuracy=1, loss=0.0217, lr=0.00032]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3, Val Loss: 0.09847910181473893, Val Accuracy: 0.9704418789808917\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 40.86it/s, accuracy=0.969, loss=0.0536, lr=0.000256]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4, Val Loss: 0.09328038286333191, Val Accuracy: 0.9703423566878981\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.96it/s, accuracy=1, loss=0.0286, lr=0.000205]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, Val Loss: 0.09371538460402969, Val Accuracy: 0.9716361464968153\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.08it/s, accuracy=1, loss=0.0204, lr=0.000164]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6, Val Loss: 0.08114962456384556, Val Accuracy: 0.9735270700636943\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.62it/s, accuracy=1, loss=0.000817, lr=0.000131]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7, Val Loss: 0.07321490912669383, Val Accuracy: 0.9776074840764332\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.70it/s, accuracy=1, loss=0.00262, lr=0.000105]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8, Val Loss: 0.07522130582885879, Val Accuracy: 0.9770103503184714\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 37.68it/s, accuracy=1, loss=0.000332, lr=8.39e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9, Val Loss: 0.07272856022000211, Val Accuracy: 0.9788017515923567\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.22it/s, accuracy=1, loss=0.000681, lr=6.71e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10, Val Loss: 0.07416676395776704, Val Accuracy: 0.9791003184713376\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.28it/s, accuracy=1, loss=0.00111, lr=5.37e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 11, Val Loss: 0.07232545367352049, Val Accuracy: 0.979796974522293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.75it/s, accuracy=1, loss=0.000544, lr=4.29e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 12, Val Loss: 0.07338712322391051, Val Accuracy: 0.9800955414012739\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 37.94it/s, accuracy=1, loss=0.00512, lr=3.44e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 13, Val Loss: 0.07518774124555189, Val Accuracy: 0.9796974522292994\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.61it/s, accuracy=1, loss=5.56e-5, lr=2.75e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 14, Val Loss: 0.07418632473482037, Val Accuracy: 0.9796974522292994\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.79it/s, accuracy=1, loss=4.18e-5, lr=2.2e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 15, Val Loss: 0.07558404049308981, Val Accuracy: 0.9792993630573248\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.54it/s, accuracy=1, loss=0.000142, lr=1.76e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 16, Val Loss: 0.07463592946807612, Val Accuracy: 0.9798964968152867\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.35it/s, accuracy=1, loss=0.00025, lr=1.41e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 17, Val Loss: 0.07631344289926624, Val Accuracy: 0.9794984076433121\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.17it/s, accuracy=1, loss=0.000394, lr=1.13e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 18, Val Loss: 0.07646855755939477, Val Accuracy: 0.9799960191082803\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.88it/s, accuracy=1, loss=4.57e-5, lr=9.01e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 19, Val Loss: 0.07665017647432316, Val Accuracy: 0.9799960191082803\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 37.92it/s, accuracy=1, loss=0.000121, lr=7.21e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 20, Val Loss: 0.07771345748152694, Val Accuracy: 0.9800955414012739\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.08it/s, accuracy=1, loss=0.00029, lr=5.76e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 21, Val Loss: 0.07754000947442441, Val Accuracy: 0.9798964968152867\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.57it/s, accuracy=1, loss=0.000115, lr=4.61e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 22, Val Loss: 0.07849436264202767, Val Accuracy: 0.9799960191082803\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.31it/s, accuracy=1, loss=6.15e-5, lr=3.69e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 23, Val Loss: 0.07888295703830603, Val Accuracy: 0.980593152866242\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.34it/s, accuracy=1, loss=2.1e-5, lr=2.95e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 24, Val Loss: 0.07924514369370152, Val Accuracy: 0.9798964968152867\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.76it/s, accuracy=1, loss=2.69e-5, lr=2.36e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 25, Val Loss: 0.07909918496580085, Val Accuracy: 0.9802945859872612\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.77it/s, accuracy=1, loss=2.92e-5, lr=1.89e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 26, Val Loss: 0.07933592153320605, Val Accuracy: 0.9800955414012739\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.93it/s, accuracy=1, loss=0.000142, lr=1.51e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 27, Val Loss: 0.07958318455628, Val Accuracy: 0.9801950636942676\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.93it/s, accuracy=1, loss=5.39e-5, lr=1.21e-6]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 28, Val Loss: 0.07979402123937493, Val Accuracy: 0.9801950636942676\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.57it/s, accuracy=1, loss=0.000124, lr=9.67e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 29, Val Loss: 0.07971016922183216, Val Accuracy: 0.9800955414012739\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.74it/s, accuracy=1, loss=8.71e-5, lr=7.74e-7]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 30, Val Loss: 0.07971786918694529, Val Accuracy: 0.9800955414012739\n",
            "LR: 0.0005 Gamma: 0.8 Hdim: 256 Best Accuracy: 0.980593152866242\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 37.96it/s, accuracy=1, loss=0.0808, lr=0.0005]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1, Val Loss: 0.18932842485796494, Val Accuracy: 0.9460589171974523\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.73it/s, accuracy=0.938, loss=0.243, lr=0.00045]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2, Val Loss: 0.1575164764846681, Val Accuracy: 0.9538216560509554\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 37.87it/s, accuracy=0.969, loss=0.123, lr=0.000405]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3, Val Loss: 0.13177325466208778, Val Accuracy: 0.9597929936305732\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.08it/s, accuracy=1, loss=0.0196, lr=0.000365]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4, Val Loss: 0.10638118003477241, Val Accuracy: 0.9664609872611465\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.13it/s, accuracy=1, loss=0.0149, lr=0.000328]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, Val Loss: 0.0996130722180783, Val Accuracy: 0.9689490445859873\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.97it/s, accuracy=1, loss=0.0117, lr=0.000295]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6, Val Loss: 0.10558137886825308, Val Accuracy: 0.9686504777070064\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.44it/s, accuracy=1, loss=0.0213, lr=0.000266]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7, Val Loss: 0.09356283743800536, Val Accuracy: 0.9718351910828026\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.49it/s, accuracy=1, loss=0.0103, lr=0.000239]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8, Val Loss: 0.10281463027332062, Val Accuracy: 0.9693471337579618\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.23it/s, accuracy=1, loss=0.00946, lr=0.000215]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9, Val Loss: 0.11269544417219857, Val Accuracy: 0.9672571656050956\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.12it/s, accuracy=1, loss=0.00333, lr=0.000194]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10, Val Loss: 0.10464821711646402, Val Accuracy: 0.9701433121019108\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 39.06it/s, accuracy=1, loss=0.00106, lr=0.000174]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 11, Val Loss: 0.09032029031143671, Val Accuracy: 0.976015127388535\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.08it/s, accuracy=0.969, loss=0.101, lr=0.000157]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 12, Val Loss: 0.12033160647722539, Val Accuracy: 0.9682523885350318\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.35it/s, accuracy=1, loss=0.000602, lr=0.000141]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 13, Val Loss: 0.09296293137493102, Val Accuracy: 0.9755175159235668\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.05it/s, accuracy=1, loss=0.000519, lr=0.000127]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 14, Val Loss: 0.09233884567486705, Val Accuracy: 0.9757165605095541\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.03it/s, accuracy=1, loss=0.000386, lr=0.000114]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 15, Val Loss: 0.09565240474979884, Val Accuracy: 0.9747213375796179\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.38it/s, accuracy=1, loss=0.000914, lr=0.000103]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 16, Val Loss: 0.09495131402022447, Val Accuracy: 0.9746218152866242\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.78it/s, accuracy=1, loss=0.000333, lr=9.27e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 17, Val Loss: 0.09674555454635692, Val Accuracy: 0.976015127388535\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.12it/s, accuracy=1, loss=0.000359, lr=8.34e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 18, Val Loss: 0.09719193643158681, Val Accuracy: 0.9756170382165605\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.23it/s, accuracy=1, loss=0.000381, lr=7.5e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 19, Val Loss: 0.09773079265379371, Val Accuracy: 0.9758160828025477\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.92it/s, accuracy=1, loss=0.000513, lr=6.75e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 20, Val Loss: 0.09796212924164914, Val Accuracy: 0.9759156050955414\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.10it/s, accuracy=1, loss=0.000319, lr=6.08e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 21, Val Loss: 0.09795809379085339, Val Accuracy: 0.9766122611464968\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.47it/s, accuracy=1, loss=0.00038, lr=5.47e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 22, Val Loss: 0.09954835910335173, Val Accuracy: 0.9758160828025477\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 39.07it/s, accuracy=1, loss=8.81e-5, lr=4.92e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 23, Val Loss: 0.10180104621044964, Val Accuracy: 0.9762141719745223\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.41it/s, accuracy=1, loss=8.61e-5, lr=4.43e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 24, Val Loss: 0.10292088325806593, Val Accuracy: 0.9756170382165605\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.13it/s, accuracy=1, loss=0.000268, lr=3.99e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 25, Val Loss: 0.10431678564078788, Val Accuracy: 0.976015127388535\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.67it/s, accuracy=1, loss=0.00054, lr=3.59e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 26, Val Loss: 0.10557973939146768, Val Accuracy: 0.9754179936305732\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.30it/s, accuracy=1, loss=7.62e-5, lr=3.23e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 27, Val Loss: 0.10618078308464278, Val Accuracy: 0.9761146496815286\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.93it/s, accuracy=1, loss=6.67e-5, lr=2.91e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 28, Val Loss: 0.10766838773326402, Val Accuracy: 0.9762141719745223\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.17it/s, accuracy=1, loss=9.45e-6, lr=2.62e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 29, Val Loss: 0.10887393832243057, Val Accuracy: 0.9763136942675159\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.05it/s, accuracy=1, loss=3.06e-5, lr=2.36e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 30, Val Loss: 0.11114420627881359, Val Accuracy: 0.9757165605095541\n",
            "LR: 0.0005 Gamma: 0.9 Hdim: 64 Best Accuracy: 0.9766122611464968\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.52it/s, accuracy=0.875, loss=0.291, lr=0.0005]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1, Val Loss: 0.2139858936736728, Val Accuracy: 0.9353105095541401\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.28it/s, accuracy=0.969, loss=0.145, lr=0.00045]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2, Val Loss: 0.1459854177917075, Val Accuracy: 0.9568073248407644\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.44it/s, accuracy=1, loss=0.131, lr=0.000405]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3, Val Loss: 0.12423385256177681, Val Accuracy: 0.9620820063694268\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.36it/s, accuracy=1, loss=0.0314, lr=0.000365]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4, Val Loss: 0.11232266298036987, Val Accuracy: 0.9657643312101911\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.44it/s, accuracy=1, loss=0.0207, lr=0.000328]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, Val Loss: 0.129396776423662, Val Accuracy: 0.9594944267515924\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.50it/s, accuracy=1, loss=0.00404, lr=0.000295]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6, Val Loss: 0.10094611934192663, Val Accuracy: 0.9712380573248408\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 40.82it/s, accuracy=1, loss=0.0224, lr=0.000266]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7, Val Loss: 0.0964371536566044, Val Accuracy: 0.9707404458598726\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.79it/s, accuracy=1, loss=0.0051, lr=0.000239]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8, Val Loss: 0.09421849134215933, Val Accuracy: 0.9741242038216561\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 40.81it/s, accuracy=1, loss=0.000477, lr=0.000215]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9, Val Loss: 0.09989670265483956, Val Accuracy: 0.9718351910828026\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.27it/s, accuracy=1, loss=0.00119, lr=0.000194]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10, Val Loss: 0.10889797414359, Val Accuracy: 0.9723328025477707\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.21it/s, accuracy=1, loss=0.0111, lr=0.000174]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 11, Val Loss: 0.1139022854963543, Val Accuracy: 0.9657643312101911\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.34it/s, accuracy=1, loss=0.0016, lr=0.000157]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 12, Val Loss: 0.09035588467971538, Val Accuracy: 0.9767117834394905\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.55it/s, accuracy=1, loss=4.14e-5, lr=0.000141]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 13, Val Loss: 0.0874643704362008, Val Accuracy: 0.9769108280254777\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.80it/s, accuracy=1, loss=0.00139, lr=0.000127]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 14, Val Loss: 0.08938748574254601, Val Accuracy: 0.9767117834394905\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.89it/s, accuracy=1, loss=5.34e-5, lr=0.000114]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 15, Val Loss: 0.0916476279202873, Val Accuracy: 0.9769108280254777\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.23it/s, accuracy=1, loss=0.000261, lr=0.000103]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 16, Val Loss: 0.09003724162342043, Val Accuracy: 0.9774084394904459\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.21it/s, accuracy=1, loss=0.00183, lr=9.27e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 17, Val Loss: 0.17701720343650598, Val Accuracy: 0.9519307324840764\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.65it/s, accuracy=1, loss=0.000419, lr=8.34e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 18, Val Loss: 0.09374503366673383, Val Accuracy: 0.9773089171974523\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.65it/s, accuracy=1, loss=0.000179, lr=7.5e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 19, Val Loss: 0.09472358984992602, Val Accuracy: 0.9777070063694268\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.25it/s, accuracy=1, loss=0.000101, lr=6.75e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 20, Val Loss: 0.09498218395518364, Val Accuracy: 0.9775079617834395\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.21it/s, accuracy=1, loss=8.43e-5, lr=6.08e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 21, Val Loss: 0.09462557528916327, Val Accuracy: 0.9774084394904459\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.04it/s, accuracy=1, loss=4.36e-5, lr=5.47e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 22, Val Loss: 0.09494244520496427, Val Accuracy: 0.9781050955414012\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.97it/s, accuracy=1, loss=0.0003, lr=4.92e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 23, Val Loss: 0.09662080050553475, Val Accuracy: 0.977906050955414\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.73it/s, accuracy=1, loss=4.1e-5, lr=4.43e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 24, Val Loss: 0.09655842049970115, Val Accuracy: 0.9777070063694268\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.33it/s, accuracy=1, loss=6e-5, lr=3.99e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 25, Val Loss: 0.09951946499684058, Val Accuracy: 0.9782046178343949\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.32it/s, accuracy=1, loss=4.52e-5, lr=3.59e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 26, Val Loss: 0.10079575540910687, Val Accuracy: 0.9781050955414012\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.95it/s, accuracy=1, loss=7.15e-6, lr=3.23e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 27, Val Loss: 0.09977115047011004, Val Accuracy: 0.9778065286624203\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.21it/s, accuracy=1, loss=0.000126, lr=2.91e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 28, Val Loss: 0.10316061036486529, Val Accuracy: 0.9780055732484076\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 37.75it/s, accuracy=1, loss=4.05e-5, lr=2.62e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 29, Val Loss: 0.1030563016528166, Val Accuracy: 0.977906050955414\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.77it/s, accuracy=1, loss=1.05e-5, lr=2.36e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 30, Val Loss: 0.10523266736814063, Val Accuracy: 0.977906050955414\n",
            "LR: 0.0005 Gamma: 0.9 Hdim: 128 Best Accuracy: 0.9782046178343949\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:25<00:00, 36.98it/s, accuracy=1, loss=0.0979, lr=0.0005]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1, Val Loss: 0.19374353556069218, Val Accuracy: 0.9461584394904459\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.89it/s, accuracy=0.938, loss=0.324, lr=0.00045]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2, Val Loss: 0.1671598356151894, Val Accuracy: 0.9492436305732485\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.79it/s, accuracy=1, loss=0.0319, lr=0.000405]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3, Val Loss: 0.12300612329754528, Val Accuracy: 0.9619824840764332\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:25<00:00, 36.67it/s, accuracy=0.969, loss=0.0566, lr=0.000365]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4, Val Loss: 0.12971192976136567, Val Accuracy: 0.9578025477707006\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.94it/s, accuracy=1, loss=0.00735, lr=0.000328]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, Val Loss: 0.10153341671165708, Val Accuracy: 0.9701433121019108\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.79it/s, accuracy=0.969, loss=0.095, lr=0.000295]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6, Val Loss: 0.11153350239245156, Val Accuracy: 0.9663614649681529\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.80it/s, accuracy=1, loss=0.00589, lr=0.000266]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7, Val Loss: 0.09554681365715954, Val Accuracy: 0.9703423566878981\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.48it/s, accuracy=1, loss=0.00412, lr=0.000239]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8, Val Loss: 0.13212661868796216, Val Accuracy: 0.9592953821656051\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.56it/s, accuracy=1, loss=0.0013, lr=0.000215]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9, Val Loss: 0.1067080853033661, Val Accuracy: 0.9686504777070064\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.59it/s, accuracy=1, loss=0.000375, lr=0.000194]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10, Val Loss: 0.08680619383919473, Val Accuracy: 0.9774084394904459\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.34it/s, accuracy=1, loss=0.00573, lr=0.000174]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 11, Val Loss: 0.10106459154422928, Val Accuracy: 0.9724323248407644\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.23it/s, accuracy=1, loss=0.000104, lr=0.000157]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 12, Val Loss: 0.07678680743990855, Val Accuracy: 0.9795979299363057\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:25<00:00, 37.16it/s, accuracy=0.969, loss=0.0559, lr=0.000141]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 13, Val Loss: 0.08623654169384141, Val Accuracy: 0.9770103503184714\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 39.07it/s, accuracy=1, loss=0.00309, lr=0.000127]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 14, Val Loss: 0.10667488904548275, Val Accuracy: 0.9719347133757962\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.03it/s, accuracy=1, loss=2.65e-5, lr=0.000114]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 15, Val Loss: 0.08036717614541092, Val Accuracy: 0.9793988853503185\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 39.05it/s, accuracy=1, loss=5.85e-5, lr=0.000103]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 16, Val Loss: 0.07978795398650855, Val Accuracy: 0.9796974522292994\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.99it/s, accuracy=1, loss=0.000208, lr=9.27e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 17, Val Loss: 0.07938758664141397, Val Accuracy: 0.9796974522292994\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.76it/s, accuracy=1, loss=0.000266, lr=8.34e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 18, Val Loss: 0.08056375965576648, Val Accuracy: 0.9801950636942676\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.47it/s, accuracy=1, loss=7.98e-5, lr=7.5e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 19, Val Loss: 0.07952299575846186, Val Accuracy: 0.9807921974522293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.31it/s, accuracy=1, loss=9.77e-5, lr=6.75e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 20, Val Loss: 0.0820502515465246, Val Accuracy: 0.9801950636942676\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.49it/s, accuracy=1, loss=0.000105, lr=6.08e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 21, Val Loss: 0.08090090693193218, Val Accuracy: 0.9799960191082803\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.16it/s, accuracy=1, loss=5.08e-5, lr=5.47e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 22, Val Loss: 0.08443003451170523, Val Accuracy: 0.979796974522293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 39.26it/s, accuracy=1, loss=1.48e-5, lr=4.92e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 23, Val Loss: 0.0832220833728156, Val Accuracy: 0.9802945859872612\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.57it/s, accuracy=1, loss=6.55e-6, lr=4.43e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 24, Val Loss: 0.08397037429702292, Val Accuracy: 0.9806926751592356\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:25<00:00, 36.85it/s, accuracy=1, loss=1.26e-5, lr=3.99e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 25, Val Loss: 0.08410356779102915, Val Accuracy: 0.9803941082802548\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:25<00:00, 37.43it/s, accuracy=1, loss=5.47e-6, lr=3.59e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 26, Val Loss: 0.08645242922616099, Val Accuracy: 0.9804936305732485\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.90it/s, accuracy=1, loss=0.000123, lr=3.23e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 27, Val Loss: 0.08970396884368675, Val Accuracy: 0.9786027070063694\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:25<00:00, 36.94it/s, accuracy=1, loss=9.15e-6, lr=2.91e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 28, Val Loss: 0.08779371127719944, Val Accuracy: 0.9796974522292994\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.69it/s, accuracy=1, loss=2.21e-5, lr=2.62e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 29, Val Loss: 0.08805721780997436, Val Accuracy: 0.9798964968152867\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 38.01it/s, accuracy=1, loss=2.01e-5, lr=2.36e-5]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 30, Val Loss: 0.0882182371973993, Val Accuracy: 0.9799960191082803\n",
            "LR: 0.0005 Gamma: 0.9 Hdim: 256 Best Accuracy: 0.9807921974522293\n"
          ]
        }
      ],
      "source": [
        "epochs = 30\n",
        "lrs = [3e-4, 4e-4, 5e-4]\n",
        "gammas = [0.8, 0.9]\n",
        "hdims = [64, 128, 256]\n",
        "best_accs = []\n",
        "for lr in lrs:\n",
        "    for gamma in gammas:\n",
        "        for hdim in hdims:\n",
        "            torch.manual_seed(42)\n",
        "            best_acc = 0\n",
        "            # Define model\n",
        "            model = SineKAN(layers_hidden=[28 * 28, hdim, 10], grid_size=8)\n",
        "            device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "            model.to(device)\n",
        "            # Define optimizer\n",
        "            optimizer = optim.AdamW(model.parameters(), lr=lr)\n",
        "            # Define learning rate scheduler\n",
        "            scheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=gamma)\n",
        "\n",
        "            # Define loss\n",
        "            criterion = nn.CrossEntropyLoss()\n",
        "            for epoch in range(epochs):\n",
        "                # Train\n",
        "                model.train()\n",
        "                with tqdm(train_loader) as pbar:\n",
        "                    for i, (images, labels) in enumerate(pbar):\n",
        "                        images = images.view(-1, 28 * 28).to(device)\n",
        "                        optimizer.zero_grad()\n",
        "                        output = model(images)\n",
        "                        loss = criterion(output, labels.to(device))\n",
        "                        loss.backward()\n",
        "                        optimizer.step()\n",
        "                        accuracy = (output.argmax(dim=1) == labels.to(device)).float().mean()\n",
        "                        pbar.set_postfix(loss=loss.item(), accuracy=accuracy.item(), lr=optimizer.param_groups[0]['lr'])\n",
        "\n",
        "                # Validation\n",
        "                model.eval()\n",
        "                val_loss = 0\n",
        "                val_accuracy = 0\n",
        "                with torch.no_grad():\n",
        "                    for images, labels in val_loader:\n",
        "                        images = images.view(-1, 28 * 28).to(device)\n",
        "                        output = model(images)\n",
        "                        val_loss += criterion(output, labels.to(device)).item()\n",
        "                        val_accuracy += (\n",
        "                            (output.argmax(dim=1) == labels.to(device)).float().mean().item()\n",
        "                        )\n",
        "                val_loss /= len(val_loader)\n",
        "                val_accuracy /= len(val_loader)\n",
        "                if val_accuracy > best_acc:\n",
        "                    best_acc = val_accuracy\n",
        "\n",
        "                # Update learning rate\n",
        "                scheduler.step()\n",
        "\n",
        "                print(\n",
        "                    f\"Epoch {epoch + 1}, Val Loss: {val_loss}, Val Accuracy: {val_accuracy}\"\n",
        "                )\n",
        "            best_accs.append(best_acc)\n",
        "            print(f\"LR: {lr} Gamma: {gamma} Hdim: {hdim} Best Accuracy: {best_acc}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "HxaS2pjy0LnB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3c625f26-e7ae-4242-9284-db9f4fb9a9a6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:24<00:00, 37.56it/s, accuracy=1, loss=0.0692, lr=0.0004]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Val Loss: 0.16268330219848331, Val Accuracy: 0.9529259554140127\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.86it/s, accuracy=0.906, loss=0.275, lr=0.00036]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2, Val Loss: 0.14255919809020154, Val Accuracy: 0.9580015923566879\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.88it/s, accuracy=1, loss=0.0132, lr=0.000324]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3, Val Loss: 0.10198302628846163, Val Accuracy: 0.9684514331210191\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.33it/s, accuracy=1, loss=0.0249, lr=0.000292]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4, Val Loss: 0.09559300348049943, Val Accuracy: 0.9710390127388535\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.85it/s, accuracy=1, loss=0.00579, lr=0.000262]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5, Val Loss: 0.09191350254436338, Val Accuracy: 0.9723328025477707\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.73it/s, accuracy=0.969, loss=0.0676, lr=0.000236]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6, Val Loss: 0.09226651151926304, Val Accuracy: 0.9713375796178344\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.82it/s, accuracy=1, loss=0.00222, lr=0.000213]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7, Val Loss: 0.09061818127865642, Val Accuracy: 0.9742237261146497\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.77it/s, accuracy=1, loss=0.000201, lr=0.000191]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8, Val Loss: 0.07371367068296254, Val Accuracy: 0.9799960191082803\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.76it/s, accuracy=1, loss=0.00253, lr=0.000172]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 9, Val Loss: 0.08511863183317056, Val Accuracy: 0.9767117834394905\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.16it/s, accuracy=1, loss=0.00893, lr=0.000155]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10, Val Loss: 0.0765442930585453, Val Accuracy: 0.9795979299363057\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.47it/s, accuracy=1, loss=0.00221, lr=0.000139]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11, Val Loss: 0.07330246285608641, Val Accuracy: 0.980593152866242\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.07it/s, accuracy=1, loss=0.000173, lr=0.000126]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 12, Val Loss: 0.07304841110169563, Val Accuracy: 0.9802945859872612\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.34it/s, accuracy=1, loss=0.000441, lr=0.000113]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 13, Val Loss: 0.07337076902374187, Val Accuracy: 0.9813893312101911\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.25it/s, accuracy=1, loss=4.41e-5, lr=0.000102]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 14, Val Loss: 0.07428832331727393, Val Accuracy: 0.9815883757961783\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.10it/s, accuracy=1, loss=1.34e-5, lr=9.15e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 15, Val Loss: 0.07602980589387899, Val Accuracy: 0.9807921974522293\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.94it/s, accuracy=1, loss=9.73e-5, lr=8.24e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 16, Val Loss: 0.07402158349321863, Val Accuracy: 0.9822850318471338\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.04it/s, accuracy=1, loss=3.76e-5, lr=7.41e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 17, Val Loss: 0.07623745155496442, Val Accuracy: 0.981687898089172\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.55it/s, accuracy=1, loss=0.000195, lr=6.67e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 18, Val Loss: 0.080273074994145, Val Accuracy: 0.9812898089171974\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.09it/s, accuracy=1, loss=3.32e-5, lr=6e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 19, Val Loss: 0.07944238454232401, Val Accuracy: 0.9814888535031847\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.00it/s, accuracy=1, loss=0.000105, lr=5.4e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 20, Val Loss: 0.07933777327949522, Val Accuracy: 0.9811902866242038\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.89it/s, accuracy=1, loss=3.7e-5, lr=4.86e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 21, Val Loss: 0.07888760418271488, Val Accuracy: 0.9809912420382165\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.50it/s, accuracy=1, loss=0.000158, lr=4.38e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 22, Val Loss: 0.07913396816734562, Val Accuracy: 0.9813893312101911\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.94it/s, accuracy=1, loss=5.25e-5, lr=3.94e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 23, Val Loss: 0.07900293199113646, Val Accuracy: 0.9810907643312102\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.74it/s, accuracy=1, loss=1.63e-5, lr=3.55e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 24, Val Loss: 0.07992233224483568, Val Accuracy: 0.9812898089171974\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.63it/s, accuracy=1, loss=1.53e-5, lr=3.19e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 25, Val Loss: 0.07873199070985457, Val Accuracy: 0.9808917197452229\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.31it/s, accuracy=1, loss=2.81e-5, lr=2.87e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 26, Val Loss: 0.07968120776516646, Val Accuracy: 0.981687898089172\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.34it/s, accuracy=1, loss=7.61e-5, lr=2.58e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 27, Val Loss: 0.08057059961648945, Val Accuracy: 0.981687898089172\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.31it/s, accuracy=1, loss=2.9e-5, lr=2.33e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 28, Val Loss: 0.08352149130179046, Val Accuracy: 0.9808917197452229\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.03it/s, accuracy=1, loss=2.88e-5, lr=2.09e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 29, Val Loss: 0.08322582516319092, Val Accuracy: 0.981687898089172\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.58it/s, accuracy=1, loss=1.91e-5, lr=1.88e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 30, Val Loss: 0.08315801701411195, Val Accuracy: 0.9815883757961783\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.94it/s, accuracy=1, loss=0.0602, lr=0.0004]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Val Loss: 0.16883290946412427, Val Accuracy: 0.9516321656050956\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.69it/s, accuracy=0.906, loss=0.336, lr=0.00036]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2, Val Loss: 0.13559028068794662, Val Accuracy: 0.9583996815286624\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.82it/s, accuracy=1, loss=0.0173, lr=0.000324]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3, Val Loss: 0.09745028500955338, Val Accuracy: 0.9712380573248408\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.35it/s, accuracy=0.969, loss=0.049, lr=0.000292]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4, Val Loss: 0.09824009248460376, Val Accuracy: 0.9697452229299363\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.66it/s, accuracy=1, loss=0.0275, lr=0.000262]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5, Val Loss: 0.09267383954742817, Val Accuracy: 0.9712380573248408\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.76it/s, accuracy=1, loss=0.0333, lr=0.000236]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6, Val Loss: 0.08973285986242555, Val Accuracy: 0.9721337579617835\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.14it/s, accuracy=1, loss=0.00447, lr=0.000213]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7, Val Loss: 0.08449632023744044, Val Accuracy: 0.9736265923566879\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.13it/s, accuracy=1, loss=0.00317, lr=0.000191]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8, Val Loss: 0.09329362061975086, Val Accuracy: 0.9701433121019108\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.02it/s, accuracy=1, loss=0.00156, lr=0.000172]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 9, Val Loss: 0.08567749172833579, Val Accuracy: 0.9735270700636943\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.71it/s, accuracy=1, loss=0.00101, lr=0.000155]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10, Val Loss: 0.08321548188741061, Val Accuracy: 0.9743232484076433\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.14it/s, accuracy=1, loss=0.00676, lr=0.000139]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11, Val Loss: 0.08417115958217188, Val Accuracy: 0.9747213375796179\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.06it/s, accuracy=1, loss=0.000825, lr=0.000126]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 12, Val Loss: 0.07186751382236091, Val Accuracy: 0.978702229299363\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.29it/s, accuracy=1, loss=0.00551, lr=0.000113]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 13, Val Loss: 0.07381054708329064, Val Accuracy: 0.9793988853503185\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.80it/s, accuracy=1, loss=0.000345, lr=0.000102]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 14, Val Loss: 0.07166607691725246, Val Accuracy: 0.9796974522292994\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.59it/s, accuracy=1, loss=8.25e-5, lr=9.15e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 15, Val Loss: 0.0716535459543309, Val Accuracy: 0.9798964968152867\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.07it/s, accuracy=1, loss=0.000163, lr=8.24e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 16, Val Loss: 0.06890558557422957, Val Accuracy: 0.9815883757961783\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.51it/s, accuracy=1, loss=0.00201, lr=7.41e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 17, Val Loss: 0.09892444134135411, Val Accuracy: 0.9705414012738853\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.18it/s, accuracy=1, loss=0.00127, lr=6.67e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 18, Val Loss: 0.0701859331158281, Val Accuracy: 0.9804936305732485\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.67it/s, accuracy=1, loss=0.000298, lr=6e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 19, Val Loss: 0.06928897703959849, Val Accuracy: 0.9803941082802548\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.39it/s, accuracy=1, loss=0.000298, lr=5.4e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 20, Val Loss: 0.07073833221535636, Val Accuracy: 0.9800955414012739\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.10it/s, accuracy=1, loss=0.000907, lr=4.86e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 21, Val Loss: 0.0700200003385162, Val Accuracy: 0.980593152866242\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.12it/s, accuracy=1, loss=0.00113, lr=4.38e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 22, Val Loss: 0.0739550033922685, Val Accuracy: 0.9808917197452229\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.39it/s, accuracy=1, loss=0.000177, lr=3.94e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 23, Val Loss: 0.07115772243292062, Val Accuracy: 0.9813893312101911\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.88it/s, accuracy=1, loss=0.000178, lr=3.55e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 24, Val Loss: 0.07038558684711212, Val Accuracy: 0.9810907643312102\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.89it/s, accuracy=1, loss=5.14e-5, lr=3.19e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 25, Val Loss: 0.07097771750807531, Val Accuracy: 0.9812898089171974\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.52it/s, accuracy=1, loss=4.89e-5, lr=2.87e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 26, Val Loss: 0.0721980978480917, Val Accuracy: 0.9812898089171974\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.72it/s, accuracy=1, loss=0.000462, lr=2.58e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 27, Val Loss: 0.07126283128080547, Val Accuracy: 0.9820859872611465\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.50it/s, accuracy=1, loss=0.000121, lr=2.33e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 28, Val Loss: 0.07270644924896134, Val Accuracy: 0.9808917197452229\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.74it/s, accuracy=1, loss=0.000264, lr=2.09e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 29, Val Loss: 0.07277745503516478, Val Accuracy: 0.9812898089171974\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.94it/s, accuracy=1, loss=0.000167, lr=1.88e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 30, Val Loss: 0.074091750803718, Val Accuracy: 0.981687898089172\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.71it/s, accuracy=1, loss=0.0473, lr=0.0004]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Val Loss: 0.1700799769371938, Val Accuracy: 0.9513335987261147\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.03it/s, accuracy=0.906, loss=0.326, lr=0.00036]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2, Val Loss: 0.15096695331417642, Val Accuracy: 0.9556130573248408\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.49it/s, accuracy=1, loss=0.0326, lr=0.000324]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3, Val Loss: 0.0949281652518815, Val Accuracy: 0.9703423566878981\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.58it/s, accuracy=1, loss=0.064, lr=0.000292]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4, Val Loss: 0.09743918449422166, Val Accuracy: 0.9693471337579618\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.81it/s, accuracy=1, loss=0.0563, lr=0.000262]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5, Val Loss: 0.09240742413635561, Val Accuracy: 0.9722332802547771\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.68it/s, accuracy=1, loss=0.0196, lr=0.000236]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6, Val Loss: 0.07970927421438839, Val Accuracy: 0.9751194267515924\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.10it/s, accuracy=1, loss=0.0202, lr=0.000213]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7, Val Loss: 0.0766555011286519, Val Accuracy: 0.9757165605095541\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.93it/s, accuracy=1, loss=0.00337, lr=0.000191]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8, Val Loss: 0.07848570007344435, Val Accuracy: 0.9743232484076433\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.23it/s, accuracy=1, loss=0.0239, lr=0.000172]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 9, Val Loss: 0.07731982803679958, Val Accuracy: 0.9755175159235668\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.45it/s, accuracy=1, loss=0.00163, lr=0.000155]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10, Val Loss: 0.08362125514154314, Val Accuracy: 0.9747213375796179\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.85it/s, accuracy=1, loss=0.0244, lr=0.000139]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11, Val Loss: 0.07550541126609715, Val Accuracy: 0.9762141719745223\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.67it/s, accuracy=1, loss=0.0108, lr=0.000126]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 12, Val Loss: 0.06599270288796245, Val Accuracy: 0.9784036624203821\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.44it/s, accuracy=1, loss=0.00642, lr=0.000113]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 13, Val Loss: 0.06897054398621456, Val Accuracy: 0.9798964968152867\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.67it/s, accuracy=1, loss=0.00108, lr=0.000102]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 14, Val Loss: 0.06738470393701722, Val Accuracy: 0.9789012738853503\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.22it/s, accuracy=1, loss=0.00099, lr=9.15e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 15, Val Loss: 0.0760952129100941, Val Accuracy: 0.9768113057324841\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.16it/s, accuracy=1, loss=0.00801, lr=8.24e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 16, Val Loss: 0.06313110442993759, Val Accuracy: 0.9793988853503185\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.31it/s, accuracy=1, loss=0.000641, lr=7.41e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 17, Val Loss: 0.06445298299464863, Val Accuracy: 0.9795979299363057\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.30it/s, accuracy=1, loss=0.00262, lr=6.67e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 18, Val Loss: 0.05678971621269901, Val Accuracy: 0.9830812101910829\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.66it/s, accuracy=1, loss=0.00446, lr=6e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 19, Val Loss: 0.06369758045851776, Val Accuracy: 0.9814888535031847\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.69it/s, accuracy=1, loss=0.00988, lr=5.4e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 20, Val Loss: 0.06776050908321768, Val Accuracy: 0.9796974522292994\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.83it/s, accuracy=1, loss=0.00421, lr=4.86e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 21, Val Loss: 0.06104023119469333, Val Accuracy: 0.9814888535031847\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.66it/s, accuracy=1, loss=0.00084, lr=4.38e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 22, Val Loss: 0.05943897162711194, Val Accuracy: 0.9823845541401274\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.57it/s, accuracy=1, loss=0.00114, lr=3.94e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 23, Val Loss: 0.061792411656367056, Val Accuracy: 0.9817874203821656\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.62it/s, accuracy=1, loss=0.000553, lr=3.55e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 24, Val Loss: 0.059527349916270914, Val Accuracy: 0.9822850318471338\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.99it/s, accuracy=1, loss=0.00124, lr=3.19e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 25, Val Loss: 0.08111935976629848, Val Accuracy: 0.9750199044585988\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.24it/s, accuracy=1, loss=0.000256, lr=2.87e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 26, Val Loss: 0.05942840575073588, Val Accuracy: 0.9820859872611465\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.91it/s, accuracy=1, loss=0.00123, lr=2.58e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 27, Val Loss: 0.06071838898811145, Val Accuracy: 0.9825835987261147\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.31it/s, accuracy=1, loss=0.000385, lr=2.33e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 28, Val Loss: 0.06279815924758883, Val Accuracy: 0.9821855095541401\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.15it/s, accuracy=1, loss=0.000989, lr=2.09e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 29, Val Loss: 0.062220685771112616, Val Accuracy: 0.9823845541401274\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.55it/s, accuracy=1, loss=0.000655, lr=1.88e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 30, Val Loss: 0.06078146737749263, Val Accuracy: 0.9831807324840764\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.21it/s, accuracy=0.969, loss=0.065, lr=0.0004]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Val Loss: 0.17548179410540374, Val Accuracy: 0.9486464968152867\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.32it/s, accuracy=0.906, loss=0.409, lr=0.00036]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2, Val Loss: 0.15340641447621736, Val Accuracy: 0.9518312101910829\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.28it/s, accuracy=1, loss=0.0681, lr=0.000324]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3, Val Loss: 0.10302813356938019, Val Accuracy: 0.9684514331210191\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.73it/s, accuracy=0.969, loss=0.0763, lr=0.000292]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4, Val Loss: 0.10051607874601748, Val Accuracy: 0.9697452229299363\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.96it/s, accuracy=1, loss=0.0468, lr=0.000262]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5, Val Loss: 0.08756824924497847, Val Accuracy: 0.9731289808917197\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.36it/s, accuracy=1, loss=0.0673, lr=0.000236]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6, Val Loss: 0.08189635840096292, Val Accuracy: 0.9743232484076433\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.81it/s, accuracy=1, loss=0.0136, lr=0.000213]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7, Val Loss: 0.08237372451108899, Val Accuracy: 0.9731289808917197\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.21it/s, accuracy=1, loss=0.0156, lr=0.000191]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8, Val Loss: 0.08998770220427638, Val Accuracy: 0.9712380573248408\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.00it/s, accuracy=1, loss=0.016, lr=0.000172]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 9, Val Loss: 0.0651436332599512, Val Accuracy: 0.9778065286624203\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.11it/s, accuracy=1, loss=0.00632, lr=0.000155]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10, Val Loss: 0.06419560011508904, Val Accuracy: 0.9792993630573248\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.11it/s, accuracy=0.969, loss=0.0767, lr=0.000139]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11, Val Loss: 0.06896201538805892, Val Accuracy: 0.9790007961783439\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.20it/s, accuracy=1, loss=0.0192, lr=0.000126]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 12, Val Loss: 0.062322741649469514, Val Accuracy: 0.9790007961783439\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.15it/s, accuracy=1, loss=0.0203, lr=0.000113]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 13, Val Loss: 0.053722311874775955, Val Accuracy: 0.982484076433121\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.93it/s, accuracy=1, loss=0.00558, lr=0.000102]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 14, Val Loss: 0.05891142046617739, Val Accuracy: 0.9806926751592356\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.82it/s, accuracy=1, loss=0.00242, lr=9.15e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 15, Val Loss: 0.055677576549925875, Val Accuracy: 0.9817874203821656\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.86it/s, accuracy=1, loss=0.00363, lr=8.24e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 16, Val Loss: 0.051556871264003615, Val Accuracy: 0.9834792993630573\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.27it/s, accuracy=1, loss=0.00143, lr=7.41e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 17, Val Loss: 0.05160354531149878, Val Accuracy: 0.9845740445859873\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.98it/s, accuracy=1, loss=0.0185, lr=6.67e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 18, Val Loss: 0.049273361940942347, Val Accuracy: 0.9837778662420382\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.09it/s, accuracy=1, loss=0.00567, lr=6e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 19, Val Loss: 0.04936294545309958, Val Accuracy: 0.9834792993630573\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.77it/s, accuracy=1, loss=0.0143, lr=5.4e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 20, Val Loss: 0.051825905737028145, Val Accuracy: 0.9828821656050956\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.88it/s, accuracy=1, loss=0.02, lr=4.86e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 21, Val Loss: 0.050582466255702896, Val Accuracy: 0.984375\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.77it/s, accuracy=1, loss=0.00294, lr=4.38e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 22, Val Loss: 0.051275268720279765, Val Accuracy: 0.9835788216560509\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.00it/s, accuracy=1, loss=0.00806, lr=3.94e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 23, Val Loss: 0.04998159193984199, Val Accuracy: 0.9842754777070064\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.05it/s, accuracy=1, loss=0.00337, lr=3.55e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 24, Val Loss: 0.047994964099891416, Val Accuracy: 0.9840764331210191\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.01it/s, accuracy=1, loss=0.00197, lr=3.19e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 25, Val Loss: 0.04705077828723562, Val Accuracy: 0.9851711783439491\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.78it/s, accuracy=1, loss=0.00142, lr=2.87e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 26, Val Loss: 0.04655806879338435, Val Accuracy: 0.9853702229299363\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.69it/s, accuracy=1, loss=0.00797, lr=2.58e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 27, Val Loss: 0.04764530213125596, Val Accuracy: 0.9845740445859873\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.81it/s, accuracy=1, loss=0.0019, lr=2.33e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 28, Val Loss: 0.048083726238925754, Val Accuracy: 0.9844745222929936\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.25it/s, accuracy=1, loss=0.0133, lr=2.09e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 29, Val Loss: 0.04769496622908368, Val Accuracy: 0.9840764331210191\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.03it/s, accuracy=1, loss=0.0045, lr=1.88e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 30, Val Loss: 0.046017297768861924, Val Accuracy: 0.9851711783439491\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.80it/s, accuracy=1, loss=0.0594, lr=0.0004]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Val Loss: 0.17526146214882468, Val Accuracy: 0.9483479299363057\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.16it/s, accuracy=0.875, loss=0.448, lr=0.00036]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2, Val Loss: 0.1598278251980805, Val Accuracy: 0.9502388535031847\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.26it/s, accuracy=1, loss=0.0773, lr=0.000324]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3, Val Loss: 0.12038892472850954, Val Accuracy: 0.9667595541401274\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.51it/s, accuracy=0.969, loss=0.0827, lr=0.000292]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4, Val Loss: 0.11797106626093577, Val Accuracy: 0.9645700636942676\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.67it/s, accuracy=0.969, loss=0.0635, lr=0.000262]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5, Val Loss: 0.09331533919491918, Val Accuracy: 0.9713375796178344\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.30it/s, accuracy=0.969, loss=0.0658, lr=0.000236]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6, Val Loss: 0.08213021077814803, Val Accuracy: 0.9753184713375797\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.27it/s, accuracy=1, loss=0.0179, lr=0.000213]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7, Val Loss: 0.07283929120348825, Val Accuracy: 0.9769108280254777\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.56it/s, accuracy=1, loss=0.0222, lr=0.000191]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8, Val Loss: 0.08527460730312819, Val Accuracy: 0.9726313694267515\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.92it/s, accuracy=1, loss=0.00975, lr=0.000172]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 9, Val Loss: 0.06588199884841325, Val Accuracy: 0.9780055732484076\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.23it/s, accuracy=1, loss=0.00861, lr=0.000155]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10, Val Loss: 0.060434201355225696, Val Accuracy: 0.9811902866242038\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.58it/s, accuracy=0.938, loss=0.121, lr=0.000139]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11, Val Loss: 0.06844103464687681, Val Accuracy: 0.9784036624203821\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.30it/s, accuracy=1, loss=0.0447, lr=0.000126]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 12, Val Loss: 0.0667498192818877, Val Accuracy: 0.9789012738853503\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.40it/s, accuracy=1, loss=0.0395, lr=0.000113]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 13, Val Loss: 0.05696409771138814, Val Accuracy: 0.9821855095541401\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.39it/s, accuracy=1, loss=0.0124, lr=0.000102]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 14, Val Loss: 0.06008705747896616, Val Accuracy: 0.9802945859872612\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.67it/s, accuracy=1, loss=0.0102, lr=9.15e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 15, Val Loss: 0.05943986001590587, Val Accuracy: 0.9802945859872612\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.28it/s, accuracy=1, loss=0.0138, lr=8.24e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 16, Val Loss: 0.05492060884513009, Val Accuracy: 0.9827826433121019\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.69it/s, accuracy=1, loss=0.0063, lr=7.41e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 17, Val Loss: 0.058331807186993156, Val Accuracy: 0.9815883757961783\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.45it/s, accuracy=1, loss=0.0532, lr=6.67e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 18, Val Loss: 0.05401355431416623, Val Accuracy: 0.9837778662420382\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.14it/s, accuracy=1, loss=0.019, lr=6e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 19, Val Loss: 0.0529847058788636, Val Accuracy: 0.9834792993630573\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.87it/s, accuracy=1, loss=0.039, lr=5.4e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 20, Val Loss: 0.055740888359708786, Val Accuracy: 0.9826831210191083\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.61it/s, accuracy=0.969, loss=0.0756, lr=4.86e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 21, Val Loss: 0.053605992355529264, Val Accuracy: 0.9833797770700637\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.49it/s, accuracy=1, loss=0.0178, lr=4.38e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 22, Val Loss: 0.05417897936199928, Val Accuracy: 0.9829816878980892\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.56it/s, accuracy=1, loss=0.0219, lr=3.94e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 23, Val Loss: 0.05561374571688983, Val Accuracy: 0.9820859872611465\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.10it/s, accuracy=1, loss=0.00923, lr=3.55e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 24, Val Loss: 0.05084598212889218, Val Accuracy: 0.9831807324840764\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.79it/s, accuracy=1, loss=0.00641, lr=3.19e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 25, Val Loss: 0.05059250823376332, Val Accuracy: 0.9837778662420382\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.89it/s, accuracy=1, loss=0.00683, lr=2.87e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 26, Val Loss: 0.04914428156972596, Val Accuracy: 0.9837778662420382\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.00it/s, accuracy=0.969, loss=0.0571, lr=2.58e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 27, Val Loss: 0.050159918492108255, Val Accuracy: 0.9831807324840764\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.92it/s, accuracy=1, loss=0.00621, lr=2.33e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 28, Val Loss: 0.05109463986265177, Val Accuracy: 0.9837778662420382\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.89it/s, accuracy=1, loss=0.049, lr=2.09e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 29, Val Loss: 0.050402699629917065, Val Accuracy: 0.9844745222929936\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.46it/s, accuracy=1, loss=0.0215, lr=1.88e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 30, Val Loss: 0.04801143857277485, Val Accuracy: 0.9851711783439491\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.12it/s, accuracy=1, loss=0.0434, lr=0.0004]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Val Loss: 0.17357781664675967, Val Accuracy: 0.9498407643312102\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.12it/s, accuracy=0.875, loss=0.443, lr=0.00036]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2, Val Loss: 0.18351807336139073, Val Accuracy: 0.9434713375796179\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.27it/s, accuracy=1, loss=0.0621, lr=0.000324]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3, Val Loss: 0.12313766399350991, Val Accuracy: 0.9657643312101911\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.60it/s, accuracy=0.969, loss=0.0875, lr=0.000292]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4, Val Loss: 0.11008790161780965, Val Accuracy: 0.9686504777070064\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.04it/s, accuracy=1, loss=0.0667, lr=0.000262]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5, Val Loss: 0.09112195906697944, Val Accuracy: 0.9730294585987261\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.92it/s, accuracy=1, loss=0.0687, lr=0.000236]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6, Val Loss: 0.09152157333009182, Val Accuracy: 0.9724323248407644\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.83it/s, accuracy=1, loss=0.0227, lr=0.000213]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7, Val Loss: 0.08325190221665392, Val Accuracy: 0.9746218152866242\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.12it/s, accuracy=1, loss=0.0364, lr=0.000191]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8, Val Loss: 0.09387975775749441, Val Accuracy: 0.9715366242038217\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.85it/s, accuracy=1, loss=0.0195, lr=0.000172]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 9, Val Loss: 0.0863392103017029, Val Accuracy: 0.9734275477707006\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.69it/s, accuracy=1, loss=0.0199, lr=0.000155]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10, Val Loss: 0.07702889105684724, Val Accuracy: 0.9778065286624203\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.18it/s, accuracy=0.906, loss=0.195, lr=0.000139]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11, Val Loss: 0.08540395909926267, Val Accuracy: 0.9728304140127388\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.73it/s, accuracy=1, loss=0.063, lr=0.000126]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 12, Val Loss: 0.07970815769375961, Val Accuracy: 0.9759156050955414\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.59it/s, accuracy=0.969, loss=0.0778, lr=0.000113]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 13, Val Loss: 0.07197866778506358, Val Accuracy: 0.979796974522293\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.96it/s, accuracy=1, loss=0.0244, lr=0.000102]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 14, Val Loss: 0.07435854127375516, Val Accuracy: 0.977906050955414\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.35it/s, accuracy=1, loss=0.0101, lr=9.15e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 15, Val Loss: 0.07266770059666981, Val Accuracy: 0.9784036624203821\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.27it/s, accuracy=0.969, loss=0.0581, lr=8.24e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 16, Val Loss: 0.07256174511884214, Val Accuracy: 0.9785031847133758\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.76it/s, accuracy=1, loss=0.017, lr=7.41e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 17, Val Loss: 0.0707587476606202, Val Accuracy: 0.9791003184713376\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.45it/s, accuracy=0.938, loss=0.119, lr=6.67e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 18, Val Loss: 0.06807942520663925, Val Accuracy: 0.9802945859872612\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.66it/s, accuracy=1, loss=0.0318, lr=6e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 19, Val Loss: 0.06681404269751233, Val Accuracy: 0.9813893312101911\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.01it/s, accuracy=0.938, loss=0.105, lr=5.4e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 20, Val Loss: 0.06745042937072172, Val Accuracy: 0.9795979299363057\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.49it/s, accuracy=0.938, loss=0.122, lr=4.86e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 21, Val Loss: 0.06672204049470223, Val Accuracy: 0.9793988853503185\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.88it/s, accuracy=1, loss=0.0474, lr=4.38e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 22, Val Loss: 0.0669352571316105, Val Accuracy: 0.9801950636942676\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.52it/s, accuracy=1, loss=0.0525, lr=3.94e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 23, Val Loss: 0.06817983333644262, Val Accuracy: 0.9790007961783439\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.18it/s, accuracy=1, loss=0.0142, lr=3.55e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 24, Val Loss: 0.06441727825692577, Val Accuracy: 0.9808917197452229\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.86it/s, accuracy=1, loss=0.0155, lr=3.19e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 25, Val Loss: 0.06214599847752055, Val Accuracy: 0.9820859872611465\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.22it/s, accuracy=1, loss=0.0251, lr=2.87e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 26, Val Loss: 0.06184368273228478, Val Accuracy: 0.981687898089172\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.32it/s, accuracy=0.969, loss=0.0954, lr=2.58e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 27, Val Loss: 0.06265558888557349, Val Accuracy: 0.9809912420382165\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.48it/s, accuracy=1, loss=0.0187, lr=2.33e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 28, Val Loss: 0.06349251327285792, Val Accuracy: 0.9811902866242038\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.45it/s, accuracy=0.938, loss=0.0687, lr=2.09e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 29, Val Loss: 0.06298046139629475, Val Accuracy: 0.9818869426751592\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.26it/s, accuracy=1, loss=0.0575, lr=1.88e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 30, Val Loss: 0.06067556608097545, Val Accuracy: 0.9826831210191083\n"
          ]
        }
      ],
      "source": [
        "epochs = 30\n",
        "wds = [0.01, 0.1, 0.25, 0.5, 0.75, 1.]\n",
        "wd_accs = np.empty((6, epochs))\n",
        "for h, wd in enumerate(wds):\n",
        "    torch.manual_seed(42)\n",
        "    best_acc = 0\n",
        "    # Define model\n",
        "    model = SineKAN(layers_hidden=[28 * 28, hdim, 10], grid_size=8)\n",
        "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "    model.to(device)\n",
        "    # Define optimizer\n",
        "    optimizer = optim.AdamW(model.parameters(), lr=4e-4, weight_decay=wd)\n",
        "    # Define learning rate scheduler\n",
        "    scheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=gamma)\n",
        "\n",
        "    # Define loss\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "    for epoch in range(epochs):\n",
        "        # Train\n",
        "        model.train()\n",
        "        with tqdm(train_loader) as pbar:\n",
        "            for i, (images, labels) in enumerate(pbar):\n",
        "                images = images.view(-1, 28 * 28).to(device)\n",
        "                optimizer.zero_grad()\n",
        "                output = model(images)\n",
        "                loss = criterion(output, labels.to(device))\n",
        "                loss.backward()\n",
        "                optimizer.step()\n",
        "                accuracy = (output.argmax(dim=1) == labels.to(device)).float().mean()\n",
        "                pbar.set_postfix(loss=loss.item(), accuracy=accuracy.item(), lr=optimizer.param_groups[0]['lr'])\n",
        "\n",
        "        # Validation\n",
        "        model.eval()\n",
        "        val_loss = 0\n",
        "        val_accuracy = 0\n",
        "        with torch.no_grad():\n",
        "            for images, labels in val_loader:\n",
        "                images = images.view(-1, 28 * 28).to(device)\n",
        "                output = model(images)\n",
        "                val_loss += criterion(output, labels.to(device)).item()\n",
        "                val_accuracy += (\n",
        "                    (output.argmax(dim=1) == labels.to(device)).float().mean().item()\n",
        "                )\n",
        "\n",
        "        val_loss /= len(val_loader)\n",
        "        val_accuracy /= len(val_loader)\n",
        "        wd_accs[h, epoch] = val_accuracy\n",
        "\n",
        "        # Update learning rate\n",
        "        scheduler.step()\n",
        "\n",
        "        print(\n",
        "            f\"Epoch {epoch + 1}, Val Loss: {val_loss}, Val Accuracy: {val_accuracy}\"\n",
        "        )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UvJ4YGFGdyui",
        "outputId": "0395f43e-1acf-40a7-8d9e-fb4ae95d13e2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.65it/s, accuracy=0.906, loss=0.269, lr=0.0004]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Val Loss: 0.31872255487996304, Val Accuracy: 0.9132165605095541\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 46.99it/s, accuracy=0.969, loss=0.153, lr=0.00036]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2, Val Loss: 0.28569505121677546, Val Accuracy: 0.9208797770700637\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.96it/s, accuracy=0.875, loss=0.244, lr=0.000324]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3, Val Loss: 0.2380579393001119, Val Accuracy: 0.9288415605095541\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.56it/s, accuracy=0.969, loss=0.181, lr=0.000292]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4, Val Loss: 0.20225586344126115, Val Accuracy: 0.93859474522293\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.75it/s, accuracy=0.875, loss=0.302, lr=0.000262]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5, Val Loss: 0.20422641249598972, Val Accuracy: 0.9428742038216561\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.60it/s, accuracy=0.938, loss=0.159, lr=0.000236]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6, Val Loss: 0.19887586605325816, Val Accuracy: 0.943968949044586\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.37it/s, accuracy=0.969, loss=0.13, lr=0.000213]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7, Val Loss: 0.1929992445726778, Val Accuracy: 0.9426751592356688\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.76it/s, accuracy=0.969, loss=0.0999, lr=0.000191]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8, Val Loss: 0.16667827789428508, Val Accuracy: 0.9549164012738853\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.15it/s, accuracy=0.938, loss=0.179, lr=0.000172]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 9, Val Loss: 0.16387633058082335, Val Accuracy: 0.9517316878980892\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.86it/s, accuracy=0.938, loss=0.239, lr=0.000155]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10, Val Loss: 0.16043159327096052, Val Accuracy: 0.9528264331210191\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.45it/s, accuracy=1, loss=0.0848, lr=0.000139]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11, Val Loss: 0.15626705418916265, Val Accuracy: 0.9564092356687898\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.05it/s, accuracy=1, loss=0.0655, lr=0.000126]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 12, Val Loss: 0.16874558964423883, Val Accuracy: 0.9528264331210191\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.33it/s, accuracy=0.938, loss=0.146, lr=0.000113]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 13, Val Loss: 0.15891046652712734, Val Accuracy: 0.9524283439490446\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.60it/s, accuracy=0.969, loss=0.114, lr=0.000102]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 14, Val Loss: 0.15174191722432803, Val Accuracy: 0.956906847133758\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.03it/s, accuracy=0.938, loss=0.161, lr=9.15e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 15, Val Loss: 0.15176966994894064, Val Accuracy: 0.9575039808917197\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.90it/s, accuracy=0.969, loss=0.0836, lr=8.24e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 16, Val Loss: 0.15590978070357991, Val Accuracy: 0.9570063694267515\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.19it/s, accuracy=0.906, loss=0.332, lr=7.41e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 17, Val Loss: 0.15328020204428086, Val Accuracy: 0.9555135350318471\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.48it/s, accuracy=1, loss=0.0526, lr=6.67e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 18, Val Loss: 0.1463602222858151, Val Accuracy: 0.956906847133758\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.55it/s, accuracy=0.938, loss=0.229, lr=6e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 19, Val Loss: 0.1430880326398049, Val Accuracy: 0.9599920382165605\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.48it/s, accuracy=0.969, loss=0.132, lr=5.4e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 20, Val Loss: 0.140851997153442, Val Accuracy: 0.9617834394904459\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.25it/s, accuracy=0.906, loss=0.309, lr=4.86e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 21, Val Loss: 0.13807301287343549, Val Accuracy: 0.960390127388535\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.58it/s, accuracy=0.938, loss=0.218, lr=4.38e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 22, Val Loss: 0.13911410057205398, Val Accuracy: 0.9606886942675159\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.88it/s, accuracy=1, loss=0.0698, lr=3.94e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 23, Val Loss: 0.14023957099279116, Val Accuracy: 0.9583996815286624\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.26it/s, accuracy=0.969, loss=0.0836, lr=3.55e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 24, Val Loss: 0.13853478622118567, Val Accuracy: 0.9598925159235668\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.14it/s, accuracy=0.875, loss=0.189, lr=3.19e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 25, Val Loss: 0.1413019074091486, Val Accuracy: 0.9578025477707006\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 46.97it/s, accuracy=1, loss=0.0233, lr=2.87e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 26, Val Loss: 0.1362035812418552, Val Accuracy: 0.9610867834394905\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.71it/s, accuracy=0.969, loss=0.0944, lr=2.58e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 27, Val Loss: 0.13803730492782632, Val Accuracy: 0.9597929936305732\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.68it/s, accuracy=0.969, loss=0.16, lr=2.33e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 28, Val Loss: 0.13715244471362442, Val Accuracy: 0.9611863057324841\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.13it/s, accuracy=0.969, loss=0.101, lr=2.09e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 29, Val Loss: 0.13683287661439578, Val Accuracy: 0.9601910828025477\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.26it/s, accuracy=1, loss=0.0423, lr=1.88e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 30, Val Loss: 0.13630016991026273, Val Accuracy: 0.9611863057324841\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.29it/s, accuracy=0.875, loss=0.254, lr=0.0004]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Val Loss: 0.2367548702201646, Val Accuracy: 0.9375995222929936\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.24it/s, accuracy=0.906, loss=0.233, lr=0.00036]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2, Val Loss: 0.21063593059635846, Val Accuracy: 0.9377985668789809\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.17it/s, accuracy=0.969, loss=0.119, lr=0.000324]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3, Val Loss: 0.1904696092946799, Val Accuracy: 0.9461584394904459\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.11it/s, accuracy=1, loss=0.0541, lr=0.000292]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4, Val Loss: 0.14181842055219185, Val Accuracy: 0.9605891719745223\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.09it/s, accuracy=0.906, loss=0.175, lr=0.000262]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5, Val Loss: 0.1458548634953702, Val Accuracy: 0.957703025477707\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.65it/s, accuracy=0.938, loss=0.123, lr=0.000236]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6, Val Loss: 0.13923344903204377, Val Accuracy: 0.9576035031847133\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.49it/s, accuracy=1, loss=0.0576, lr=0.000213]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7, Val Loss: 0.13227336037500648, Val Accuracy: 0.9612858280254777\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.77it/s, accuracy=1, loss=0.0381, lr=0.000191]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8, Val Loss: 0.12613620976828466, Val Accuracy: 0.9643710191082803\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.94it/s, accuracy=1, loss=0.0199, lr=0.000172]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 9, Val Loss: 0.11377047529521214, Val Accuracy: 0.9674562101910829\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.63it/s, accuracy=1, loss=0.0608, lr=0.000155]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10, Val Loss: 0.11256546213344973, Val Accuracy: 0.9662619426751592\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.21it/s, accuracy=1, loss=0.0409, lr=0.000139]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11, Val Loss: 0.1114174442775073, Val Accuracy: 0.9675557324840764\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.90it/s, accuracy=0.969, loss=0.0731, lr=0.000126]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 12, Val Loss: 0.10173895654333814, Val Accuracy: 0.9703423566878981\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.29it/s, accuracy=0.969, loss=0.0946, lr=0.000113]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 13, Val Loss: 0.09732371728546965, Val Accuracy: 0.9713375796178344\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.15it/s, accuracy=1, loss=0.0531, lr=0.000102]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 14, Val Loss: 0.10301093146454092, Val Accuracy: 0.9699442675159236\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.07it/s, accuracy=0.969, loss=0.0781, lr=9.15e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 15, Val Loss: 0.09527874397036803, Val Accuracy: 0.9716361464968153\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.40it/s, accuracy=0.969, loss=0.0695, lr=8.24e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 16, Val Loss: 0.08861143754759601, Val Accuracy: 0.9751194267515924\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.36it/s, accuracy=1, loss=0.0283, lr=7.41e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 17, Val Loss: 0.09110723484817679, Val Accuracy: 0.9737261146496815\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.14it/s, accuracy=0.906, loss=0.121, lr=6.67e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 18, Val Loss: 0.09333870550884515, Val Accuracy: 0.9729299363057324\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.44it/s, accuracy=0.969, loss=0.096, lr=6e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 19, Val Loss: 0.08780269959650838, Val Accuracy: 0.9745222929936306\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.99it/s, accuracy=0.969, loss=0.0534, lr=5.4e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 20, Val Loss: 0.08595059178977209, Val Accuracy: 0.9737261146496815\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.28it/s, accuracy=1, loss=0.0257, lr=4.86e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 21, Val Loss: 0.08510990664778147, Val Accuracy: 0.9743232484076433\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.60it/s, accuracy=1, loss=0.0239, lr=4.38e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 22, Val Loss: 0.08304276800073779, Val Accuracy: 0.9764132165605095\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.60it/s, accuracy=1, loss=0.038, lr=3.94e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 23, Val Loss: 0.08389217841400391, Val Accuracy: 0.9748208598726115\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.35it/s, accuracy=1, loss=0.0573, lr=3.55e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 24, Val Loss: 0.08456381730836146, Val Accuracy: 0.9747213375796179\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.83it/s, accuracy=0.969, loss=0.138, lr=3.19e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 25, Val Loss: 0.08433212004720596, Val Accuracy: 0.9740246815286624\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.48it/s, accuracy=1, loss=0.0333, lr=2.87e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 26, Val Loss: 0.08287518858541823, Val Accuracy: 0.9755175159235668\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.96it/s, accuracy=1, loss=0.0192, lr=2.58e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 27, Val Loss: 0.08209055102175208, Val Accuracy: 0.9767117834394905\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.88it/s, accuracy=1, loss=0.0671, lr=2.33e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 28, Val Loss: 0.08195840308938034, Val Accuracy: 0.9745222929936306\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.08it/s, accuracy=1, loss=0.0136, lr=2.09e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 29, Val Loss: 0.08353482441128392, Val Accuracy: 0.9747213375796179\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.58it/s, accuracy=1, loss=0.0314, lr=1.88e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 30, Val Loss: 0.08148996536338192, Val Accuracy: 0.9754179936305732\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.11it/s, accuracy=1, loss=0.0861, lr=0.0004]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Val Loss: 0.19685669394601493, Val Accuracy: 0.943172770700637\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.24it/s, accuracy=0.938, loss=0.321, lr=0.00036]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2, Val Loss: 0.1610914072093026, Val Accuracy: 0.9515326433121019\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.68it/s, accuracy=0.969, loss=0.206, lr=0.000324]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3, Val Loss: 0.13378084996703324, Val Accuracy: 0.9598925159235668\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.52it/s, accuracy=0.969, loss=0.0616, lr=0.000292]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4, Val Loss: 0.11356679838192264, Val Accuracy: 0.966859076433121\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.76it/s, accuracy=1, loss=0.0422, lr=0.000262]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5, Val Loss: 0.10033833321071212, Val Accuracy: 0.9691480891719745\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.39it/s, accuracy=1, loss=0.0207, lr=0.000236]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6, Val Loss: 0.12378003387889902, Val Accuracy: 0.9634753184713376\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.13it/s, accuracy=1, loss=0.0449, lr=0.000213]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7, Val Loss: 0.10303637796455319, Val Accuracy: 0.9690485668789809\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.53it/s, accuracy=0.969, loss=0.0558, lr=0.000191]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8, Val Loss: 0.08854659395084781, Val Accuracy: 0.9731289808917197\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.97it/s, accuracy=1, loss=0.0237, lr=0.000172]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 9, Val Loss: 0.08704052634454172, Val Accuracy: 0.9731289808917197\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.37it/s, accuracy=1, loss=0.0472, lr=0.000155]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10, Val Loss: 0.08384363656525447, Val Accuracy: 0.9745222929936306\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.42it/s, accuracy=0.969, loss=0.0639, lr=0.000139]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11, Val Loss: 0.07937076865401783, Val Accuracy: 0.9758160828025477\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.39it/s, accuracy=0.969, loss=0.0373, lr=0.000126]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 12, Val Loss: 0.07729054615123992, Val Accuracy: 0.9767117834394905\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.92it/s, accuracy=1, loss=0.0417, lr=0.000113]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 13, Val Loss: 0.07569953876141174, Val Accuracy: 0.9762141719745223\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.13it/s, accuracy=1, loss=0.0209, lr=0.000102]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 14, Val Loss: 0.06989506123198944, Val Accuracy: 0.9780055732484076\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.34it/s, accuracy=1, loss=0.00602, lr=9.15e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 15, Val Loss: 0.07292693676753266, Val Accuracy: 0.9768113057324841\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.62it/s, accuracy=1, loss=0.0142, lr=8.24e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 16, Val Loss: 0.0728289903902336, Val Accuracy: 0.9769108280254777\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.92it/s, accuracy=1, loss=0.00709, lr=7.41e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 17, Val Loss: 0.070038186789603, Val Accuracy: 0.9769108280254777\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.50it/s, accuracy=1, loss=0.00955, lr=6.67e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 18, Val Loss: 0.06716323781841595, Val Accuracy: 0.9791998407643312\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.13it/s, accuracy=1, loss=0.0266, lr=6e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 19, Val Loss: 0.06611016417751948, Val Accuracy: 0.9802945859872612\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.63it/s, accuracy=1, loss=0.0488, lr=5.4e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 20, Val Loss: 0.0693990195794387, Val Accuracy: 0.9780055732484076\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.09it/s, accuracy=1, loss=0.019, lr=4.86e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 21, Val Loss: 0.061819307731870846, Val Accuracy: 0.9806926751592356\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.44it/s, accuracy=1, loss=0.0304, lr=4.38e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 22, Val Loss: 0.06433374262496437, Val Accuracy: 0.9798964968152867\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.13it/s, accuracy=1, loss=0.0035, lr=3.94e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 23, Val Loss: 0.06244217636023654, Val Accuracy: 0.9798964968152867\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.24it/s, accuracy=1, loss=0.00531, lr=3.55e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 24, Val Loss: 0.0635205707130924, Val Accuracy: 0.9793988853503185\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.57it/s, accuracy=1, loss=0.0117, lr=3.19e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 25, Val Loss: 0.06055089434057142, Val Accuracy: 0.981687898089172\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.43it/s, accuracy=0.969, loss=0.0831, lr=2.87e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 26, Val Loss: 0.061368629759196845, Val Accuracy: 0.9814888535031847\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.44it/s, accuracy=1, loss=0.00952, lr=2.58e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 27, Val Loss: 0.060693426166140554, Val Accuracy: 0.9807921974522293\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.18it/s, accuracy=1, loss=0.0166, lr=2.33e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 28, Val Loss: 0.05989546352325624, Val Accuracy: 0.9818869426751592\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.11it/s, accuracy=1, loss=0.00146, lr=2.09e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 29, Val Loss: 0.06113419016329334, Val Accuracy: 0.9808917197452229\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.34it/s, accuracy=1, loss=0.00666, lr=1.88e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 30, Val Loss: 0.06067516471931835, Val Accuracy: 0.9809912420382165\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.09it/s, accuracy=0.906, loss=0.253, lr=0.0004]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Val Loss: 0.2376669420724272, Val Accuracy: 0.9300358280254777\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.80it/s, accuracy=0.969, loss=0.0905, lr=0.00036]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2, Val Loss: 0.140094636281015, Val Accuracy: 0.9607882165605095\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.45it/s, accuracy=0.938, loss=0.227, lr=0.000324]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3, Val Loss: 0.138253404503795, Val Accuracy: 0.9580015923566879\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.39it/s, accuracy=0.969, loss=0.0832, lr=0.000292]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4, Val Loss: 0.09247697657270795, Val Accuracy: 0.9719347133757962\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.31it/s, accuracy=1, loss=0.045, lr=0.000262]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5, Val Loss: 0.09638880232829529, Val Accuracy: 0.9707404458598726\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.61it/s, accuracy=1, loss=0.0321, lr=0.000236]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6, Val Loss: 0.08842214876504105, Val Accuracy: 0.9722332802547771\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.38it/s, accuracy=0.969, loss=0.105, lr=0.000213]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7, Val Loss: 0.07661282350042253, Val Accuracy: 0.9764132165605095\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.76it/s, accuracy=1, loss=0.00909, lr=0.000191]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8, Val Loss: 0.07539925703209392, Val Accuracy: 0.9772093949044586\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.86it/s, accuracy=1, loss=0.00328, lr=0.000172]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 9, Val Loss: 0.07426631024884428, Val Accuracy: 0.9759156050955414\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.63it/s, accuracy=1, loss=0.0291, lr=0.000155]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10, Val Loss: 0.06959543417276115, Val Accuracy: 0.9793988853503185\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.99it/s, accuracy=1, loss=0.00996, lr=0.000139]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11, Val Loss: 0.06735008772481886, Val Accuracy: 0.9789012738853503\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.79it/s, accuracy=0.969, loss=0.0725, lr=0.000126]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 12, Val Loss: 0.06572348916877928, Val Accuracy: 0.9801950636942676\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.18it/s, accuracy=1, loss=0.0067, lr=0.000113]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 13, Val Loss: 0.059839082193326586, Val Accuracy: 0.9802945859872612\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 46.92it/s, accuracy=0.969, loss=0.061, lr=0.000102]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 14, Val Loss: 0.058253191914231436, Val Accuracy: 0.9813893312101911\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.03it/s, accuracy=1, loss=0.00746, lr=9.15e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 15, Val Loss: 0.057604046388504034, Val Accuracy: 0.9815883757961783\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.74it/s, accuracy=1, loss=0.00717, lr=8.24e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 16, Val Loss: 0.05267927911285323, Val Accuracy: 0.9833797770700637\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.63it/s, accuracy=1, loss=0.00541, lr=7.41e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 17, Val Loss: 0.05175493115182539, Val Accuracy: 0.9830812101910829\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.28it/s, accuracy=1, loss=0.00914, lr=6.67e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 18, Val Loss: 0.05175672025993692, Val Accuracy: 0.9830812101910829\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.92it/s, accuracy=1, loss=0.0114, lr=6e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 19, Val Loss: 0.05490316013333975, Val Accuracy: 0.9814888535031847\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.76it/s, accuracy=1, loss=0.009, lr=5.4e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 20, Val Loss: 0.05216238197739307, Val Accuracy: 0.9837778662420382\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.48it/s, accuracy=1, loss=0.00521, lr=4.86e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 21, Val Loss: 0.0508287643194777, Val Accuracy: 0.9834792993630573\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.40it/s, accuracy=1, loss=0.006, lr=4.38e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 22, Val Loss: 0.051809068565860476, Val Accuracy: 0.9838773885350318\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.90it/s, accuracy=1, loss=0.00932, lr=3.94e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 23, Val Loss: 0.04885052733870733, Val Accuracy: 0.9847730891719745\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.23it/s, accuracy=1, loss=0.00578, lr=3.55e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 24, Val Loss: 0.04849254644578204, Val Accuracy: 0.9847730891719745\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.12it/s, accuracy=1, loss=0.00996, lr=3.19e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 25, Val Loss: 0.04771078925901968, Val Accuracy: 0.9848726114649682\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.88it/s, accuracy=1, loss=0.00655, lr=2.87e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 26, Val Loss: 0.0496796149146709, Val Accuracy: 0.9845740445859873\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.94it/s, accuracy=1, loss=0.00123, lr=2.58e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 27, Val Loss: 0.048426733000495374, Val Accuracy: 0.984375\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.03it/s, accuracy=1, loss=0.0106, lr=2.33e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 28, Val Loss: 0.048197887821017714, Val Accuracy: 0.9850716560509554\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.05it/s, accuracy=1, loss=0.00241, lr=2.09e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 29, Val Loss: 0.04876376052904625, Val Accuracy: 0.9837778662420382\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.15it/s, accuracy=1, loss=0.00515, lr=1.88e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 30, Val Loss: 0.048022282230211745, Val Accuracy: 0.984375\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.66it/s, accuracy=0.969, loss=0.065, lr=0.0004]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Val Loss: 0.17548179410540374, Val Accuracy: 0.9486464968152867\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.03it/s, accuracy=0.906, loss=0.409, lr=0.00036]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2, Val Loss: 0.15340641447621736, Val Accuracy: 0.9518312101910829\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.54it/s, accuracy=1, loss=0.0681, lr=0.000324]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3, Val Loss: 0.10302813356938019, Val Accuracy: 0.9684514331210191\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.01it/s, accuracy=0.969, loss=0.0763, lr=0.000292]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4, Val Loss: 0.10051607874601748, Val Accuracy: 0.9697452229299363\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.00it/s, accuracy=1, loss=0.0468, lr=0.000262]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5, Val Loss: 0.08756824924497847, Val Accuracy: 0.9731289808917197\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.81it/s, accuracy=1, loss=0.0673, lr=0.000236]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6, Val Loss: 0.08189635840096292, Val Accuracy: 0.9743232484076433\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.10it/s, accuracy=1, loss=0.0136, lr=0.000213]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7, Val Loss: 0.08237372451108899, Val Accuracy: 0.9731289808917197\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.83it/s, accuracy=1, loss=0.0156, lr=0.000191]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8, Val Loss: 0.08998770220427638, Val Accuracy: 0.9712380573248408\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.62it/s, accuracy=1, loss=0.016, lr=0.000172]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 9, Val Loss: 0.0651436332599512, Val Accuracy: 0.9778065286624203\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.80it/s, accuracy=1, loss=0.00632, lr=0.000155]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10, Val Loss: 0.06419560011508904, Val Accuracy: 0.9792993630573248\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 46.94it/s, accuracy=0.969, loss=0.0767, lr=0.000139]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11, Val Loss: 0.06896201538805892, Val Accuracy: 0.9790007961783439\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.31it/s, accuracy=1, loss=0.0192, lr=0.000126]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 12, Val Loss: 0.062322741649469514, Val Accuracy: 0.9790007961783439\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.52it/s, accuracy=1, loss=0.0203, lr=0.000113]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 13, Val Loss: 0.053722311874775955, Val Accuracy: 0.982484076433121\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.91it/s, accuracy=1, loss=0.00558, lr=0.000102]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 14, Val Loss: 0.05891142046617739, Val Accuracy: 0.9806926751592356\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.20it/s, accuracy=1, loss=0.00242, lr=9.15e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 15, Val Loss: 0.055677576549925875, Val Accuracy: 0.9817874203821656\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.35it/s, accuracy=1, loss=0.00363, lr=8.24e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 16, Val Loss: 0.051556871264003615, Val Accuracy: 0.9834792993630573\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.48it/s, accuracy=1, loss=0.00143, lr=7.41e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 17, Val Loss: 0.05160354531149878, Val Accuracy: 0.9845740445859873\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.61it/s, accuracy=1, loss=0.0185, lr=6.67e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 18, Val Loss: 0.049273361940942347, Val Accuracy: 0.9837778662420382\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.53it/s, accuracy=1, loss=0.00567, lr=6e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 19, Val Loss: 0.04936294545309958, Val Accuracy: 0.9834792993630573\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.63it/s, accuracy=1, loss=0.0143, lr=5.4e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 20, Val Loss: 0.051825905737028145, Val Accuracy: 0.9828821656050956\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.27it/s, accuracy=1, loss=0.02, lr=4.86e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 21, Val Loss: 0.050582466255702896, Val Accuracy: 0.984375\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.13it/s, accuracy=1, loss=0.00294, lr=4.38e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 22, Val Loss: 0.051275268720279765, Val Accuracy: 0.9835788216560509\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.33it/s, accuracy=1, loss=0.00806, lr=3.94e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 23, Val Loss: 0.04998159193984199, Val Accuracy: 0.9842754777070064\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.99it/s, accuracy=1, loss=0.00337, lr=3.55e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 24, Val Loss: 0.047994964099891416, Val Accuracy: 0.9840764331210191\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.32it/s, accuracy=1, loss=0.00197, lr=3.19e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 25, Val Loss: 0.04705077828723562, Val Accuracy: 0.9851711783439491\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.98it/s, accuracy=1, loss=0.00142, lr=2.87e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 26, Val Loss: 0.04655806879338435, Val Accuracy: 0.9853702229299363\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.70it/s, accuracy=1, loss=0.00797, lr=2.58e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 27, Val Loss: 0.04764530213125596, Val Accuracy: 0.9845740445859873\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.50it/s, accuracy=1, loss=0.0019, lr=2.33e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 28, Val Loss: 0.048083726238925754, Val Accuracy: 0.9844745222929936\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.08it/s, accuracy=1, loss=0.0133, lr=2.09e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 29, Val Loss: 0.04769496622908368, Val Accuracy: 0.9840764331210191\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.68it/s, accuracy=1, loss=0.0045, lr=1.88e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 30, Val Loss: 0.046017297768861924, Val Accuracy: 0.9851711783439491\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.50it/s, accuracy=0.969, loss=0.103, lr=0.0004]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Val Loss: 0.20596922212725233, Val Accuracy: 0.9373009554140127\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.22it/s, accuracy=0.875, loss=0.389, lr=0.00036]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2, Val Loss: 0.1664285448166263, Val Accuracy: 0.9490445859872612\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.26it/s, accuracy=0.875, loss=0.336, lr=0.000324]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3, Val Loss: 0.11054814091266672, Val Accuracy: 0.9650676751592356\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.43it/s, accuracy=1, loss=0.0244, lr=0.000292]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4, Val Loss: 0.10040905344697415, Val Accuracy: 0.9694466560509554\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.72it/s, accuracy=1, loss=0.0444, lr=0.000262]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5, Val Loss: 0.10976943268683875, Val Accuracy: 0.9669585987261147\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.75it/s, accuracy=1, loss=0.0167, lr=0.000236]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6, Val Loss: 0.07934669948925982, Val Accuracy: 0.9743232484076433\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.53it/s, accuracy=1, loss=0.073, lr=0.000213]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7, Val Loss: 0.09651049805299385, Val Accuracy: 0.9702428343949044\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.71it/s, accuracy=1, loss=0.0245, lr=0.000191]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8, Val Loss: 0.08706238322630287, Val Accuracy: 0.9724323248407644\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.48it/s, accuracy=0.969, loss=0.0399, lr=0.000172]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 9, Val Loss: 0.08669924498150114, Val Accuracy: 0.9727308917197452\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.98it/s, accuracy=1, loss=0.00652, lr=0.000155]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10, Val Loss: 0.07193727463233184, Val Accuracy: 0.9763136942675159\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.95it/s, accuracy=1, loss=0.0225, lr=0.000139]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11, Val Loss: 0.06044462621883062, Val Accuracy: 0.9803941082802548\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.70it/s, accuracy=1, loss=0.0165, lr=0.000126]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 12, Val Loss: 0.05324663637566698, Val Accuracy: 0.9828821656050956\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.53it/s, accuracy=1, loss=0.00577, lr=0.000113]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 13, Val Loss: 0.06712974367569094, Val Accuracy: 0.9774084394904459\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.52it/s, accuracy=0.969, loss=0.0437, lr=0.000102]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 14, Val Loss: 0.057156875854318036, Val Accuracy: 0.9809912420382165\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.77it/s, accuracy=1, loss=0.00507, lr=9.15e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 15, Val Loss: 0.05267526015239266, Val Accuracy: 0.9828821656050956\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.89it/s, accuracy=1, loss=0.0209, lr=8.24e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 16, Val Loss: 0.06383360377566631, Val Accuracy: 0.9803941082802548\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.23it/s, accuracy=1, loss=0.00533, lr=7.41e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 17, Val Loss: 0.05107030813703847, Val Accuracy: 0.9818869426751592\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.55it/s, accuracy=0.969, loss=0.0341, lr=6.67e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 18, Val Loss: 0.05499083913212411, Val Accuracy: 0.9829816878980892\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.20it/s, accuracy=1, loss=0.0032, lr=6e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 19, Val Loss: 0.05765954568347401, Val Accuracy: 0.9818869426751592\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.82it/s, accuracy=1, loss=0.00255, lr=5.4e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 20, Val Loss: 0.044878845895330234, Val Accuracy: 0.9856687898089171\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.46it/s, accuracy=1, loss=0.0119, lr=4.86e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 21, Val Loss: 0.04490936822340428, Val Accuracy: 0.9839769108280255\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.60it/s, accuracy=1, loss=0.00358, lr=4.38e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 22, Val Loss: 0.05065804488716312, Val Accuracy: 0.9849721337579618\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.50it/s, accuracy=1, loss=0.0136, lr=3.94e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 23, Val Loss: 0.044558837627429425, Val Accuracy: 0.9852707006369427\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.54it/s, accuracy=1, loss=0.015, lr=3.55e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 24, Val Loss: 0.046556961433093506, Val Accuracy: 0.9849721337579618\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.57it/s, accuracy=1, loss=0.00345, lr=3.19e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 25, Val Loss: 0.04593786664028499, Val Accuracy: 0.9852707006369427\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.87it/s, accuracy=1, loss=0.00371, lr=2.87e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 26, Val Loss: 0.044472209271015736, Val Accuracy: 0.9845740445859873\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.66it/s, accuracy=1, loss=0.00206, lr=2.58e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 27, Val Loss: 0.046480941743208805, Val Accuracy: 0.9852707006369427\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.27it/s, accuracy=1, loss=0.000818, lr=2.33e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 28, Val Loss: 0.044854627632714694, Val Accuracy: 0.9858678343949044\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.59it/s, accuracy=1, loss=0.00576, lr=2.09e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 29, Val Loss: 0.04531116210829591, Val Accuracy: 0.9853702229299363\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.77it/s, accuracy=1, loss=0.00278, lr=1.88e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 30, Val Loss: 0.04356003303425038, Val Accuracy: 0.9859673566878981\n"
          ]
        }
      ],
      "source": [
        "epochs = 30\n",
        "hdims = [16, 32, 64, 128, 256, 512]\n",
        "hdim_accs = np.empty((6, epochs))\n",
        "for h, hdim in enumerate(hdims):\n",
        "    torch.manual_seed(42)\n",
        "    # Define model\n",
        "    model = SineKAN(layers_hidden=[28 * 28, hdim, 10], grid_size=8)\n",
        "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "    model.to(device)\n",
        "    # Define optimizer\n",
        "    optimizer = optim.AdamW(model.parameters(), lr=4e-4, weight_decay=.5)\n",
        "    # Define learning rate scheduler\n",
        "    scheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)\n",
        "\n",
        "    # Define loss\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "    for epoch in range(epochs):\n",
        "        # Train\n",
        "        model.train()\n",
        "        with tqdm(train_loader) as pbar:\n",
        "            for i, (images, labels) in enumerate(pbar):\n",
        "                images = images.view(-1, 28 * 28).to(device)\n",
        "                optimizer.zero_grad()\n",
        "                output = model(images)\n",
        "                loss = criterion(output, labels.to(device))\n",
        "                loss.backward()\n",
        "                optimizer.step()\n",
        "                accuracy = (output.argmax(dim=1) == labels.to(device)).float().mean()\n",
        "                pbar.set_postfix(loss=loss.item(), accuracy=accuracy.item(), lr=optimizer.param_groups[0]['lr'])\n",
        "\n",
        "        # Validation\n",
        "        model.eval()\n",
        "        val_loss = 0\n",
        "        val_accuracy = 0\n",
        "        with torch.no_grad():\n",
        "            for images, labels in val_loader:\n",
        "                images = images.view(-1, 28 * 28).to(device)\n",
        "                output = model(images)\n",
        "                val_loss += criterion(output, labels.to(device)).item()\n",
        "                val_accuracy += (\n",
        "                    (output.argmax(dim=1) == labels.to(device)).float().mean().item()\n",
        "                )\n",
        "        val_loss /= len(val_loader)\n",
        "        val_accuracy /= len(val_loader)\n",
        "        hdim_accs[h, epoch] = val_accuracy\n",
        "\n",
        "        # Update learning rate\n",
        "        scheduler.step()\n",
        "\n",
        "        print(\n",
        "            f\"Epoch {epoch + 1}, Val Loss: {val_loss}, Val Accuracy: {val_accuracy}\"\n",
        "        )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "9hvytZJwUcri",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "58fb27b8-73c3-484b-f7c2-b43758c6ebcc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.97it/s, accuracy=0.906, loss=0.253, lr=0.0004]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Val Loss: 0.2376669420724272, Val Accuracy: 0.9300358280254777\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.29it/s, accuracy=0.969, loss=0.0905, lr=0.00036]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2, Val Loss: 0.140094636281015, Val Accuracy: 0.9607882165605095\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.45it/s, accuracy=0.938, loss=0.227, lr=0.000324]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3, Val Loss: 0.138253404503795, Val Accuracy: 0.9580015923566879\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.35it/s, accuracy=0.969, loss=0.0832, lr=0.000292]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4, Val Loss: 0.09247697657270795, Val Accuracy: 0.9719347133757962\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.47it/s, accuracy=1, loss=0.045, lr=0.000262]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5, Val Loss: 0.09638880232829529, Val Accuracy: 0.9707404458598726\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.00it/s, accuracy=1, loss=0.0321, lr=0.000236]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6, Val Loss: 0.08842214876504105, Val Accuracy: 0.9722332802547771\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.12it/s, accuracy=0.969, loss=0.105, lr=0.000213]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7, Val Loss: 0.07661282350042253, Val Accuracy: 0.9764132165605095\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.68it/s, accuracy=1, loss=0.00909, lr=0.000191]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8, Val Loss: 0.07539925703209392, Val Accuracy: 0.9772093949044586\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 51.02it/s, accuracy=1, loss=0.00328, lr=0.000172]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 9, Val Loss: 0.07426631024884428, Val Accuracy: 0.9759156050955414\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 51.21it/s, accuracy=1, loss=0.0291, lr=0.000155]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10, Val Loss: 0.06959543417276115, Val Accuracy: 0.9793988853503185\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.09it/s, accuracy=1, loss=0.00996, lr=0.000139]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11, Val Loss: 0.06735008772481886, Val Accuracy: 0.9789012738853503\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.82it/s, accuracy=0.969, loss=0.0725, lr=0.000126]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 12, Val Loss: 0.06572348916877928, Val Accuracy: 0.9801950636942676\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.93it/s, accuracy=1, loss=0.0067, lr=0.000113]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 13, Val Loss: 0.059839082193326586, Val Accuracy: 0.9802945859872612\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.79it/s, accuracy=0.969, loss=0.061, lr=0.000102]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 14, Val Loss: 0.058253191914231436, Val Accuracy: 0.9813893312101911\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.76it/s, accuracy=1, loss=0.00746, lr=9.15e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 15, Val Loss: 0.057604046388504034, Val Accuracy: 0.9815883757961783\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.52it/s, accuracy=1, loss=0.00717, lr=8.24e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 16, Val Loss: 0.05267927911285323, Val Accuracy: 0.9833797770700637\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.51it/s, accuracy=1, loss=0.00541, lr=7.41e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 17, Val Loss: 0.05175493115182539, Val Accuracy: 0.9830812101910829\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.68it/s, accuracy=1, loss=0.00914, lr=6.67e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 18, Val Loss: 0.05175672025993692, Val Accuracy: 0.9830812101910829\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.29it/s, accuracy=1, loss=0.0114, lr=6e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 19, Val Loss: 0.05490316013333975, Val Accuracy: 0.9814888535031847\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 51.05it/s, accuracy=1, loss=0.009, lr=5.4e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 20, Val Loss: 0.05216238197739307, Val Accuracy: 0.9837778662420382\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.40it/s, accuracy=1, loss=0.00521, lr=4.86e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 21, Val Loss: 0.0508287643194777, Val Accuracy: 0.9834792993630573\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.92it/s, accuracy=1, loss=0.006, lr=4.38e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 22, Val Loss: 0.051809068565860476, Val Accuracy: 0.9838773885350318\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 49.27it/s, accuracy=1, loss=0.00932, lr=3.94e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 23, Val Loss: 0.04885052733870733, Val Accuracy: 0.9847730891719745\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.28it/s, accuracy=1, loss=0.00578, lr=3.55e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 24, Val Loss: 0.04849254644578204, Val Accuracy: 0.9847730891719745\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.77it/s, accuracy=1, loss=0.00996, lr=3.19e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 25, Val Loss: 0.04771078925901968, Val Accuracy: 0.9848726114649682\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.04it/s, accuracy=1, loss=0.00655, lr=2.87e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 26, Val Loss: 0.0496796149146709, Val Accuracy: 0.9845740445859873\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.32it/s, accuracy=1, loss=0.00123, lr=2.58e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 27, Val Loss: 0.048426733000495374, Val Accuracy: 0.984375\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.70it/s, accuracy=1, loss=0.0106, lr=2.33e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 28, Val Loss: 0.048197887821017714, Val Accuracy: 0.9850716560509554\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 50.06it/s, accuracy=1, loss=0.00241, lr=2.09e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 29, Val Loss: 0.04876376052904625, Val Accuracy: 0.9837778662420382\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:18<00:00, 49.82it/s, accuracy=1, loss=0.00515, lr=1.88e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 30, Val Loss: 0.048022282230211745, Val Accuracy: 0.984375\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.29it/s, accuracy=0.875, loss=0.318, lr=0.0004]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Val Loss: 0.16874621504217766, Val Accuracy: 0.9494426751592356\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.79it/s, accuracy=0.906, loss=0.339, lr=0.00036]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2, Val Loss: 0.1418894723481884, Val Accuracy: 0.9564092356687898\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.62it/s, accuracy=1, loss=0.038, lr=0.000324]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3, Val Loss: 0.15270842864789116, Val Accuracy: 0.9494426751592356\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 47.57it/s, accuracy=0.938, loss=0.0793, lr=0.000292]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4, Val Loss: 0.10500664293178755, Val Accuracy: 0.9683519108280255\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.10it/s, accuracy=0.969, loss=0.0765, lr=0.000262]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5, Val Loss: 0.12028370097955796, Val Accuracy: 0.9613853503184714\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.56it/s, accuracy=0.969, loss=0.214, lr=0.000236]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6, Val Loss: 0.08218688925447916, Val Accuracy: 0.9754179936305732\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.48it/s, accuracy=1, loss=0.0178, lr=0.000213]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7, Val Loss: 0.0892090735320079, Val Accuracy: 0.9717356687898089\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.58it/s, accuracy=0.969, loss=0.167, lr=0.000191]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8, Val Loss: 0.09553493971708023, Val Accuracy: 0.96875\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.37it/s, accuracy=1, loss=0.0488, lr=0.000172]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 9, Val Loss: 0.07042367309558051, Val Accuracy: 0.9772093949044586\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.09it/s, accuracy=1, loss=0.0194, lr=0.000155]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10, Val Loss: 0.08202768663433084, Val Accuracy: 0.9750199044585988\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.93it/s, accuracy=1, loss=0.0017, lr=0.000139]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11, Val Loss: 0.07574345822265123, Val Accuracy: 0.9767117834394905\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.77it/s, accuracy=1, loss=0.0182, lr=0.000126]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 12, Val Loss: 0.06417825112488583, Val Accuracy: 0.978702229299363\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.65it/s, accuracy=0.969, loss=0.0797, lr=0.000113]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 13, Val Loss: 0.07265686052632636, Val Accuracy: 0.9783041401273885\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.87it/s, accuracy=1, loss=0.0157, lr=0.000102]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 14, Val Loss: 0.06534795328160187, Val Accuracy: 0.9780055732484076\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 46.99it/s, accuracy=1, loss=0.0238, lr=9.15e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 15, Val Loss: 0.05903924124029533, Val Accuracy: 0.9811902866242038\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.73it/s, accuracy=1, loss=0.00399, lr=8.24e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 16, Val Loss: 0.06569081576000514, Val Accuracy: 0.9791998407643312\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:19<00:00, 48.00it/s, accuracy=0.969, loss=0.0443, lr=7.41e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 17, Val Loss: 0.05728242317131931, Val Accuracy: 0.9821855095541401\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.88it/s, accuracy=1, loss=0.026, lr=6.67e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 18, Val Loss: 0.056968471647468606, Val Accuracy: 0.9827826433121019\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.12it/s, accuracy=1, loss=0.00459, lr=6e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 19, Val Loss: 0.06211456729275817, Val Accuracy: 0.9818869426751592\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.63it/s, accuracy=1, loss=0.00262, lr=5.4e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 20, Val Loss: 0.06266652140516127, Val Accuracy: 0.9819864649681529\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.45it/s, accuracy=1, loss=0.00292, lr=4.86e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 21, Val Loss: 0.0576301462793349, Val Accuracy: 0.9830812101910829\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.52it/s, accuracy=1, loss=0.0212, lr=4.38e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 22, Val Loss: 0.05814899391597389, Val Accuracy: 0.9829816878980892\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.98it/s, accuracy=1, loss=0.000498, lr=3.94e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 23, Val Loss: 0.058646362034593204, Val Accuracy: 0.9821855095541401\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.65it/s, accuracy=1, loss=0.00616, lr=3.55e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 24, Val Loss: 0.057089446184902366, Val Accuracy: 0.9834792993630573\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.66it/s, accuracy=1, loss=0.00601, lr=3.19e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 25, Val Loss: 0.055459304453860145, Val Accuracy: 0.9834792993630573\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.46it/s, accuracy=1, loss=0.00134, lr=2.87e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 26, Val Loss: 0.0556573667426089, Val Accuracy: 0.9835788216560509\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.63it/s, accuracy=1, loss=0.0077, lr=2.58e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 27, Val Loss: 0.05626426280870908, Val Accuracy: 0.9836783439490446\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.56it/s, accuracy=1, loss=0.00783, lr=2.33e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 28, Val Loss: 0.05463365130476835, Val Accuracy: 0.9841759554140127\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.35it/s, accuracy=1, loss=0.00998, lr=2.09e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 29, Val Loss: 0.05763717692416843, Val Accuracy: 0.9826831210191083\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.10it/s, accuracy=1, loss=0.0048, lr=1.88e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 30, Val Loss: 0.05675368953510028, Val Accuracy: 0.9842754777070064\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.72it/s, accuracy=0.969, loss=0.144, lr=0.0004]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Val Loss: 0.3356318795092546, Val Accuracy: 0.8956011146496815\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.79it/s, accuracy=0.812, loss=0.284, lr=0.00036]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2, Val Loss: 0.23121833078753037, Val Accuracy: 0.9261544585987261\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.03it/s, accuracy=0.906, loss=0.239, lr=0.000324]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3, Val Loss: 0.19222391059230656, Val Accuracy: 0.9408837579617835\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.33it/s, accuracy=0.938, loss=0.136, lr=0.000292]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4, Val Loss: 0.12049418827222222, Val Accuracy: 0.9620820063694268\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.77it/s, accuracy=0.938, loss=0.306, lr=0.000262]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5, Val Loss: 0.1052536706931023, Val Accuracy: 0.9667595541401274\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.76it/s, accuracy=0.969, loss=0.0948, lr=0.000236]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6, Val Loss: 0.10677376933615346, Val Accuracy: 0.9675557324840764\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.57it/s, accuracy=0.969, loss=0.0569, lr=0.000213]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7, Val Loss: 0.08995354162628472, Val Accuracy: 0.9729299363057324\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.13it/s, accuracy=0.969, loss=0.0691, lr=0.000191]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8, Val Loss: 0.09237965422120407, Val Accuracy: 0.9710390127388535\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.39it/s, accuracy=1, loss=0.0202, lr=0.000172]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 9, Val Loss: 0.08640209651922148, Val Accuracy: 0.9732285031847133\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.68it/s, accuracy=0.969, loss=0.151, lr=0.000155]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10, Val Loss: 0.08144104871031335, Val Accuracy: 0.9732285031847133\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.17it/s, accuracy=0.938, loss=0.126, lr=0.000139]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11, Val Loss: 0.07970516378346201, Val Accuracy: 0.9748208598726115\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.53it/s, accuracy=0.969, loss=0.056, lr=0.000126]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 12, Val Loss: 0.08083940048036195, Val Accuracy: 0.9753184713375797\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.70it/s, accuracy=1, loss=0.0154, lr=0.000113]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 13, Val Loss: 0.08878041699989907, Val Accuracy: 0.9723328025477707\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.88it/s, accuracy=1, loss=0.00277, lr=0.000102]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 14, Val Loss: 0.07426302058718265, Val Accuracy: 0.9772093949044586\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.01it/s, accuracy=0.969, loss=0.115, lr=9.15e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 15, Val Loss: 0.08196259982901355, Val Accuracy: 0.9753184713375797\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.43it/s, accuracy=1, loss=0.00776, lr=8.24e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 16, Val Loss: 0.0851796131140247, Val Accuracy: 0.9753184713375797\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.87it/s, accuracy=0.969, loss=0.115, lr=7.41e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 17, Val Loss: 0.07180901071518121, Val Accuracy: 0.9781050955414012\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.17it/s, accuracy=1, loss=0.00538, lr=6.67e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 18, Val Loss: 0.07331750159766184, Val Accuracy: 0.9774084394904459\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.69it/s, accuracy=0.969, loss=0.0348, lr=6e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 19, Val Loss: 0.06899227759939697, Val Accuracy: 0.9798964968152867\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.63it/s, accuracy=1, loss=0.00273, lr=5.4e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 20, Val Loss: 0.07296926602263339, Val Accuracy: 0.9784036624203821\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.45it/s, accuracy=1, loss=0.00533, lr=4.86e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 21, Val Loss: 0.07994764528120868, Val Accuracy: 0.9754179936305732\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.18it/s, accuracy=1, loss=0.0218, lr=4.38e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 22, Val Loss: 0.06818674133051657, Val Accuracy: 0.980593152866242\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.34it/s, accuracy=1, loss=0.00817, lr=3.94e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 23, Val Loss: 0.07200525059621665, Val Accuracy: 0.978702229299363\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.60it/s, accuracy=1, loss=0.00388, lr=3.55e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 24, Val Loss: 0.0689845538088393, Val Accuracy: 0.9793988853503185\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.66it/s, accuracy=1, loss=0.00837, lr=3.19e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 25, Val Loss: 0.06968328982859866, Val Accuracy: 0.9796974522292994\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 46.03it/s, accuracy=1, loss=0.00144, lr=2.87e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 26, Val Loss: 0.07009343759307937, Val Accuracy: 0.9795979299363057\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.91it/s, accuracy=1, loss=0.00323, lr=2.58e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 27, Val Loss: 0.06906905766914095, Val Accuracy: 0.9803941082802548\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.40it/s, accuracy=1, loss=0.000939, lr=2.33e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 28, Val Loss: 0.0671017736168341, Val Accuracy: 0.9801950636942676\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 45.24it/s, accuracy=1, loss=0.0124, lr=2.09e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 29, Val Loss: 0.06882438757036502, Val Accuracy: 0.980593152866242\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.13it/s, accuracy=1, loss=0.00125, lr=1.88e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 30, Val Loss: 0.06692342810293568, Val Accuracy: 0.9814888535031847\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.94it/s, accuracy=0.906, loss=0.183, lr=0.0004]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Val Loss: 0.4847522712057563, Val Accuracy: 0.852906050955414\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.54it/s, accuracy=0.938, loss=0.299, lr=0.00036]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2, Val Loss: 0.24849414057839828, Val Accuracy: 0.9225716560509554\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.12it/s, accuracy=0.969, loss=0.0984, lr=0.000324]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3, Val Loss: 0.4554738363215498, Val Accuracy: 0.8610668789808917\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.83it/s, accuracy=0.969, loss=0.0852, lr=0.000292]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4, Val Loss: 0.20065429713577032, Val Accuracy: 0.9408837579617835\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.04it/s, accuracy=0.938, loss=0.212, lr=0.000262]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5, Val Loss: 0.2285947362168627, Val Accuracy: 0.9311305732484076\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.49it/s, accuracy=1, loss=0.0382, lr=0.000236]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6, Val Loss: 0.17950810610678544, Val Accuracy: 0.9406847133757962\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.19it/s, accuracy=0.906, loss=0.169, lr=0.000213]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7, Val Loss: 0.13323346321963392, Val Accuracy: 0.9594944267515924\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.99it/s, accuracy=0.969, loss=0.0779, lr=0.000191]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8, Val Loss: 0.17525738049565465, Val Accuracy: 0.9456608280254777\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.84it/s, accuracy=0.906, loss=0.159, lr=0.000172]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 9, Val Loss: 0.09800270902901995, Val Accuracy: 0.9692476114649682\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.76it/s, accuracy=1, loss=0.0029, lr=0.000155]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10, Val Loss: 0.10373531388723926, Val Accuracy: 0.9685509554140127\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.29it/s, accuracy=0.906, loss=0.122, lr=0.000139]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11, Val Loss: 0.1018098443905989, Val Accuracy: 0.9666600318471338\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.02it/s, accuracy=1, loss=0.0121, lr=0.000126]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 12, Val Loss: 0.10988549919499405, Val Accuracy: 0.9665605095541401\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.87it/s, accuracy=1, loss=0.0303, lr=0.000113]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 13, Val Loss: 0.09029746336186198, Val Accuracy: 0.9737261146496815\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.59it/s, accuracy=0.969, loss=0.0964, lr=0.000102]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 14, Val Loss: 0.08981731165099258, Val Accuracy: 0.9729299363057324\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:20<00:00, 44.99it/s, accuracy=1, loss=0.0115, lr=9.15e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 15, Val Loss: 0.08345353128043727, Val Accuracy: 0.9747213375796179\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 44.30it/s, accuracy=1, loss=0.042, lr=8.24e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 16, Val Loss: 0.08974831792838119, Val Accuracy: 0.9750199044585988\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.80it/s, accuracy=1, loss=0.000571, lr=7.41e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 17, Val Loss: 0.07923947612002008, Val Accuracy: 0.9778065286624203\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.30it/s, accuracy=1, loss=0.0106, lr=6.67e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 18, Val Loss: 0.07833796009220402, Val Accuracy: 0.9766122611464968\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.44it/s, accuracy=1, loss=0.00813, lr=6e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 19, Val Loss: 0.08009260598622756, Val Accuracy: 0.9768113057324841\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.59it/s, accuracy=0.969, loss=0.0404, lr=5.4e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 20, Val Loss: 0.08091839709264304, Val Accuracy: 0.977109872611465\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 42.37it/s, accuracy=1, loss=0.00292, lr=4.86e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 21, Val Loss: 0.07569800208824698, Val Accuracy: 0.9785031847133758\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.61it/s, accuracy=1, loss=0.00632, lr=4.38e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 22, Val Loss: 0.09504984865988998, Val Accuracy: 0.9722332802547771\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:23<00:00, 40.71it/s, accuracy=1, loss=0.0138, lr=3.94e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 23, Val Loss: 0.07780386942520717, Val Accuracy: 0.978702229299363\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.32it/s, accuracy=1, loss=0.00517, lr=3.55e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 24, Val Loss: 0.0843266662451723, Val Accuracy: 0.9766122611464968\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 41.02it/s, accuracy=1, loss=0.0182, lr=3.19e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 25, Val Loss: 0.07802320175975114, Val Accuracy: 0.9781050955414012\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.81it/s, accuracy=1, loss=0.00639, lr=2.87e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 26, Val Loss: 0.07563198950432197, Val Accuracy: 0.9784036624203821\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.51it/s, accuracy=1, loss=0.00692, lr=2.58e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 27, Val Loss: 0.07672752728864855, Val Accuracy: 0.9791003184713376\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:22<00:00, 40.83it/s, accuracy=1, loss=0.00515, lr=2.33e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 28, Val Loss: 0.08093475776471805, Val Accuracy: 0.9778065286624203\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 42.98it/s, accuracy=0.969, loss=0.0469, lr=2.09e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 29, Val Loss: 0.07385839822394712, Val Accuracy: 0.9802945859872612\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 938/938 [00:21<00:00, 43.97it/s, accuracy=1, loss=0.00157, lr=1.88e-5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 30, Val Loss: 0.07221318118348455, Val Accuracy: 0.9795979299363057\n"
          ]
        }
      ],
      "source": [
        "epochs = 30\n",
        "n_layers = [1, 2, 3, 4]\n",
        "layer_accs = np.empty((4, epochs))\n",
        "for h, n_layer in enumerate(n_layers):\n",
        "    torch.manual_seed(42)\n",
        "    # Define model\n",
        "    model = SineKAN(layers_hidden=[28 * 28] + [128]*n_layer + [10], grid_size=8)\n",
        "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "    model.to(device)\n",
        "    # Define optimizer\n",
        "    optimizer = optim.AdamW(model.parameters(), lr=4e-4, weight_decay=.5)\n",
        "    # Define learning rate scheduler\n",
        "    scheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)\n",
        "\n",
        "    # Define loss\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "    for epoch in range(epochs):\n",
        "        # Train\n",
        "        model.train()\n",
        "        with tqdm(train_loader) as pbar:\n",
        "            for i, (images, labels) in enumerate(pbar):\n",
        "                images = images.view(-1, 28 * 28).to(device)\n",
        "                optimizer.zero_grad()\n",
        "                output = model(images)\n",
        "                loss = criterion(output, labels.to(device))\n",
        "                loss.backward()\n",
        "                optimizer.step()\n",
        "                accuracy = (output.argmax(dim=1) == labels.to(device)).float().mean()\n",
        "                pbar.set_postfix(loss=loss.item(), accuracy=accuracy.item(), lr=optimizer.param_groups[0]['lr'])\n",
        "\n",
        "        # Validation\n",
        "        model.eval()\n",
        "        val_loss = 0\n",
        "        val_accuracy = 0\n",
        "        with torch.no_grad():\n",
        "            for images, labels in val_loader:\n",
        "                images = images.view(-1, 28 * 28).to(device)\n",
        "                output = model(images)\n",
        "                val_loss += criterion(output, labels.to(device)).item()\n",
        "                val_accuracy += (\n",
        "                    (output.argmax(dim=1) == labels.to(device)).float().mean().item()\n",
        "                )\n",
        "        val_loss /= len(val_loader)\n",
        "        val_accuracy /= len(val_loader)\n",
        "        layer_accs[h, epoch] = val_accuracy\n",
        "\n",
        "        # Update learning rate\n",
        "        scheduler.step()\n",
        "\n",
        "        print(\n",
        "            f\"Epoch {epoch + 1}, Val Loss: {val_loss}, Val Accuracy: {val_accuracy}\"\n",
        "        )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lJho8zXO5em-"
      },
      "outputs": [],
      "source": [
        "from torch.profiler import profile, record_function, ProfilerActivity\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AlCgt3EHOtbB"
      },
      "outputs": [],
      "source": [
        "def profile_model(model, inputs, num_runs=100):\n",
        "    total_time = 0\n",
        "    for _ in range(num_runs):\n",
        "        with profile(activities=[ProfilerActivity.CUDA], record_shapes=True) as prof:\n",
        "            model(inputs)\n",
        "\n",
        "        # Extract total CUDA time spent on 'model_inference'\n",
        "        for event in prof.key_averages():\n",
        "            total_time += event.cuda_time_total\n",
        "\n",
        "    average_time = total_time / num_runs\n",
        "    print(f\"Average inference time: {average_time} microseconds\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RpPdVaMb5h3B"
      },
      "outputs": [],
      "source": [
        "inputs = torch.randn((1, 784)).to(device)\n",
        "hdims = [16, 32, 64, 128, 256, 512]\n",
        "for h in hdims:\n",
        "    model = SineKAN(layers_hidden=[28 * 28, h, 10], grid_size=8).to(device)\n",
        "    print('Hidden Dims: %d' % h)\n",
        "    profile_model(model, inputs, 1000)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ocQnoJHcSUYh"
      },
      "outputs": [],
      "source": [
        "hdims = [16, 32, 64, 128, 256, 512]\n",
        "for h in hdims:\n",
        "    model = SineKAN(layers_hidden=[28 * 28, 64, 10], grid_size=8).to(device)\n",
        "    inputs = torch.randn((h, 784)).to(device)\n",
        "    print('Input Batch Size: %d' % h)\n",
        "    profile_model(model, inputs, 1000)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HZ9FGPPpg3ll"
      },
      "outputs": [],
      "source": [
        "nlayers = [1, 2, 3, 4]\n",
        "for n in nlayers:\n",
        "    model = SineKAN(layers_hidden=[28 * 28] + [64]*n +[10], grid_size=8).to(device)\n",
        "    inputs = torch.randn((1, 784)).to(device)\n",
        "    print('Number of Hidden Layers: %d' % n)\n",
        "    profile_model(model, inputs, 1000)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ExHCYJ0CYnAm"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": [],
      "mount_file_id": "1xFXxUHJOjNqzHkETUzfaAeFRFQQgodqZ",
      "authorship_tag": "ABX9TyPBe5+bYjpokTsNiQPdWsWv",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}