Pytorch构建神经网络(三).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pytorch构建神经网络(三)(29-33节)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.1&4.2 使用tensorboard可视化CNN训练指标"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* pytorch1.1.0以上的版本已经自动增加了tensorboard\n",
    "* 在终端输入“tensorboard --version”可查看tensorboard的版本\n",
    "* 在终端输入“tensorboard --logdir=runs”进入tensorboard(在写了tensorboard数据的路径下)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch.autograd.grad_mode.set_grad_enabled at 0x2ae0ef5b488>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "\n",
    "torch.set_printoptions(linewidth=120)\n",
    "torch.set_grad_enabled(True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.4.0\n",
      "0.5.0\n"
     ]
    }
   ],
   "source": [
    "print(torch.__version__)\n",
    "print(torchvision.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_num_correct(preds,labels):\n",
    "    return preds.argmax(dim=1).eq(labels).sum().item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Network(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Network,self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channels=1, out_channels=6, kernel_size=5)\n",
    "        self.conv2 = nn.Conv2d(in_channels=6, out_channels=12, kernel_size=5)\n",
    "        \n",
    "        self.fc1 = nn.Linear(in_features=12*4*4, out_features=120)\n",
    "        self.fc2 = nn.Linear(in_features=120, out_features=60)\n",
    "        self.out = nn.Linear(in_features=60, out_features=10)\n",
    "    \n",
    "    def forward(self, t):\n",
    "        t = t \n",
    "        t = F.relu(self.conv1(t))\n",
    "        t = F.max_pool2d(t, kernel_size=2, stride=2)\n",
    "        \n",
    "        t = F.relu(self.conv2(t))\n",
    "        t = F.max_pool2d(t, kernel_size=2, stride=2)\n",
    "        \n",
    "        t = t.reshape(-1, 12*4*4)  # t.flatten(start_dim=1)\n",
    "        t = F.relu(self.fc1(t))\n",
    "        \n",
    "        t = F.relu(self.fc2(t))\n",
    "        \n",
    "        t = self.out(t)\n",
    "        return t\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set = torchvision.datasets.FashionMNIST(\n",
    "    root = './data/FashionMNIST',\n",
    "    train = True,\n",
    "    download = True,\n",
    "    transform = transforms.Compose([\n",
    "        transforms.ToTensor()\n",
    "    ])\n",
    ")\n",
    "#train_loader = torch.utils.data.DataLoader(train_set, batch_size=100, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Starting out with TensorBoard (Network Graph and Images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#batch_size = 100\n",
    "#lr =0.01\n",
    "# 对不同的batchsize，lr的训练情况进行比较\n",
    "# 方法1：但此方法需要多层for循环\n",
    "batch_size_list = [100, 1000, 10000]\n",
    "lr_list = [.01, .001, .0001, .00001]\n",
    "for batch_size in batch_size_list:\n",
    "    for lr in lr_list:\n",
    "        network = Network()\n",
    "        train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True)\n",
    "        images, labels = next(iter(train_loader))\n",
    "        grid = torchvision.utils.make_grid(images)   # 创建能在tensorboard中查看的图像网格\n",
    "\n",
    "        comment = f'batch_size={batch_size} lr ={lr}'\n",
    "        tb = SummaryWriter(comment=comment)   # 在Summary Writer添加该注释，可帮助我们在tensorboard中唯一地识别该表示\n",
    "        tb.add_image('images', grid)  # 将一批图像放在grid中进行显示\n",
    "        tb.add_graph(network, images)   # 在tensorboard中看见网络结构的可视化图\n",
    "        optimizer = optim.Adam(network.parameters(), lr=lr)\n",
    "\n",
    "        for epoch in range(5):\n",
    "    \n",
    "            total_loss = 0\n",
    "            total_correct = 0\n",
    "    \n",
    "            for batch in train_loader:    # Get Batch\n",
    "                images, labels = batch\n",
    "        \n",
    "                preds = network(images) # Pass Batch\n",
    "                loss = F.cross_entropy(preds, labels)  # Calculate loss\n",
    "        \n",
    "                optimizer.zero_grad()    # 梯度清零，否则会累加\n",
    "                loss.backward()     # Calculate Gradients\n",
    "                optimizer.step()    # Update Weights\n",
    "        \n",
    "                #total_loss += loss.item()\n",
    "                total_loss += loss.item()*batch_size # 在对不同批次下的训练进行比较时，这样做可使结果更具有可比性\n",
    "                total_correct += get_num_correct(preds, labels)\n",
    "        \n",
    "                tb.add_scalar(\"Loss\", total_loss, epoch)\n",
    "                tb.add_scalar(\"Number Correct\", total_correct, epoch)\n",
    "                tb.add_scalar(\"Accuracy\", total_correct/len(train_set), epoch)\n",
    "                '''\n",
    "                这种表达方式只能看单个层的偏置，权重，及其梯度的变化趋势，无法看到全部的\n",
    "                tb.add_histogram('conv1.bias', network.conv1.bias, epoch)\n",
    "                tb.add_histogram('conv1.weight', network.conv1.weight, epoch)\n",
    "                tb.add_histogram('conv1.weight.grad', network.conv1.weight.grad, epoch)\n",
    "                '''\n",
    "                for name, weight in network.named_parameters():\n",
    "                    tb.add_histogram(name, weight, epoch)\n",
    "                    tb.add_histogram(f'{name}.grad', weight.grad, epoch)\n",
    "        print(\"epoch:\", epoch, \"total_correct:\", total_correct, \"loss\", total_loss)\n",
    "\n",
    "tb.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 对多层的偏置，权重及其梯度进行访问的原理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'network' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-f52ef863b291>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mfor\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mweight\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mnetwork\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnamed_parameters\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'network' is not defined"
     ]
    }
   ],
   "source": [
    "for name,weight in network.named_parameters():\n",
    "    print(name, weight.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for name,weight in network.named_parameters():\n",
    "    print(f'{name}.grad', weight.grad.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 更简单的方法对要更改的参数进行访问"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = dict(\n",
    "    lr = [.01, .001],\n",
    "    batc_size = [10, 100, 1000],\n",
    "    shuffle = [True, False]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[0.01, 0.001], [10, 100, 1000], [True, False]]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_values = [v for v in parameters.values()]\n",
    "param_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.01 10 True\n",
      "0.01 10 False\n",
      "0.01 100 True\n",
      "0.01 100 False\n",
      "0.01 1000 True\n",
      "0.01 1000 False\n",
      "0.001 10 True\n",
      "0.001 10 False\n",
      "0.001 100 True\n",
      "0.001 100 False\n",
      "0.001 1000 True\n",
      "0.001 1000 False\n"
     ]
    }
   ],
   "source": [
    "for lr, batch_size, shuffle in product(*param_values):\n",
    "    print(lr, batch_size, shuffle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.01, 0.001], [10, 100], [True, False]]\n"
     ]
    }
   ],
   "source": [
    "#batch_size = 100\n",
    "#lr =0.01\n",
    "# 对不同的batchsize，lr的训练情况进行比较\n",
    "# 方法2：只需一层循环\n",
    "from itertools import product\n",
    "parameters = dict(\n",
    "    lr = [.01, .001],\n",
    "    batch_size = [10, 100],\n",
    "    shuffle = [True, False]\n",
    ")\n",
    "param_values = [v for v in parameters.values()]\n",
    "print(param_values)\n",
    "for lr, batch_size, shuffle in product(*param_values):\n",
    "    network = Network()\n",
    "    train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True)\n",
    "    images, labels = next(iter(train_loader))\n",
    "    grid = torchvision.utils.make_grid(images)   # 创建能在tensorboard中查看的图像网格\n",
    "\n",
    "    comment = f'batch_size={batch_size} lr ={lr} shuffle={shuffle}'\n",
    "    tb = SummaryWriter(comment=comment)   # 在Summary Writer添加该注释，可帮助我们在tensorboard中唯一地识别该表示\n",
    "    tb.add_image('images', grid)  # 将一批图像放在grid中进行显示\n",
    "    tb.add_graph(network, images)   # 在tensorboard中看见网络结构的可视化图\n",
    "    optimizer = optim.Adam(network.parameters(), lr=lr)\n",
    "\n",
    "    for epoch in range(5):\n",
    "    \n",
    "        total_loss = 0\n",
    "        total_correct = 0\n",
    "    \n",
    "        for batch in train_loader:    # Get Batch\n",
    "            images, labels = batch\n",
    "        \n",
    "            preds = network(images) # Pass Batch\n",
    "            loss = F.cross_entropy(preds, labels)  # Calculate loss\n",
    "        \n",
    "            optimizer.zero_grad()    # 梯度清零，否则会累加\n",
    "            loss.backward()     # Calculate Gradients\n",
    "            optimizer.step()    # Update Weights\n",
    "        \n",
    "            #total_loss += loss.item()\n",
    "            total_loss += loss.item()*batch_size # 在对不同批次下的训练进行比较时，这样做可使结果更具有可比性\n",
    "            total_correct += get_num_correct(preds, labels)\n",
    "        \n",
    "            tb.add_scalar(\"Loss\", total_loss, epoch)\n",
    "            tb.add_scalar(\"Number Correct\", total_correct, epoch)\n",
    "            tb.add_scalar(\"Accuracy\", total_correct/len(train_set), epoch)\n",
    "            '''\n",
    "                这种表达方式只能看单个层的偏置，权重，及其梯度的变化趋势，无法看到全部的\n",
    "                tb.add_histogram('conv1.bias', network.conv1.bias, epoch)\n",
    "                tb.add_histogram('conv1.weight', network.conv1.weight, epoch)\n",
    "                tb.add_histogram('conv1.weight.grad', network.conv1.weight.grad, epoch)\n",
    "            '''\n",
    "            for name, weight in network.named_parameters():\n",
    "                tb.add_histogram(name, weight, epoch)\n",
    "                tb.add_histogram(f'{name}.grad', weight.grad, epoch)\n",
    "    print(\"epoch:\", epoch, \"total_correct:\", total_correct, \"loss\", total_loss)\n",
    "\n",
    "tb.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.3 RunBuilder类的编写\n",
    "* 该类的编写允许我们使用不同的参数值生成多个运行"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import OrderedDict\n",
    "from collections import namedtuple\n",
    "from itertools import product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RunBuilder():\n",
    "    @staticmethod\n",
    "    def get_runs(params):\n",
    "        Run = namedtuple('Run', params.keys())\n",
    "        runs = []\n",
    "        for v in product(*params.values()):\n",
    "            runs.append(Run(*v))\n",
    "        return runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = OrderedDict(\n",
    "    lr = [.01, .001],\n",
    "    batch_size = [1000, 10000]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Run(lr=0.01, batch_size=1000),\n",
       " Run(lr=0.01, batch_size=10000),\n",
       " Run(lr=0.001, batch_size=1000),\n",
       " Run(lr=0.001, batch_size=10000)]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "runs = RunBuilder.get_runs(params)\n",
    "runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run(lr=0.01, batch_size=1000) 0.01 1000\n",
      "Run(lr=0.01, batch_size=10000) 0.01 10000\n",
      "Run(lr=0.001, batch_size=1000) 0.001 1000\n",
      "Run(lr=0.001, batch_size=10000) 0.001 10000\n"
     ]
    }
   ],
   "source": [
    "for run in runs:\n",
    "    print(run, run.lr, run.batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-Run(lr=0.01, batch_size=1000)\n",
      "-Run(lr=0.01, batch_size=10000)\n",
      "-Run(lr=0.001, batch_size=1000)\n",
      "-Run(lr=0.001, batch_size=10000)\n"
     ]
    }
   ],
   "source": [
    "# 创建RunBuilder类以后，comment表示为：\n",
    "for run in RunBuilder.get_runs(params):\n",
    "    comment = f'-{run}'\n",
    "    print(comment)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4.4 如何试验大量的超参数\n",
    "* 构建RunManager类可实现对大量超参数的试验"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RunManager():\n",
    "    def __init__(self):\n",
    "        self.epoch_count = 0\n",
    "        self.epoch_loss = 0\n",
    "        self.epoch_num_correct = 0\n",
    "        self.epoch_start_time = None\n",
    "        \n",
    "        self.run_params = None\n",
    "        self.run_count = 0\n",
    "        self.run_data = []\n",
    "        self.run_start_time = None\n",
    "        \n",
    "        self.network = None\n",
    "        self.loader = None\n",
    "        self.tb = None\n",
    "        \n",
    "    def begin_run(self, run, network, loader):\n",
    "        self.run_start_time = time.time()\n",
    "        \n",
    "        self.run_params = run\n",
    "        self.run_count += 1\n",
    "        \n",
    "        self.network = network\n",
    "        self.loader = loader\n",
    "        self.tb = SummaryWriter(comment=f'-{run}')\n",
    "        \n",
    "        images. labels = next(iter(self.loader))\n",
    "        grid = torchvision.utils.make_grid(images)\n",
    "        \n",
    "        self.tb.add_image('images', grid)\n",
    "        self.tb.add_graph(self.network, images)\n",
    "        \n",
    "    def end_run(self):\n",
    "        self.tb.close()\n",
    "        self.epoch_count = 0\n",
    "        \n",
    "    def begin_epoch(self):\n",
    "        self.epoch_start_time = time.time()\n",
    "        \n",
    "        self.epoch_count += 1\n",
    "        self.epoch_loss = 0\n",
    "        self.epoch_num_correct = 0\n",
    "    \n",
    "    def end_epoch(self):\n",
    "        epoch_duration = time.time() - self.epoch_start_time\n",
    "        run_duration = time.time() - self.run_start_time\n",
    "        \n",
    "        loss = self.epoch_loss/len(self.loader.dataset)\n",
    "        accuracy = self.epoch_num_correct/len(self.loader.dataset)\n",
    "        \n",
    "        self.tb.add_scalar('Loss', loss, self.epoch_count)\n",
    "        self.tb.add_scalar('Accuracy', accuracy, self.epoch_count)\n",
    "        \n",
    "        for name, param in self.network.named_parameters():\n",
    "            self.tb.add_histogram(name, param, self.epoch_count)\n",
    "            self.tb.add_histogram(f'{name}.grad', param.grad, self.epoch_count)\n",
    "            \n",
    "            results = OrderedDict()\n",
    "            results[\"run\"] = self.run_count\n",
    "            results[\"epoch\"] = self.epoch_count\n",
    "            results[\"loss\"] = loss\n",
    "            results[\"accuracy\"] = accuracy\n",
    "            results[\"epoch duration\"] = epoch_duration\n",
    "            results[\"run duration\"] = run_duration\n",
    "            for k,v in slef.run_params._asdict().items(): results[k] = v\n",
    "            self.run_data.append(results)\n",
    "            df = pd.DataFrame.from_dict(self.run_data, orient='columns')\n",
    "            \n",
    "            clear_output(wait=True)\n",
    "            display(df)\n",
    "    \n",
    "    def track_loss(self, loss):\n",
    "        self.epoch_loss += loss.item()*self.loader.batch_size\n",
    "    \n",
    "    def track_num_correct(self, preds, labels):\n",
    "        self.epoch_num_correct += self._get_num_correct(preds, labels)\n",
    "    \n",
    "    @torch.no_grad()\n",
    "    def _get_num_correct(self, preds, labels):\n",
    "        return preds.argmax(dim=1).eq(labels).sum().item()\n",
    "    \n",
    "    def save(self, fileName):\n",
    "        pd.DataFrame.from_dict(\n",
    "            self.run_data,\n",
    "            orient='columns').to_csv(f'{fileName}.csv')\n",
    "        with open(f'{fileName},json','w', encoding='utf-8') as f:\n",
    "            json.dump(self.run_data, f, ensure_ascii=False, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用RunManager和RunBuilder类可以使得程序更易扩展\n",
    "params = OrderedDict(\n",
    "    lr = [.01],\n",
    "    batch_size =[1000, 2000],\n",
    "    shuffle = [True, False]\n",
    ")\n",
    "m = RunManager()\n",
    "for run in RunBuilder.get_runs(params):\n",
    "    \n",
    "    network = Network()\n",
    "    loader = DataLoader(train_set, batch_size=run.batch_size, shuffle=run.shuffle)\n",
    "    optimizer = optim.Adam(network.parameters(), lr=run.lr)\n",
    "    \n",
    "    m.begin_run(run, network, loader)\n",
    "    for epoch in range(5):\n",
    "        m.begin_epoch()\n",
    "        for batch in loader:\n",
    "            images, labels = batch\n",
    "            preds = network(images)\n",
    "            loss = F.cross_entropy(preds, labels)\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            \n",
    "            m.track_loss(loss)\n",
    "            m.track_num_correct(preds, labels)\n",
    "            \n",
    "        m.end_epoch()\n",
    "    m.end_run()\n",
    "m.save('resuls')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4.5 使用DataLoader的多进程功能加速神经网络训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 使用data loader类的num_workers可选属性可加速神经网络的训练\n",
    "* num_workers属性告诉data loader实例有多少个单元处理器用于数据加载\n",
    "* num_workers值的选择的最好方式是进行试验"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用RunManager和RunBuilder类可以使得程序更易扩展\n",
    "params = OrderedDict(\n",
    "    lr = [.01],\n",
    "    batch_size =[1000, 2000],\n",
    "    shuffle = [True, False]，\n",
    "    num_workers = [0,1,2,4,8,16]\n",
    ")\n",
    "m = RunManager()\n",
    "for run in RunBuilder.get_runs(params):\n",
    "    \n",
    "    network = Network()\n",
    "    loader = DataLoader(train_set, batch_size=run.batch_size, shuffle=run.shuffle, num_workers=run.num_workers)\n",
    "    optimizer = optim.Adam(network.parameters(), lr=run.lr)\n",
    "    \n",
    "    m.begin_run(run, network, loader)\n",
    "    for epoch in range(5):\n",
    "        m.begin_epoch()\n",
    "        for batch in loader:\n",
    "            images, labels = batch\n",
    "            preds = network(images)\n",
    "            loss = F.cross_entropy(preds, labels)\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            \n",
    "            m.track_loss(loss)\n",
    "            m.track_num_correct(preds, labels)\n",
    "            \n",
    "            \n",
    "        m.end_epoch()\n",
    "    m.end_run()\n",
    "m.save('resuls')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}