Multilayer-Perceptron.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/Users/ZRC/miniconda3/envs/tryit/lib/python36.zip',\n",
       " '/Users/ZRC/miniconda3/envs/tryit/lib/python3.6',\n",
       " '/Users/ZRC/miniconda3/envs/tryit/lib/python3.6/lib-dynload',\n",
       " '',\n",
       " '/Users/ZRC/miniconda3/envs/tryit/lib/python3.6/site-packages',\n",
       " '/Users/ZRC/miniconda3/envs/tryit/lib/python3.6/site-packages/IPython/extensions',\n",
       " '/Users/ZRC/.ipython',\n",
       " '/Users/ZRC']"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.append(\"/Users/ZRC\")\n",
    "sys.path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data import RandomSampler\n",
    "\n",
    "\n",
    "from torchvision import datasets\n",
    "from torchvision import transforms\n",
    "\n",
    "from torchsummary import summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "from coke.visualization.image import show_batch\n",
    "from coke.models.customized_metrics import compute_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda: 0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperparameters\n",
    "batch_size = 64\n",
    "num_features = 28 * 28\n",
    "learning_rate = 0.1\n",
    "random_seed = 7\n",
    "num_epochs = 3\n",
    "DROP_OUT_PROB = 0.5\n",
    "\n",
    "\n",
    "# Architecture\n",
    "HIDDEN_NUMS_1 = 128\n",
    "HIDDEN_NUMS_2 = 256\n",
    "num_classes = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_transforms = {\n",
    "    'train': transforms.Compose([\n",
    "#         transforms.RandomHorizontalFlip(),\n",
    "        transforms.ToTensor(),\n",
    "    ]),\n",
    "    'val': transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "    ]),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = datasets.MNIST(root = \"data\", \n",
    "                               train = True, \n",
    "                               transform= data_transforms[\"train\"], \n",
    "                               download= True )\n",
    "\n",
    "val_dataset = datasets.MNIST(root = \"data\", \n",
    "                               train = False, \n",
    "                               transform= data_transforms[\"val\"], \n",
    "                               ) \n",
    "\n",
    "train_loader = DataLoader(dataset = train_dataset, \n",
    "                          batch_size=batch_size, \n",
    "                          shuffle= True)\n",
    "\n",
    "val_loader = DataLoader(dataset = val_dataset,\n",
    "                         batch_size=batch_size,\n",
    "                         shuffle= False)\n",
    "\n",
    "data_loader = {\"train\": train_loader, \"val\": val_loader}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs, classes = next(iter(data_loader[\"train\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([64, 1, 28, 28]), torch.Size([64]))"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs.size(), classes.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAusAAAI4CAYAAADTU6e8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdebyUZf3/8feHww4qmyKyCLK4LwjibpapqCnaYlop9rVQw1zSkvr21erXomnlhgsKgWaaSyqpqUSU5oKI4sKOCwIi5Aqyn3Ou3x9M9z3XxBzmPmdm7mvmvJ6PR4/zueZzn7k/2HyYi3uuuW5zzgkAAABAeFqkXQAAAACALWOyDgAAAASKyToAAAAQKCbrAAAAQKCYrAMAAACBYrIOAAAABKpJk3UzG25m881skZmNKVZRQDWjb4Bk6BkgOfqmelhj91k3sxpJCyQdLWmppBmSTnfOzSleeUB1oW+AZOgZIDn6prq0bMLvDpO0yDn3piSZ2T2SRkjK+0JobW1cW3VowinRVOu1RhvdBku7jmYsUd/QM+mjZ1LHe00Fom9Sx3tNhWmoZ5oyWe8paUnWeKmkA3MPMrNRkkZJUlu114F2VBNOiaaa7qamXUJzt9W+oWfCQs+kjveaCkTfpI73mgrTUM+U/AumzrlxzrmhzrmhrdSm1KcDKh49AyRH3wDJ0DOVoymT9WWSemeNe2UeA5AffQMkQ88AydE3VaQpk/UZkgaaWT8zay3pNEmTi1MWULXoGyAZegZIjr6pIo1es+6cqzWz8yU9IalG0gTn3OyiVQZUIfoGSIaeAZKjb6pLU75gKufcY5IeK1ItQLNA3wDJ0DNAcvRN9eAOpgAAAECgmKwDAAAAgWKyDgAAAASKyToAAAAQKCbrAAAAQKCYrAMAAACBatLWjQCwNe9dfEgUj/r2X7zcqO3eLug5zl92mDeeedt+Udz1tucaXxwAAIHjyjoAAAAQKCbrAAAAQKCqehlMTaftotj16eHl5n1n2yju3udDL/f0vn/K+5wr6tZF8eGPfa/gWrq8XBPFOz75rperfz8+f/3q1QU/JxCKFh06RLH17eXlXrz0hije5Oq83CZX2PPf2PNf3njF/02J4hM6/MDL7Xjts4U9KQAAFYAr6wAAAECgmKwDAAAAgWKyDgAAAASqqtesd3o0/rfIpL53ebnJazpH8evr/DW2v3x/77zPWWP1UXzWwf462n5tVkZx39bve7kDT9wUxS0u9/+N9J2lR0Tx3587yMvteusHUVw3d2HeuoA0zbt+9yiefexNOdkaFeL+T3fyxj979MtRvOOeK73ck3vdE8VretcLqAQ1nTt743k/HxTFLTpt9HKH918UxZfu+KSXG/HMd6L4swMWeLlpi+LnHPjr9V6u/pW5CSsGEAKurAMAAACBYrIOAAAABKqql8Hs0DbeBnF51paLknTzOd+O4pppLxXlfC/0HBbFdTv4H3euHrhNXMth/n51Nx0/MYpv/Mo/vNz+vc+K4l5fanqNQDG885NDvPFLx1yTNcr/18rX3jjRG288L95e1db6H9n3f/v5KM6+C6okaa84rOm11ktlbyNZv2ZN3lqAcqj93JAoPu6GqV7uL52m5h6+Rfu9cLY3btM2XjKze4flXu6WI5+O4juH7Ojl7v3ikVFcN8dfPgOEaNPnh3jjs258OIrvOXQfL1f3gb8Nd6Fquu8QxXOv7O3lDhz0VhR/e8d/ernfHHhkfO73P1ApcWUdAAAACBSTdQAAACBQTNYBAACAQFX1mvXnfzs0io//0q5erlu7eDu5wjaW27raZe/Gg+xYUoe2+0Zxu+UdvNyONauyRvz7CWFq2atnFJ96ir92r43l/6tk6NXfjeJeDy31cnVvN33d7MuH3u6Nv7zzN+IB63JRBi17xtuOzrm8p5d77YQboridtfZyD2RtITzuHP9LSfU1FsU7Tc35XpWLv/c0pe0OXupvreOtiBeN6+vl7Io47vtVAUHK/k7UNd+Y4OUGtorXhv+/73/Fz101L4rf/cbuXm5Dlzj+7Bf8fjqm07NRfHKHT/PW9ejatv4DdeXbNpiZIQAAABAoJusAAABAoKp6Gcx2dz2fFZf33O9d5G81d913b4niQ9tu8nL1Wf9mOuzlr3u5Pj+rzzoOSE/X++KtUH/c7dWcbLyY7Nwln/EyO14bf8RYW6RaWlmxFq8BTbd0bLwF6VvDbvNyn9THS1b2veF8L9fnltlRXPOx/9F8oa/w+vX+lqfKGvc7LbdPgfDUHbm/N3767KujuFtNh5yj4/HCM2/2U2dmD6YVpbaZG+JtUi//9XlerttHzxXlHIXgyjoAAAAQKCbrAAAAQKC2Olk3swlmttLMXs96rIuZTTGzhZmfnRt6DqC5oW+AZOgZIDn6pnkoZM36REk3Sroj67ExkqY65640szGZ8WXFLy881jL+T1Z/4F5ebtX/xlv+vLjvDcrngU+7eeNf3Xx6FGev75VYp17BJqrC+6ams//3+7Yt10bxJlfn5f66Nn5Nv3Lb3l6uq4q/ri/7/IOf+ZaX22XxG0U/H8pioiqkZ9zB+3rjx/cfG8XjPxng5R74cvwdjp6z/b/f/S5qnAU3DfPG9x93YxR/54oLvVynO8q3xhZlM1EV0jf5rO7dxhv/9zr12AYXf+fv9Y3Oy81c3zeKfzXtC15ux6fja9Od5nzi5eZdEJ/vwc+N9XJn/+7iKO4+zu/fctrqlXXn3FOSPsx5eISkSZl4kqSTi1wXUNHoGyAZegZIjr5pHhq7G0x359zyTPyepO75DjSzUZJGSVJbtW/k6YCqUFDf0DNAhPcaIDnea6pMk7dudM45M3MN5MdJGidJ21qXvMeFpOWO8et648CdvNzqH8bb1/1zX3+LrmzjPtnFG9886cQo7nmV/1HKjkrvoxWko6G+CaVnPr2nkzf+dY8n8h47/uThUdx1Tnk/aq9b6r/J1K9ZU9bzozxCeq9ZfJH/9DvUxK/Ba+7+opfrM7vpf7/nLklbMHbnKH7tiOu9XPZdUo+82O/FWXcIzUwlvNd0+5d/x/eDX4nv5vvJv/x/Z3SZHy8e63D/9LzPOUgv5D/hvv7dTdtssyGKJ354qJfrfn0Y87PG7gazwsx6SFLm58rilQRULfoGSIaeAZKjb6pMYyfrkyWNzMQjJT1cnHKAqkbfAMnQM0By9E2VKWTrxrslPSdpVzNbamZnS7pS0tFmtlDS5zNjABn0DZAMPQMkR980D1tds+6cOz1P6qgi1xKM+ZfE681nf81fD9gi6983b9Vu9HLH/yu+lfSgCxZ7uZ4fhLHuCeVRqX2z4NYD4nivW7zcpqwVjfd/6n+Xw9bm3PK8yG674LqSPj/SV0k902Z6R/+Bw+Kw6yHveama7jvkf6J1WX3Trq2XWnJmvAXk1eeM93JHt5uaNWqtfOavyv1e4Yr8taAiVVLf5FP7lj9f2va4rFjF34r30Dtf9sbfaPPvKP7DV47OOXpe0c/fGNzBFAAAAAgUk3UAAAAgUE3eurEabDjuAG/8j69enTXy76x1xcrBUfy36/wtfvpPjLfJKsad6YBya9sl/3KWFzfGH7ff+e0TvVyLt1/OPbyoBrf2ryvk3kEVKKdej/qba7x9YXx333/ufb9/8Ev5n+eu1fESma9vU5wNO2ZtrI3iT37Z28u1ZhkMmosWNd7wnR8fGMWnbne1l/vazy+N4q6vhnmXX66sAwAAAIFisg4AAAAEisk6AAAAECjWrEtq98/Z3vjeVftE8ejO873clGW7RXGXiWGubQIK1bLfzt74sD5vRnEr89f8/fh7o6K43dMN3Mq5SLXsdM/7eWsB0lQ3f5E3/tI1P4jii0b7a9ZP6ehvS5dtRMclUfxpzs3eb/kofh/6Xhd/+7gWsrzP+a2rL4riHZ5gy2A0T598zf8u4mVfi/ty+EOXeLmBt4U/l+PKOgAAABAoJusAAABAoFgGI6l+7Vpv/Pi3D4/iEfe86uVm7H9vFA//+wlebsNVPaK4zT9e83Juw4Ym1wkUm/3evwvvdT2nRfEml7P0JOdj+mJb8Xn/rqj39bwvby1T17WP4kHjP/RybOqIcut+Q7zc5O4b/Nfx3dop9/CCrBsxLIq/d1P+uyhmb9UoST3+Fm/PSC+gOcnehvtHV9zh5R79aN8o3v06/y7DfgeFiSvrAAAAQKCYrAMAAACBYrIOAAAABIo161tgz70SxV/9+fe93NpjV0fxywf/3v/FCXG428OjvdSuF8fPyfp1pGntKfFtl2/Y+fqcbLwl3M/f38fLdFz4cRQXay1sdi23/+javLXkOu/vZ0bxoDkzilQNEI4lx8dfEmloq8YvT/HfawYtoB/QPGSvUZekJ26/KYpf3uBfi178nf5R7N58vbSFlQBX1gEAAIBAMVkHAAAAAsUymK3omnNnq+3vireMG/LdC73czAuui+J5I8Z6ueMeOi+KWz35YjFLBBJ59/D4I/VBrfJ/vH73Y0d4435zmn6Xt0+/cqA3HvvreBlOQ7UcOus0b7zbxXOiuL7JVQHpa7HNNt74wsOmFPR7e1z+jjeuhG3ogGJYPMJ/z2hjraL4ksvO8XIdX5xelppKhSvrAAAAQKCYrAMAAACBYrIOAAAABIo16wnVr10bxT2eXecnL8j/ey1XbypRRUBp9Pth09eo/9dzXjzfGze0Tj1b+5s6eeP6NQuLVhMQgvm/2MMbP9Lpn3mP3fUfZ0fxgA9ml6wmIDSLf3pIFP/p2Ou83KCJ8SSs333Ff/9KE1fWAQAAgEAxWQcAAAACxTKYhD75+kFRfP3Pb8h73D6/99fE9J3+QslqAhpS07mzN+6554oobmU1Xm7o1d+N4h31bFHO/8S7s6J4k8u992l8/t3u8+/EOOCi56O4jbgrI6qPtYzfgs/57N/zHvd8zk2vB/0svpN23aaNRa8LCMVbvzzYG88eGc+7xn480Mv1m7ymLDWlgSvrAAAAQKCYrAMAAACB2upk3cx6m9k0M5tjZrPN7MLM413MbIqZLcz87Ly15wKaA3oGSI6+AZKhZ5qPQtas10q6xDn3kpltI2mmmU2RdJakqc65K81sjKQxki4rXanl07LnTlE879I+Xu6Nr94SxZuc/2+dn7+/TxQPuHWJl6utz12riyoWVs/stIM3fHzPO6N4k8s5NnecpWW/nePD2rf1ch/tE78X9P/uPC+XvU596rr2Xu6Xl4yM4kF/e83L1ecvBdUprL4pg7f/b1gUX9rlxrzHvbSunzeum7+oZDWholRlz7TYL97G9PQTnvJyK+riLbNvfvA4L9f3+erarjHbVq+sO+eWO+deysSrJc2V1FPSCEmTModNknRyqYoEKgk9AyRH3wDJ0DPNR6LdYMysr6TBkqZL6u6cW55JvSepe57fGSVplCS1VfstHQJULXoGSI6+AZKhZ6pbwZN1M+so6QFJFznnVpnFdx50zjkz2+IH6M65cZLGSdK21qWBD9nzq9l++yh+51b/I/2dron/CPbsK4U/5+7xlj/zzu3q5f58UnxXrN1b+x8+PLk2fkHf8u7RXm7DWR2iuHbJ2wXXguqUZs8Uw6LfHeSNf3H8n6L4pA4rcg9vQLw943l/P9PLDHo43tKUZS+QKr9vGtKirb987Pun/rmg37vnJ8O9cUdNL1pNqHzV1jPzRsdzqb9u79+hd9envxPFfX9cvctechW0G4yZtdLmF8Jdzrn//O2ywsx6ZPI9JK0sTYlA5aFngOToGyAZeqZ5KGQ3GJM0XtJc59xvs1KTJf3n22EjJT1c/PKAykPPAMnRN0Ay9EzzUcgymEMlnSHpNTP7z60IfyTpSkn3mtnZkhZLOrU0JQIVh54BkqNvgGTomWZiq5N159y/JFme9FHFLWfLrGO8TnzmgRO93N4nXhDF3Xr7a2yzvXeYvxzrluMmRPFn2q3NOTr+wOHBT/018tdcc1oUd70td73Uv/OeH81HCD2Tzdau98YPrukRxad0WO7lbrow3j5uWBu/Z7K3YGzILR/v5o3/durQKN5t8Rwvxzp1/EdofVMK86/Z1xufte2zeY89/a34O1EdH3ixZDWhclVrz0w66rYoPmfpwV6u35nzoziYRfZlwB1MAQAAgEAxWQcAAAAClWif9RC9dub18eDM/Me1yPl3SX3WB/DT1nX0chfc9z9RPPA2f5lA1zebz1ZBqA61by32xpc/+pUoPuXU673c4Na1WaMaFWr/8RdFcb8/f+Tl6ufMLfh5gGpT0y3eGnjqib/Jyebf2/r1J3aN4t71+ZfLAJWoRYd4e8Zef/dzh7aJ52ffv3E/L9dpQ/Ocg3FlHQAAAAgUk3UAAAAgUEzWAQAAgEBVxJr1uiXLonifiRd4uVfPuj738C365mJ/F6MZ03aP4v7Xv+Hl+q2I10TVCqguA773fBSf8r1hRXnOnRWvqWU7RiBLTfzdjz4t869R3/3ps7zxwAlvRzHvQ6g2dfsMiOLubV7ycvu98I0o7jPlLS/XXHuBK+sAAABAoJisAwAAAIGqiGUwrjb+4KPv//rb9pz0vwcU+Cwfe6O+ip+nsPsyAgBQPLVZ7z473dHGzy17t9zlACVjQ/b0xlfffWsUv7mpm5d7+bt9orh2+XulLaxCcGUdAAAACBSTdQAAACBQTNYBAACAQFXEmnUAACpR3YqVUXx8z/3zHtdGM8pRDpCOFv614W0s/i7iL355hpfrstT/biK4sg4AAAAEi8k6AAAAECiWwQAAAKBk3IzXvPG5Ox8WxV3Espet4co6AAAAECgm6wAAAECgmKwDAAAAgTLnXPlOZvZvSYsldZP0ftlOnF8odUjlq2Vn59z2ZTgPiiDAnpHCqYWewRZl+maNwnidSuH0jETfYAt4r2lQ6j1T1sl6dFKzF51zQ8t+4kDrkMKqBeEJ6fURSi2h1IEwhfT6oBZUipBeH6HUEkIdLIMBAAAAAsVkHQAAAAhUWpP1cSmdN1codUhh1YLwhPT6CKWWUOpAmEJ6fVALKkVIr49Qakm9jlTWrAMAAADYOpbBAAAAAIFisg4AAAAEqqyTdTMbbmbzzWyRmY0p87knmNlKM3s967EuZjbFzBZmfnYuQx29zWyamc0xs9lmdmFataAy0Df0DZKhZ+gZJEPPROcNsm/KNlk3sxpJYyUdJ2kPSaeb2R7lOr+kiZKG5zw2RtJU59xASVMz41KrlXSJc24PSQdJGp3575BGLQgcfROhb1AQeiZCz6Ag9IwnyL4p55X1YZIWOefedM5tlHSPpBHlOrlz7ilJH+Y8PELSpEw8SdLJZahjuXPupUy8WtJcST3TqAUVgb4RfYNE6BnRM0iEnolrCbJvyjlZ7ylpSdZ4aeaxNHV3zi3PxO9J6l7Ok5tZX0mDJU1PuxYEi77JQd9gK+iZHPQMtoKe2YKQ+oYvmGa4zXtYlm0fSzPrKOkBSRc551alWQvQWPQNkAw9AySTxus0tL4p52R9maTeWeNemcfStMLMekhS5ufKcpzUzFpp84vgLufcn9OsBcGjbzLoGxSInsmgZ1AgeiZLiH1Tzsn6DEkDzayfmbWWdJqkyWU8/5ZMljQyE4+U9HCpT2hmJmm8pLnOud+mWQsqAn0j+gaJ0DOiZ5AIPZMRat+U9Q6mZna8pGsl1Uia4Jz7RRnPfbekIyV1k7RC0hWSHpJ0r6Q+khZLOtU5l/slh2LXcZikpyW9Jqk+8/CPtHlNVFlrQWWgb+gbJEPP0DNIhp6Jagmyb8o6WQcAAABQOL5gCgAAAASKyToAAAAQKCbrAAAAQKCYrAMAAACBYrIOAAAABIrJegHMrK2ZvWBmr5jZbDP7ado1AaEzs+FmNt/MFpnZmLTrASqBmV1oZq9n3msuSrseIGRmtquZzcr636pq7Bu2bixAZpP8Ds65TzN3tvqXpAudc8+nXBoQJDOrkbRA0tGSlmrzTTdOd87NSbUwIGBmtpekeyQNk7RR0uOSznXOLUq1MKACZN53lkk60Dm3OO16iokr6wVwm32aGbbK/I9/5QD5DZO0yDn3pnNuozZPQEakXBMQut0lTXfOrXXO1Ur6p6QvplwTUCmOkvRGtU3UJSbrBTOzGjObJWmlpCnOuelp1wQErKekJVnjpZnHAOT3uqTDzayrmbWXdLyk3inXBFSK0yTdnXYRpcBkvUDOuTrn3H6Sekkalvm4EgCAonDOzZV0laQntXkJzCxJdakWBVQAM2st6SRJ96VdSykwWU/IOfexpGmShqddCxCwZfKvCPbKPAagAc658c65Ic65IyR9pM3f/QDQsOMkveScW5F2IaXAZL0AZra9mXXKxO20+Utz89KtCgjaDEkDzaxf5orHaZImp1wTEDwz2yHzs482r1f/Y7oVARXhdFXpEhhJapl2ARWih6RJmW8at5B0r3PukZRrAoLlnKs1s/MlPSGpRtIE59zslMsCKsEDZtZV0iZJozOf5gLIw8w6aPNF1HPSrqVU2LoRAAAACBTLYAAAAIBAMVkHAAAAAsVkHQAAAAgUk3UAAAAgUEzWAQAAgEAxWQcAAAACxWQdAAAACBSTdQAAACBQTNYBAACAQDFZBwAAAALFZB0AAAAIVJMm62Y23Mzmm9kiMxtTrKKAakbfAMnQM0By9E31MOdc437RrEbSAklHS1oqaYak051zc4pXHlBd6BsgGXoGSI6+qS4tm/C7wyQtcs69KUlmdo+kEZLyvhBaWxvXVh2acEo01Xqt0Ua3wdKuoxlL1Df0TPromdTxXlOB6JvU8V5TYRrqmaZM1ntKWpI1XirpwIZ+oa066EA7qgmnRFNNd1PTLqG5S9Q39Ez66JnU8V5Tgeib1PFeU2Ea6pmmTNYLYmajJI2SpLZqX+rTARWPngGSo2+AZOiZytGUL5guk9Q7a9wr85jHOTfOOTfUOTe0ldo04XRAVdhq39AzgIf3GiA53muqSFMm6zMkDTSzfmbWWtJpkiYXpyygatE3QDL0DJAcfVNFGr0MxjlXa2bnS3pCUo2kCc652UWrDKhC9A2QDD0DJEffVJcmrVl3zj0m6bEi1QI0C/QNkAw9AyRH31QP7mAKAAAABIrJOgAAABAoJusAAABAoJisAwAAAIFisg4AAAAEisk6AAAAEKgmbd0IAJK09pQDo/jdI8zLHXrQnCi+Y+enCn7Ow0efE8XtH5zehOqA9LRoH9/G/XMvrPByF3VeEMU/f38fL/fX3x4RxZ0nPVei6gBUAq6sAwAAAIFisg4AAAAEimUwW1Cz64AoXvGZ7b3czJ/cHMWbXJ2XO+hn58cDl/Ok2SsDEuS6jePjT4ThiXdnNZBtKNc4T4+9NYr7H3Gulxtw8fNFPx9QCu+dtV8UX9D5Oi9XnxX/qJvfQ0dcPi+Kf7V4pJer+cdLxSsQQPC4sg4AAAAEisk6AAAAECgm6wAAAECgWLO+Bbv8YUkU39/jj15uk6vJiv01689ffmPeXCvL/3sN5Y4c8fUo7nBDJy/X+vEZW/4DAEWw6HcH5TwSr6k9c/ERXmbFwavy/l5jt27M9sZXb/HGZx4Unz/73EBo/mf0o436vSPabozi18b+3ctNOWrXKK59z98OEqhmK79zSBTvd+ZrXu73fZ6O4jpX7+WW162N4s/98ftert+Y8L8byJV1AAAAIFBM1gEAAIBAsQxG0objDvDG1+0UbxlXrxov1yJrn8Xs5Sulyj2335/iWsb7+zp+oecQAcWUfSfS3KUn2Z55fg9vPEDxVoq52ypmf0h/rPbzctlLZrKXy0gNL5nJzvX/Hds6Ilx/umJ4FO991W15jxvcZo03bm+to/i8Tgu93LLHOkfx64e09XL169c3qk4gRAt+789zFh0TLzeuz9nr+vj5J0bx4g87e7lXD7ozimefcaOX+8JD34wHz7/a6FpLiSvrAAAAQKCYrAMAAACBYrIOAAAABKrZrlnPXqd+0fV3e7nsdVDF2oKxFDmg2Pr9YG7eXPZ2jcVaF579PG9lrZeXJI0tbJvH3LX1hz91ThS3f3B644sDiqDD/fFr8Ff375P3uDeu8bc8nXv62LzH/rz7C1F88q5n+slX8vcwUAkW/WFwHH/W/57Hnat3jOK7zxru5bLXm/fSu15q4HXfieKFX77Jyy04M/7ex6BAv/LElXUAAAAgUEzWAQAAgEA122Uw/xgff7SSu7ykGFswXrrcv8Pjiyt7R7Fz5uWeH3xPo84HNFXu3Uaf2Dn/do1v/Xr3KG6v4i8vyV2ycuyD8TaP3Z/b1ss1tK3j02PjrVcP1zlejmUxCFX/S/3P33erGR3Fc069Ie/vLTrdv7P1Lq8Uty6g1NadPMwbL/hs/D6Uuz3jnaPj7RlbPj+z4HMMmrg6ipedstbL3XrM76P42k6HeLm6jz8p+BylxJV1AAAAIFBM1gEAAIBAbXWybmYTzGylmb2e9VgXM5tiZgszPzs39BxAc0PfAMnQM0By9E3zUMia9YmSbpR0R9ZjYyRNdc5daWZjMuPLil9e02Rvz7hmtL/uaJObmRU3bivFQX85z8t1nRnndvjnSi/XeYF/u+hsA26N19XOPsHfroutGyvWRFVA3+Rue5gte6tGKd313tnr5SUVvK3ju0f43w8Z8GCxKkIJTFQF9ExT1AzcxRsv/Fb3KN5hpr82d9fbP4ri9V+p9XJtLX7rdq3830OzM1EV3jdLjsmf2+3e0d54wNTG7a1Y17F1FL9b287LHdVuQxT/rm9P/xdnVciadefcU5I+zHl4hKRJmXiSpJOLXBdQ0egbIBl6BkiOvmkeGrsbTHfn3PJM/J6k7vkONLNRkkZJUlu1b+TpgKpQUN/QM0CE9xogOd5rqkyTt250zjkzy/s5nHNunKRxkrStdSnp53XZy16khrdnbGgbxIa2YHzjgPVRPEgvKJ8kC1a6zIz/b2j1BbZubA4a6pty9kxDcpeelGK7xkLlLsHpf8S5UdzQUp5DD5rjjVcUtyyUUUjvNUm0aB9Pgk54eIaX+9Z2b0bxtcfv4eWeOm5QFJ889zQv9/ge90fxd4590ss9cYm/zSmat1Dfa2r23DWKXzrxWi+3rK4+igdNWuXl6tU4y46Ml74MadPIJ0lRY3eDWWFmPSQp83PlVo4HQP/v7XMAACAASURBVN8ASdEzQHL0TZVp7GR9sqSRmXikpIeLUw5Q1egbIBl6BkiOvqkyhWzdeLek5yTtamZLzexsSVdKOtrMFkr6fGYMIIO+AZKhZ4Dk6JvmYatr1p1zp+dJHVXkWhole536Rdff7eWy16k3tO1hbi57nfrbX8/ZxkdvNKLKwiXZRhLhCrlvFv3uoKzRLC/X/0/xWvABDzZui6yQ3LGzv8XjsdovpUqwNSH3TFNYm3iBbPYa9VwXdfG/X9F5ypoo3qfNkry/d/Mjx3rjXfRc0hJRwSq1b+pbx9PPbVu09XL7P/ntKB40a6aKYZej3ori7O8CStKCTfF3EVt8UJw18sXGHUwBAACAQDFZBwAAAALV5K0b0zb19lujuF7+zkMNbXtY6PaMpV72IkkfHhDfna6hOtm6EcXQ0FaHlWLAxVlLdL6aXh1AqXxz23jpS30wH8YDxZc7dyuGlr17eeNv7hQvicw933cWxiuJWi9ZXPRaioEr6wAAAECgmKwDAAAAgWKyDgAAAASq4tasL/7Zwd64XvG2Pkm2PUxze8ZcC46P1xCzdSOQTPZ2k1LDa/LXnnJgFLd/cHrJagK2pEWC62P+3/0NHLfa8ieBCpC7lWKhajpt540Xjdkjin/1xbu83Jc6xlsy1jn/fOt/3yOKW4s16wAAAAASYLIOAAAABKoilsG8f0689OW1s2/0cpWyPWNDCv0zsHUjGsO/Y6mUe9fS5uTdI+J+GvBgioWg2aj/NL4T6dBrvuvlJlxwbRTv1dr/aD576Uvu1o1fWXRiFPe56gUvV/xN8IDSyt1KsW+v96N41df8969/7x/HPzzhIS931rbT8j5n9tKX3FznVz6Kjyuw5nLjyjoAAAAQKCbrAAAAQKCYrAMAAACBqog16wd/66UoTrK1YXbuhVsHe7mueq6YJSbS2O0nB/3lPC83SP5aRQBAWNymjVG84++e9XL/9+QZUfzu57p6ua+f80QUX9B5npfr2iZeB7/cuOaGytNiY20UL8jqEUmatufDUVx3tf99jTtX7xjFv5w13MvdPLVDFJ/83Wle7odd50TxbveN9nIDZj9faNmpocsBAACAQDFZBwAAAAJVEctgbuwZ32lwkyt8a8O9x58fxTvfnt6yF0lacNsBcXx84dtPPrku/lin70NsygVI/p1IG7pjaa4BF4f/cSeaj7rZ86O4+2w/d8/GY6L4gsv9ZTC39P57FJ/S7ij/OXOWFAAhyn7tf++0c7zckqM7RnGXuf7S4A73x/PBfno17/PPOrOX/0DWMpg2H1TederKqxgAAABoJpisAwAAAIFisg4AAAAEqiLWrGdvZ5hk68adLy/vOvUNx8Xr0teM/sTLLRgcr6tN8me45vx4a6/Wj88oSp1AIbLXhUtS+wen5zmy9Bb9zr/ldKHr1Pv/6VxvPECsWUd6Nh0z1Bt/NPrTKO52dVsvt8NzH6kQrl9P/4FXVjWuOCAtz/trz3s38q/pmoG7RPG1O9+Rk20fRR3fqbzv/3FlHQAAAAgUk3UAAAAgUBWxDCZ3O8NsDW17WArvjzo4b+6FK8ZGcb38j1kaqjM7d+nyI7wcS1/QVDs9lfOR31fzH+stL8k57nDF22uVY0lMY7dnzPZff3agzFpss00U/+SW273cgW02RfG/JvnLYH78428X9PyLTu/kjXd5JWmFQHWo79guirvXtPNyD62J+6TLPS95uUp4l+DKOgAAABAoJusAAABAoLY6WTez3mY2zczmmNlsM7sw83gXM5tiZgszPzuXvlwgfPQMkBx9AyRDzzQfhaxZr5V0iXPuJTPbRtJMM5si6SxJU51zV5rZGEljJF1WiiIbu3VjoRb/zF+Hvu2Q96PYOfNyzw++Me/56pW/lobqzF6n/vbXc7bh0hsNVI5Apd4z2XLXl5/5g/j1dsfOTxX8PE+PvXWLzyFJzzy/RxQPuNjfdyt77fm7R/j9dOhB8S2g/7uWWQXVdeZiv5YVB8db17VXettNIrGg+qZYrGX8Npu9Rj3XYW3Xe+P7rroma9Sm2GWhOlRlzzRWXcfWUZz9XUBJ+sW846J4+w3zy1ZTsWz1yrpzbrlz7qVMvFrSXEk9JY2QNClz2CRJJ5eqSKCS0DNAcvQNkAw903wk2g3GzPpKGixpuqTuzrnlmdR7krrn+Z1RkkZJUtusTemB5oCeAZKjb4Bk6JnqVvBk3cw6SnpA0kXOuVVm8UcMzjlnZlvc/cY5N07SOEna1ro0aoecQrc9zM09tizenqehrRTrNbOBXOO2YMzNPbmuQxT/ZN5JXq7LFxZkjVj2Ui3S7JmGZC8T0buNe47/WrKSPf6vrSELW86SRPadSXOX3aCyhdo35Za99Vy96vMe1+Yjy5tD80DPbLbsyOye8f84H33QMYq3L1tFxVPQbjBm1kqbXwh3Oef+nHl4hZn1yOR7SFpZmhKBykPPAMnRN0Ay9EzzUMhuMCZpvKS5zrnfZqUmSxqZiUdKerj45QGVh54BkqNvgGTomeajkGUwh0o6Q9JrZvafz7N/JOlKSfea2dmSFks6tTQlAhWHngGSo2+AZOiZZmKrk3Xn3L8k5VsUd1Rxy9myAY/EtzmffcJYL9fQlogh5a45/4wo7vL4DKF6hdAzhTp89DneuN8P5kZxkm0diyF3C8bs7SB3espffzjgQdapV5tK6pu0nTTvlCju9ZsXvFzFLzxGwegZX22H+NWfu3VjpeMOpgAAAECgmKwDAAAAgUq0z3paBp0TLxvZ/2cXerljjn8xin/Tw/9ovLHbLDaUO3TWaVGce3dTfwtGX2ux9AXhyb276YoH4/jwU/wlMtl3H82+86jU+CUz2ctwcmsZIJa6oPLVfRJvlXrysWd4udW7bhfFt/7mWi83qFVr5bPhNz2iuE3t0qaWCFSFX33xrijO3bqx6zP5+6kScGUdAAAACBSTdQAAACBQTNYBAACAQFXEmvVsO1/+nDde9If+UXzwEUP9g7OXlOfuZ9XIXNfb/fMD1eq/1pBnrWdfkXPssdqvcefQ9K0fBFSy+ngb3/rX53mpDq/H8fceOLjgp2zDd6AA1XTazhuf3OHjrJH/ncIPDt0YxV1vL2VVpcGVdQAAACBQTNYBAACAQFXcMphcdQveiOKuWTEAAACq04YhA3IemRZFuVs3tl/QpgwVlQ5X1gEAAIBAMVkHAAAAAsVkHQAAAAhUxa9ZBwAAQPPScupMb3x8z/3zHttLz5a6nJLiyjoAAAAQKCbrAAAAQKCYrAMAAACBYrIOAAAABIrJOgAAABAoJusAAABAoMw5t/WjinUys39LWiypm6T3y3bi/EKpQypfLTs757Yvw3lQBAH2jBROLfQMtijTN2sUxutUCqdnJPoGW8B7TYNS75myTtajk5q96JwbWvYTB1qHFFYtCE9Ir49QagmlDoQppNcHtaBShPT6CKWWEOpgGQwAAAAQKCbrAAAAQKDSmqyPS+m8uUKpQwqrFoQnpNdHKLWEUgfCFNLrg1pQKUJ6fYRSS+p1pLJmHQAAAMDWsQwGAAAACBSTdQAAACBQZZ2sm9lwM5tvZovMbEyZzz3BzFaa2etZj3UxsylmtjDzs3MZ6uhtZtPMbI6ZzTazC9OqBZWBvqFvkAw9Q88gGXomOm+QfVO2ybqZ1UgaK+k4SXtIOt3M9ijX+SVNlDQ857ExkqY65wZKmpoZl1qtpEucc3tIOkjS6Mx/hzRqQeDomwh9g4LQMxF6BgWhZzxB9k05r6wPk7TIOfemc26jpHskjSjXyZ1zT0n6MOfhEZImZeJJkk4uQx3LnXMvZeLVkuZK6plGLagI9I3oGyRCz4ieQSL0TFxLkH1Tzsl6T0lLssZLM4+lqbtzbnkmfk9S93Ke3Mz6ShosaXratSBY9E0O+gZbQc/koGewFfTMFoTUN3zBNMNt3sOybPtYmllHSQ9Iusg5tyrNWoDGom+AZOgZIJk0Xqeh9U05J+vLJPXOGvfKPJamFWbWQ5IyP1eW46Rm1kqbXwR3Oef+nGYtCB59k0HfoED0TAY9gwLRM1lC7JtyTtZnSBpoZv3MrLWk0yRNLuP5t2SypJGZeKSkh0t9QjMzSeMlzXXO/TbNWlAR6BvRN0iEnhE9g0TomYxQ+6asdzA1s+MlXSupRtIE59wvynjuuyUdKambpBWSrpD0kKR7JfWRtFjSqc653C85FLuOwyQ9Lek1SfWZh3+kzWuiyloLKgN9Q98gGXqGnkEy9ExUS5B9U9bJOgAAAIDC8QVTAAAAIFBM1gEAAIBAMVkHAAAAAsVkHQAAAAgUk3UAAAAgUEzWC2Rmw81svpktMrMxadcDhI6eAZIxs7Zm9oKZvWJms83sp2nXBITOzC40s9czPXNR2vWUAls3FsDMaiQtkHS0pKXafAOB051zc1ItDAgUPQMkl7khSwfn3KeZuyj+S9KFzrnnUy4NCJKZ7SXpHknDJG2U9Likc51zi1ItrMi4sl6YYZIWOefedM5t1OYXxoiUawJCRs8ACbnNPs0MW2X+xxU1IL/dJU13zq11ztVK+qekL6ZcU9ExWS9MT0lLssZLM48B2DJ6BmgEM6sxs1mSVkqa4pybnnZNQMBel3S4mXU1s/aSjpfUO+Waio7JOgAAgXDO1Tnn9pPUS9KwzMf8ALbAOTdX0lWSntTmJTCzJNWlWlQJMFkvzDL5/1LrlXkMwJbRM0ATOOc+ljRN0vC0awFC5pwb75wb4pw7QtJH2vx9qarCZL0wMyQNNLN+ZtZa0mmSJqdcExAyegZIyMy2N7NOmbidNn9Be166VQFhM7MdMj/7aPN69T+mW1HxtUy7gErgnKs1s/MlPSGpRtIE59zslMsCgkXPAI3SQ9KkzG5KLSTd65x7JOWagNA9YGZdJW2SNDrzqVRVYetGAAAAIFAsgwEAAAACxWQdAAAACBSTdQAAACBQTNYBAACAQDFZBwAAAALFZB0AAAAIFJN1AAAAIFBM1gEAAIBAMVkHAAAAAsVkHQAAAAgUk3UAAAAgUEzWAQAAgEA1abJuZsPNbL6ZLTKzMcUqCqhm9A2QDD0DJEffVA9zzjXuF81qJC2QdLSkpZJmSDrdOTcn3++0tjaurTo06nwojvVao41ug6VdR3OVtG/omfTRM+nivaYy0Tfp4r2m8jTUMy2b8LzDJC1yzr0pSWZ2j6QRkvL+BdpWHXSgHdWEU6KpprupaZfQ3CXqG3omffRM6nivqUD0Tep4r6kwDfVMU5bB9JS0JGu8NPOYx8xGmdmLZvbiJm1owumAqrDVvqFnAA/vNUByvNdUkZJ/wdQ5N845N9Q5N7SV2pT6dEDFo2eA5OgbIBl6pnI0ZbK+TFLvrHGvzGMA8qNvgGToGSA5+qaKNGWyPkPSQDPrZ2atJZ0maXJxygKqFn0DJEPPAMnRN1Wk0V8wdc7Vmtn5kp6QVCNpgnNudtEqA6oQfQMkQ88AydE31aUpu8HIOfeYpMeKVAvQLNA3QDL0DJAcfVM9uIMpAAAAECgm6wAAAECgmKwDAAAAgWKyDgAAAASKyToAAAAQqCbtBlOtWmyzTRQf+ex7Xm7x+q5RPH/Mnl6u5dSZTT53ze4DvfG8c+PztdpxrZdr+2zHKN7xumebfG4AAACEhSvrAAAAQKCYrAMAAACBYhnMFvz7tL2i+NIu/8x73O7f2tUbD5i5XUHPv+Rb/vKZ3ie8HcU/6ftHLzekdU3+89d/Mx5cV9CpgdIbtrc3fOTBiVG85x3ne7l+P3yuHBUBQWrZu1cUz7+qW8G/95ldFkXxuN5Pebl6uShuIfNyAx84L4p7TXVert3DLxR8fiB0C2840Bu37h4vIx439E4vd0TbOL7qA38p8lPHx/O8+g8+9HJuU21WvLHRtRaCK+sAAABAoJisAwAAAIFisg4AAAAEijXrW3DC+U9t/SBJc4/4vf/A7ELPMK2BXP416svq/K0bOz/WvtATAmWz4NttvXG96qP411/21wreduMRUVy77N3SFgakIes7HENufcVLDekQv9ec1OEjL5fdNy1yrqtl5+obyOVej5v7pRuj+KeHD/Fyz9QfFMVt/8L6dYRv+SWHeON2K+PvYVw7fJKXO6796rzP837d+ig+u9MsL/e95+fl/b1Bj50bx9+e0XCxTcSVdQAAACBQTNYBAACAQDXbZTDWMv6jL7x6qJd7pNvY7COLcr5P3YYo/ue6rl7uoifPiOKuM/1/P7VaE3+s89Fufq7Pndy1FJXluPb+R/3jOrRLqRKgNFZ81/9o/paLb4jiA9r47ycNbbOYfS1tyIxveJnO4+O7bDe0ZGX5Q7t740f2vy2Kf9n9VS936OgB8eAveZ8SKCtr1dobt+gUb5G90wmLvdxfdp3cqHMcd8WlUdzl60u83CO7PZz391q0i7dulOX0r/O3Rm0qrqwDAAAAgWKyDgAAAASKyToAAAAQqGa7Zn3d8P2jeMGpN+Vk47VHj6/zt0e8+P5vxoOcJUmd5uc/X/uV8dqmNo/5W/wM1PSGi83YZuuHABVl/nnbR/GAi99IsRKgOLLXqEvS4Db5t1kc+3H/KJ508/FebuR5j0Vx7/NXebnapXMLqqXHyf5xX/qf70fxmZc85uX+vvefovjQsy/wcl3HP1fQ+YBie+sn/hajk06Ptx9dVts57+99Ur/eG9dkzes6tmjj5epO/jCKf9v/vpxnaq18hvR9J4pXtfaPcxs25B7eJFxZBwAAAALFZB0AAAAIVLNZBlP7Of+jlKuuvzlr5G+588Ca+KOVCV870cv1e5GPA4Ekcu++mK2+Q10ZKwFKY8jL8VKX/96eMX79527P+Mj5n4vinRa94+WmPLRnFNcuXVaUOrtMiN+/rv/8Z73cuZ95M4qPOf8ZLzdzPNf1UD4t+/aJ4su/fK+XG9ImO/a3An5mfasoPn/cd73cmn7xUuQFJ97s5aYP+WPWKP+yl7+u9RcjL/9NvN1p+w2FLWduLDoQAAAACBSTdQAAACBQTNYBAACAQG11zbqZTZD0BUkrnXN7ZR7rIulPkvpKelvSqc65j/I9R1r+fd7BUXzT92/0crnrCrNdfeXXorgLa9TRCJXcN8VWr/r8yeLekRkVrJJ65oOzD/bG53a9Oorr1c7L+a///NfHapcsLUpthdp+cltvXP+ZBvoUwaqkvilUXdd4bfipHVfmPe6u1T288X3HHxLFPd981svtPrNxX9FctCnegvHWz43wcu2XlHaderZCrqxPlDQ857ExkqY65wZKmpoZA4hNFH0DJDFR9AyQ1ETRN1Vvq5N159xTkj7MeXiEpEmZeJKkk4tcF1DR6BsgGXoGSI6+aR4au3Vjd+fc8kz8nqTu+Q40s1GSRklSW7XPd1hR1B7lb894yw+uj+IhrWsKfp57rog/0qy7Iv9ymVxPr43vRvfzaSd5uV1/vy4evLrAyxX7TlcIVkF9U86eSdsu9/LROxoU5HvNNy95xBv3qImXvlyxcrCXe+byg6L46uvHerl1l30cxR2nFbPCrdvmnue98U8vG5LnSFSgin6vWb9jYbW8uLqfN6598+0orj9sPy93RpdxWSN/Prhg08Yo/tqs//FyXW+Ma2m1ZGZBdZVCk79g6pxzamDlqXNunHNuqHNuaCu1yXcY0Kw01Df0DPDfeK8BkuO9pjo0drK+wsx6SFLmZ/5vAAD4D/oGSIaeAZKjb6pMYyfrkyWNzMQjJT1cnHKAqkbfAMnQM0By9E2VKWTrxrslHSmpm5ktlXSFpCsl3WtmZ0taLOnUUhbZkOx1Sb+6/RYvl2Sdera+LRu3dqv/tu9G8Vkj/FqUtePPgCdGealB//Nio86HcIXeN6FosyRes1uXYh1IXyX1zKjt3vbG2dszTr7nMC/X8y/xFnIjh1zo5WZ9+7ooPmmYv1ZWL7zWxCrRHFRS3xTq/bPXFnTct7o95Y1P+9lFUbyxq/99qH0amA+e8epZUbzjyXMLOne5bXWy7pw7PU/qqCLXAlQN+gZIhp4BkqNvmgfuYAoAAAAEqrFbNwZj+6sWR3GSZS+funi7xOGvfcPLrXoq7+5g6vlU/PHMB3v6d6r7eI/4C9ff/Nw/vNwPu86J4kXHjvNyA+/4VhTvfsUHXq72rcUCKsnfj/mdN26Rc0fHbHMv7RLFg0a9UbKagKZaN2JYFLfQSznZ+LpX2/fz35a3z0/8uyoeuuSCKO6qT5tWINDM7Nnan8K+dvaNeY5s2EfLt43i7ZtUUelwZR0AAAAIFJN1AAAAIFBM1gEAAIBAVfya9Y9HdoriQVec7eVatamN4v/b9zEvN370yVG87d/8W8huq8LWznZ7JmecFT+zjb/y6ZgH4/M9uftDXm7hUbdH8X7b+Ovnd/pS1jr8eja3Q/i+9Irfh88N+UPeY1uvqPi/gtBMdPreO1Fcn3NDyLEf94/iruOfK/g5kxwLoDiuWDnYG+9+/aooDnWWxZV1AAAAIFBM1gEAAIBAVfxn0HWL3oriAWfkP+4O9fbGrTQzz5HFUb96tTdu+fl4/OjCjl7uhPbxll2zhvlLBg44Z3QUb38zH5kifNOH/NEb1+c5TpL637kyikP9+BGQpL2y7lDdQublRneKl07+VUPKVlNT1B7l13nSdvGWwpM/GZx7OFA26xZvEw8ObNxzjFpypDee8dDeUdznVv8upXUfzW/cScqIK+sAAABAoJisAwAAAIFisg4AAAAEquLXrFei/7vpLG98wqX5b5Fbd+zH8eDmEhUEFFHutlhX7FDa74cA5Za7dWN9g9/MCFPfX/jrdAe3if8MZzx5qJfbRXxfCuWz/YtZg1Mb9xxdWq3xxtnr1Os++qhxT5oirqwDAAAAgWKyDgAAAASqqpbB1B+2nzeuWbMpit3Ls8tdTl5dX9uQdglAyfyy+6veeJPjmgCqS+7Wjbd8PCClSpL54OyDo/ix3mO93IwNcZ/uchnLXlBaLXv3iuKhj7zl5U7Z9rrsIxv1/P+41d/zsdtHlf2a5l0UAAAACBSTdQAAACBQTNYBAACAQFXcmvUWbdt644Xjd4/ifxx+g5c78t5Lo7j/y6Wt67+0qPGGNbv0ieJTb3i04KdZt75V0UoCymGTq/PG2dvavbzBvz5gGzaWpSagqf7y9l5R/NMd/DeUUZ0WRfEjOqBsNW1N9rpgSfrhZXdFce72k9+c9N0o7qNnS1sYmp0W7dt746U3bBPFD3V7Nefopk9NOy6v2/pBFYQr6wAAAECgmKwDAAAAgaq4ZTBv3znQG88/ZHzWyP+YZeAd8d0/y3F/uZY9d4riD27za3lm33sLeo63a9d6412uiSt3uQcDFeaOD/w7I7qPPkmpEiCZnU6ZE8Uvv+W/owxuHV/3Wv7Q7l6ux8lzlZY3vtXHG5/UIb5z45cWneDl+vyUpS8onYX/bx9vPPeAsXmOxJZwZR0AAAAIFJN1AAAAIFBbnaybWW8zm2Zmc8xstpldmHm8i5lNMbOFmZ+dS18uED56BkiOvgGSoWeaj0LWrNdKusQ595KZbSNppplNkXSWpKnOuSvNbIykMZIuK12pm8099E5vXJe1kHvf68/3cr3mvFDQc9YM6OeNbVNtFNd36ujl3hjTOorvPug2L7dNi01R3L9lu7zn2+BqvfHe086J4h4PtfZyHV+blfd5EKygeiYk8z/ZwRu3WvtuSpUgQBXTN1977tveePZn4veCL+w828vNLPMH2G/++uAovvPL/nbGYz/uH8WfXOOvZ2+r90pbGEoh6J5Z/4VhUfzYl3+Tk20rFG6rf4s455Y7517KxKslzZXUU9IISZMyh02SdHKpigQqCT0DJEffAMnQM81Hot1gzKyvpMGSpkvq7pxbnkm9J6l7nt8ZJWmUJLXN2a0FqHb0DJAcfQMkQ89Ut4In62bWUdIDki5yzq0ysyjnnHNmtsWdBZ1z4ySNk6RtrUtJdx9cu+d6b7zi3PgjmDW9/FMPOyLeTusHO93t5f5d1yGKj2y7Sfnl/ufL/5/z3KWHR/Hs3+7t5Qb86fm8v8d2jZWrEnqmFJbWrvPGvbKWhK1a73/02aXWXxIGVELfdHzOn9i0+Ez8IfUvu/t3Y9zl+niZ48ALphe9ljWP7+KN5+x9YxSvqPN78a7fHBfFXf7yXNFrQTpC7Zl3snYH7dey9MteLn73kChuP8Xvw3Js311KBS2mM7NW2vxCuMs59+fMwyvMrEcm30PSytKUCFQeegZIjr4BkqFnmodCdoMxSeMlzXXO/TYrNVnSyEw8UtLDxS8PqDz0DJAcfQMkQ880H4UsgzlU0hmSXjOz/2xN8iNJV0q618zOlrRY0qmlKRGoOPQMkBx9AyRDzzQTW52sO+f+JcnypI8qbjlNs/Co2/0HCq7O3y5RrfKvU5+6rk0U/++8U7xc/V+6RvEOf3zdy7l18drBbWrzr1FH5auknimFzz15sTeed/xNKVWCSlJJfbPTJP/v97Hnxlsiju70hpeb+6V4Dfmhr1zg5bqOL2zd+LoRw7xxp++9E8WPDLjPy9Vnrc495Sff93JdJrBOvZpUUs+Uw6raeH5Wv35VipUUH3cwBQAAAALFZB0AAAAIVKJ91kOw64TzvLHrvzaKfzz4US/3zsZuUbx4XVcv9883B+Q9x6ZV8bKYfg/4uxm1WxB/qbrL2wvyPkelbxMENNbA32/0xn89Mr7T9dg97/Jyp03K2tZu5EulLQwokrpV/kfsf/htvCXiIT++3ssNbh1fE5v+s7Ferv5n8ftLi5zVDPXKzr2UN3fFysFe7pUTe0dxl6Use0F6tp9eEw9OKv7zT13nb6G64uK+WSN/68ZKx5V1AAAAIFBM1gEAAIBAMVkHAAAAAlVxa9b7/jj/Gry71KuB31zjjfrplUadn5ujAw2zZ/3e+uWCeD3v0/v90cu1aFnSu8IDZZG9BeMP3zrXy/X5ZfzdpnG9/+Hl6r1vN7XIm5u5wc994bm3VwAAHcVJREFU47lvRfGul/k3p6xduqygmoFS6zwp3qZ6xKPDvdz8/423O/3GZ5/2cj/uln+9+aAn4u857XpOznGbqmudejaurAMAAACBYrIOAAAABKrilsEAqCydT1gYxSfpAC/XXy+XuxygpFr+faY3fvegOP6ChhTlHNl9w9JMBMvFyxzr/v1vLzXgonj8vFp5uYb6ZJBejJ++qfVVEK6sAwAAAIFisg4AAAAEisk6AAAAECgm6wAAAECgmKwDAAAAgWKyDgAAAASKyToAAAAQKCbrAAAAQKCYrAMAAACBYrIOAAAABKplOU92yLHD9Pjj95XzlMhhZk+kXQMKR8+kj56pPPRN+uibykLPpK+hnjHnXDkLeVxSt8z/3i/bifMLpQ6pfLW875wbXobzoAgC7BkpnFroGWxRpm92UxivUymcnpHoG2wB7zUNSr1nyjpZj05q9qJzbmjZTxxoHVJYtSA8Ib0+QqkllDoQppBeH9SCShHS6yOUWkKogzXrAAAAQKCYrAMAAACBSmuyPi6l8+YKpQ4prFoQnpBeH6HUEkodCFNIrw9qQaUI6fURSi2p15HKmnUAAAAAW8cyGAAAACBQZZ2sm9lwM5tvZovMbEyZzz3BzFaa2etZj3UxsylmtjDzs3MZ6uhtZtPMbI6ZzTazC9OqBZWBvqFvkAw9Q88gGXomOm+QfVO2ybqZ1UgaK+k4SXtIOt3M9ijX+SVNlJS7f+UYSVOdcwMlTc2MS61W0iXOuT0kHSRpdOa/Qxq1IHD0TYS+QUHomQg9g4LQM54g+6acV9aHSVrknHvTObdR0j2SRpTr5M65pyR9mPPwCEmTMvEkSSeXoY7lzrmXMvFqSXMl9UyjFlQE+kb0DRKhZ0TPIBF6Jq4lyL4p52S9p6QlWeOlmcfS1N05tzwTvyepezlPbmZ9JQ2WND3tWhAs+iYHfYOtoGdy0DPYCnpmC0LqG75gmuE2b4tTtq1xzKyjpAckXeScW5VmLUBj0TdAMvQMkEwar9PQ+qack/VlknpnjXtlHkvTCjPrIUmZnyvLcVIza6XNL4K7nHN/TrMWBI++yaBvUCB6JoOeQYHomSwh9k05J+szJA00s35m1lrSaZIml/H8WzJZ0shMPFLSw6U+oZmZpPGS5jrnfptmLagI9I3oGyRCz4ieQSL0TEaofVPWmyKZ2fGSrpVUI2mCc+4XZTz33ZKOlNRN0gpJV0h6SNK9kvpIWizpVOdc7pccil3HYZKelvSapPrMwz/S5jVRZa0FlYG+oW+QDD1DzyAZeiaqJci+4Q6mAAAAQKD4gikAAAAQKCbrAAAAQKCYrAMAAACBYrIOAAAABIrJOgAAABAoJusFMrO3zew1M5tlZi+mXQ8QOjObYGYrzez1tGsBKoWZDTez+Wa2yMzGpF0PEDozu9jMZpvZ62Z2t5m1TbumYmOynsxnnXP7OeeGpl0IUAEmShqedhFApTCzGkljJR0naQ9Jp5vZHulWBYTLzHpKukDSUOfcXtq8T/xp6VZVfEzWAZSEc+4pSdxsBSjcMEmLnHNvOuc2SrpH0oiUawJC11JSOzNrKam9pHdTrqfomKwXzkl60sxmmtmotIsBAFSdnpKWZI2XZh4DsAXOuWWSrpH0jqTlkj5xzj2ZblXFx2S9cIc55/bX5o8nR5vZEWkXBAAA0FyZWWdt/vSpn6SdJHUws2+kW1XxMVkvUOZfb3LOrZT0oDZ/XAkAQLEsk9Q7a9wr8xiALfu8pLecc/92zm2S9GdJh6RcU9ExWS+AmXUws23+E0s6RhI7XAAAimmGpIFm1s/MWmvzF+Ump1wTELJ3JB1kZu3NzCQdJWluyjUVHZP1wnSX9C8ze0XSC5Iedc49nnJNQNDM7G5Jz0na1cyWmtnZadcEhMw5VyvpfElPaPOE417n3Ox0qwLC5ZybLul+SS9Jek2b57XjUi2qBMw5l3YNAAAAALaAK+sAAABAoJisAwAAAIFisg4AAAAEisk6AAAAECgm6wAAAECgmKwDAAAAgWKyDgAAAASKyToAAAAQKCbrAAAAQKCYrAMAAACBYrIOAAAABIrJOgAAABCoJk3WzWy4mc03s0VmNqZYRQHVjL4BkqFngOTom+phzrnG/aJZjaQFko6WtFTSDEmnO+fmFK88oLrQN0Ay9AyQHH1TXVo24XeHSVrknHtTkszsHkkjJOV9IbS2Nq6tOjThlGiq9VqjjW6DpV1HM5aob+iZ9NEzqeO9pgLRN6njvabCNNQzTZms95S0JGu8VNKBuQeZ2ShJoySprdrrQDuqCadEU013U9Muobnbat/QM2GhZ1LHe00Fom9Sx3tNhWmoZ0r+BVPn3Djn3FDn3NBWalPq0wEVj54BkqNvgGTomcrRlMn6Mkm9s8a9Mo8ByI++AZKhZ4Dk6Jsq0pTJ+gxJA82sn5m1lnSapMnFKQuoWvQNkAw9AyRH31SRRq9Zd87Vmtn5kp6QVCNpgnNudtEqA6oQfQMkQ88AydE31aUpXzCVc+4xSY8VqRagWaBvgGToGSC55tw3HZ7a3hvf1/+JKD7xM/+/vTuPkrI68zj+e7pZm00Qg4Q9bIpRwAVZlJmIoiQ5QWOUOJOAS0Iw6KjxqMTJyZxMMgljEpe4RIlBIDoZjAY0aozEMBGVRUESEcTGhYiggKggS0PTd/6g8r51K1R3VXUtt7q/n3Ny+rl136r3SZ96qOvbT933fG/u4IY3i5JTY3AHUwAAACBQLNYBAACAQDWqDQYAAAAotoo2bbxx7eNx68tv+j/izdXJFSWnQuHKOgAAABAoFusAAABAoFisAwAAAIGiZx0AAABl5eMJQ73xn465K2lkxU2mwLiyDgAAAASKxToAAAAQKNpgAOTVnvNO9cabx8Z/jnx90t05vWb/+dO88YBrluX0OkA52vboYG/8wkm/zuh5leZfjzvo6qJ4WY1/7NdnXxHFvX7wfJYZAsXx2s9HRPEjE25LmW2Z9nnj134xiqt27813WgXHlXUAAAAgUCzWAQAAgECxWAcAAAACRc96I1Qe0cl/oHXrOD7yCG/qnbO7RvGebv5tb3ucuCWKu7TZ7c0N7LAtilcPzzVToLC6Le0YxfP63JP31/+HXvdJcUg/O5qCyoGf8sbVX+8Wxc8N/4k3Vyf/Nuvp1LmDaedGtPbHC7/+4yi+YvF0b86eW53R+YB8qzzO/77G/HPujOJjW6bvUV+421+Dtfnijiiu3bUrT9kVD1fWAQAAgECxWAcAAAAC1WzbYD6cPCqKt52x35s7bXB1Rq9x1dGLvPGwVpn9Olfu9/80ubamRxTfdP+XvLl9C7okjdZn9PpAIWy4ZWQU57oFY6rkFpbU9pXkLSD7Xb/Om5vX55m0ufRX+tcESqni08d44+qL4z/V/2Ti/d7c56o+Shpl1vbSGP1axOfYOKGtN9f3uYKfHohUdu4cxa9/1+/XGt4qvsZc4w54c99577QoXn/pQG+ubpf/GVJuuLIOAAAABIrFOgAAABAoFusAAABAoJpNz/r+s0/2xhfd8Psonth+jTfXu0VVFP+tdo83N+/D+Fa3Fy78N/8kSTsyttrp/3dQz8Xx7W1bvbbFm6vd8m4U95J/m+f0G28BxZVrn3pyX/onn/G3LR2wIH1PedWC5VH83NiR/mRSz3qq5DzPvmZYpmkCBbH5utFRfN1lD3pzF3V4L6fXrFNd/JpbRntzv1s9NIp7P+J/Dp0386konn7E6zmdGyi0faf0j+I1p6XfCnjzQX+FtO6k2uRRvtMqKa6sAwAAAIFisQ4AAAAEqkm3wew/55Qovu3u2725f73zW1G86KGh3pyrirewsj37vLnaNzdG8QDlti1cbcOHAMFJbmepryWmEHcUHTNybcbHTt44Nmm0s9HnBrJR2bGjN552ye+iOJu2l4/q4s+eW9/328D+cFu8RV2X+5Z6c4P0YpxL0hZ4kvTxwcJvAQlkq6JdO2/sLI6f2ttO6Vzx52944+T3flPDlXUAAAAgUCzWAQAAgEA1uFg3s9lmttXM1iQ91sXMFplZdeJn5/peA2huqBsgO9QMkD3qpnnIpGd9jqQ7JM1LemyGpKedczPNbEZifEP+02ucty6It4k7rmUrb67Hz1ZGcW1NTdFyQrMxR2VaN+kk956f/ozfK7h5rB32uMbYc96pUTyvT/rtu1LNS9rW8WyxdWMZmaOmUDOtWnrDM6rWR/GGA6kHxz6/4FveuPMrcU0dea/fl95F8biiqsqbs6q2Ufzu7K7e3A1H/jHt+ZO3g6zYb2mPQ3DmqMzr5tU7jvHGZvHa7fbjUr5TmLRey1ePurVuHcUVHdqnPa5u18dpcym0Bq+sO+eekbQj5eGJkuYm4rmSzs1zXkBZo26A7FAzQPaom+Yh191gujnn/n5nn3cldUt3oJlNlTRVktqoKt1hQHOQUd1QM0CEzxoge3zWNDGN3rrROecs+W8W/zg/S9IsSepoXdIelw8tuh/tjR86464oHvDU5d7coJqVAkqlvropZs3kKvnuopI0YEH+z9Hv+qZ1Bzo0TkifNfU5uP19b3x139FpjvRlsxWwJbV1HvlHv8Xzvj5PpR6ekZOWXxzFvf/z+fQHoqyUw2fNl4au8sY/7Ba3t4z+lyu8udStSnPxxsxR3vgTw+ItVf/v+N+kfd4xi7/mjQd85aVG55KpXHeDec/MuktS4ufW/KUENFnUDZAdagbIHnXTxOS6WH9U0pREPEXSI/lJB2jSqBsgO9QMkD3qponJZOvGX0taKmmwmW0ys8skzZR0lplVSzozMQaQQN0A2aFmgOxRN81Dgz3rzrmL0kyNy3MujVZ9ZT9vPKxV/H+v1Sa/r6/yuMFR7N7a5M3V7d5dgOzQnJRT3YRiwy0pt1Tvc3dOr9N//rQozqYPGKVFzfha9OoZxW9c2tubGz4+/j7HfX3Sb8dYnyF/vswb958cbdOtIL8og8NqCnVTkdJSX6HGbx1a89lTvHHlt+K+9FePvbOeZ6Y/9wOj7/XG3x0T15A9tzq7BLPEHUwBAACAQLFYBwAAAALV6K0bS62y2yeieNak9Hc5XHtpyp89Lo3DX+3yt3z83rNfiOIjl/l3o+s654UodrW12aQKIEVy68vrk3Jrezl9un831QELaH1BedhxabyF3I7j/VaAeRPjrYdHtM6tMeXxPZ288W1Xxh0TA5as9ebq+DxDEZ2y+mAU39jV30p7yb42UXxE9d6cXj+57UWSnjw23l+4LvXgDJ3U2h9Pmf27KJ43uFeOr5oZrqwDAAAAgWKxDgAAAASKxToAAAAQqLLvWa/b8WEUT1/t72C0Z2fc99TmrZRmo6TdeTqN8G/u9aezbo3i3hOqvLlrLx8RxdUTe3hztZveySxpoJnac96p3jjXPvXJG8dGcdWC5Y3KCSiWrdNHe+M/zfhJFLevaJ16eKPNrD7HG3d6Mv7OVa59u0Autk8d5Y2v63pzFLc0f2vthR+eGMUVz6bfEnHvxBHe+Ds/vS+Kx7Vd5c0lv98f2NXdm7tpzfgoHnTUNm/uNwOeSHv+2W+PieIW+lva4/KBK+sAAABAoFisAwAAAIEq+zYYd2B/FPc8/5W8vOb0I+OtGzd+4xhv7vlv/jSK1y/xf33fO/3cKKYlBjgkufWl3/Xr6jkyc2/edGwUV4k2GJSHfV39cSFaX4AQ1bb17wxaldT6sqLGn3vxhydFcbuUf98rOnSI4vZX+3ef/0zbfUkj/zXHr/1iFLee0d6b67Uyvnvv7tOGeXMr5sWvk7qF6huvxdt+D6INBgAAAGieWKwDAAAAgWKxDgAAAASq7HvWC+Hg+zuiuOcPn/fmzl86PYoX/uoub+6YR7ZE8dqx7by5ut2785kikLHU7RI3j4178AZcsyzv59twy0hvnOv2jP3nT4vi1DzpU0c56vP4Tm886OjLo/jUEzZ4c+sf8L8vlazHhW9G8YKBj6U9bsbAJ73xPcPi72PVrV5bf7JAHp05Of1nzS3vjPfG7R5O/+/7+h8NieNBd6U97vilk71x7y+/GsWutjbt87af4G/XfVTl3qRRG2/ujvHzovhnSl+v+cCVdQAAACBQLNYBAACAQNEGk6XKxfFdsT7z3Wu8uWXfvzOKJz3p/1ln1+m0waB4ui3tGMXz+tyT/sBJ6aeS7xIqSc8tG5LmSGnMyPhP6n/o0/i7kkqFadEB6lNRFf8JfNtFQ725rvfH//a7mpqcXt+9uMYbD3oxjj9IOfYTel7pvHLyyfFgYPrz/WVPb29c8VH8OcQdTFFMZ3d6Oe3cd3o+7o0v+Wa8tur+1BZvbtAQf7vGdL5/wqPe+GcTvhzFlTX+u3/7p+NtJB+66sfeXL8WcevL9oN7vbnr5lwZxb3qqdd84Mo6AAAAECgW6wAAAECgWKwDAAAAgaJnvRG6zv+rN/7R1XFP7+19FnpzF5x/bRTXty0RkIvU7RJz7RtPNq/PM/4DqeM8e2/UzoYPAgrosepno7hOS7y5Wdf2jeLHz/fr7eC66oLmlavfv+N/z6TTmxvSHAnkX4vuR0dxu4pVaY87rpW/FF3x77dH8e3f9L+UcWXnzGrt/Pb+58kX7v55Rs9L3Z4xuU993C+u9+Z6/aCwferJuLIOAAAABIrFOgAAABAo2mAaIfWupE/+5z9F8bdv8+8Ot/WC+E8p/R4ubF5AOfrD5tXe+PTp34jiqgW0jqG0pnZ6K4ofHDDBm2uzLv/na9GzRxRXT/e3YFx11k+TRq29uds/iNsGulzlb1F3MH/pAQ16+6JPRfHQVvtTZltm9BqZtr3kywd1+7zxuFlx60sx215ScWUdAAAACBSLdQAAACBQDS7WzayXmS02s7Vm9oqZXZV4vIuZLTKz6sTPzoVPFwgfNQNkj7oBskPNNB+Z9KzXSrrWObfKzDpIWmlmiyRdLOlp59xMM5shaYakGwqXavg6vbA5il874Pc9jej9tyjeVrSMUCJFr5kxI9c2fFCRTN441hs/tyzePu71SZlvKbnkznvi17zef022eWySmvVnzaZvj/bGN102O4rHt92dcnRrpXPn6vi7UwOqX8pLbghW0DXT/ea4x/v446d7c0vG3RbF3Srb5vT6dfK/k7GxNu6L71KRemxsV53z5m7ZOi6KV9xxojfXa27p+tSTNXhl3Tm3xTm3KhHvkrROUg9JEyXNTRw2V9K5hUoSKCfUDJA96gbIDjXTfGS1G4yZ9ZU0XNJySd2cc1sSU+9K6pbmOVMlTZWkNqrKNU+gLFEzQPaoGyA71EzTlvFi3czaS3pY0tXOuZ1mFs0555yZucM9zzk3S9IsSepoXQ57TFNUmfLrqLC6NEeiqSpmzaS2hfS/ZVoUZ9N6kqn+86d5408+E6eZus3iAC2L4rOvGZb2Nfecd6o33jw2/n0NuGZZ6uFoosrhs2bbUP+js2vliIyet2VUpTf+3JkvRPEvjrrJm+teT2vAARdvwjjt7TO9uYG3HojiZvOB28yVQ80MumSlN/7aSfHWvG9c0NGbq+1eE8VVHWq8uXM/Fd85fnJn/3Phyj5jorhmwilpc6la8bo3Pvj+jijurKVpn1dKGe0GY2YtdeiN8IBz7reJh98zs+6J+e6SthYmRaD8UDNA9qgbIDvUTPOQyW4wJumXktY5525OmnpU0pREPEXSI/lPDyg/1AyQPeoGyA4103xk0gYzRtJXJb1sZn+/xeCNkmZKetDMLpO0UdKFhUkRKDvUDJA96gbIDjXTTJhzxetq62hd3Kk2ruEDy9XIE6LwiYfneFNTNp4RxdtGf1isjP7Bcve0drod1vCRCEEhambDLSNzel5z7RunZspPrnXzxDurorguoI7vp/a288Yv7o5v4/780FbFTicj1E15afLrszJQX81wB1MAAAAgUCzWAQAAgEBltc86fNbC//VtnnEgzZHSsqXHRHF/Nc92AoShubazAA05funkKH5p1H3eXEUBrm19UBff6br6gL9V41cfuzyKB9/7kTdX99dX854LgHBxZR0AAAAIFIt1AAAAIFAs1gEAAIBA0bPeCBv++2RvvP6Uu6J40V6//3DwHVuiuLawaQEActDrS2uiePhDl3hzfxk1N6fXPK/681Fc/Wxfb67zq/H2kJ3u979LMlDLo7gupzMDaCq4sg4AAAAEisU6AAAAECjaYBqwa5J/t8cOUzdF8YZj7/bmVtQcjOKbpn3Fm2v55soCZAcAKITklhhJ+rxOyvGV4hbIvkkxAGSKK+sAAABAoFisAwAAAIFisQ4AAAAEip71BnSYn3Jr9vlxeLaGpX1eS9GjDgAAgMbhyjoAAAAQKBbrAAAAQKBYrAMAAACBYrEOAAAABIrFOgAAABAoFusAAABAoMw5V7yTmW2TtFFSV0nbi3bi9ELJQypeLn2cc0cV4TzIgwBrRgonF2oGh5Wom90K430qhVMzEnWDw+Czpl4lr5miLtajk5q96Jw7uegnDjQPKaxcEJ6Q3h+h5BJKHghTSO8PckG5COn9EUouIeRBGwwAAAAQKBbrAAAAQKBKtVifVaLzpgolDymsXBCekN4foeQSSh4IU0jvD3JBuQjp/RFKLiXPoyQ96wAAAAAaRhsMAAAAECgW6wAAAECgirpYN7NzzGy9mW0wsxlFPvdsM9tqZmuSHutiZovMrDrxs3MR8uhlZovNbK2ZvWJmV5UqF5QH6oa6QXaoGWoG2aFmovMGWTdFW6ybWaWkOyVNkDRE0kVmNqRY55c0R9I5KY/NkPS0c26gpKcT40KrlXStc26IpJGSpid+D6XIBYGjbiLUDTJCzUSoGWSEmvEEWTfFvLI+QtIG59wbzrn9kv5X0sRindw594ykHSkPT5Q0NxHPlXRuEfLY4pxblYh3SVonqUcpckFZoG5E3SAr1IyoGWSFmolzCbJuirlY7yHp7aTxpsRjpdTNObclEb8rqVsxT25mfSUNl7S81LkgWNRNCuoGDaBmUlAzaAA1cxgh1Q1fME1wh/awLNo+lmbWXtLDkq52zu0sZS5ArqgbIDvUDJCdUrxPQ6ubYi7W35HUK2ncM/FYKb1nZt0lKfFzazFOamYtdehN8IBz7relzAXBo24SqBtkiJpJoGaQIWomSYh1U8zF+guSBppZPzNrJenLkh4t4vkP51FJUxLxFEmPFPqEZmaSfilpnXPu5lLmgrJA3Yi6QVaoGVEzyAo1kxBq3RT1DqZm9llJt0qqlDTbOfdfRTz3ryX9s6Sukt6T9B+SFkp6UFJvSRslXeicS/2SQ77zOE3SEkkvS6pLPHyjDvVEFTUXlAfqhrpBdqgZagbZoWaiXIKsm6Iu1gEAAABkji+YAgAAAIFisQ4AAAAEisU6AAAAECgW6wAAAECgWKwDAAAAgWKxniEze8vMXjaz1Wb2YqnzAUJmZr3MbLGZrTWzV8zsqlLnBITOzNqY2Qoz+0uibr5X6pyA0JnZEWb2kJm9ambrzGxUqXPKN7ZuzJCZvSXpZOfc9lLnAoQucYe37s65VWbWQdJKSec659aWODUgWIkbsrRzzn2cuIvis5Kucs4tK3FqQLDMbK6kJc65exM3dapyzn1Y6rzyqUWpEwDQ9Djntkjakoh3mdk6ST0ksVgH0nCHrp59nBi2TPyPK2pAGmbWSdJYSRdLknNuv6T9pcypEGiDyZyT9JSZrTSzqaVOBigXZtZX0nAdugMcgHqYWaWZrZa0VdIi5xx1A6TXT9I2SfeZ2Utmdq+ZtSt1UvnGYj1zpznnTpQ0QdJ0Mxtb6oSA0JlZe0kPS7raObez1PkAoXPOHXTODZPUU9IIM/t0qXMCAtZC0omSfu6cGy5pt6QZpU0p/1isZ8g5907i51ZJCySNKG1GQNgSPbcPS3rAOffbUucDlJNEz+1iSeeUOhcgYJskbUr6C9RDOrR4b1JYrGfAzNolviSnxJ9XxktaU9qsgHAlvij3S0nrnHM3lzofoByY2VFmdkQibivpLEmvljYrIFzOuXclvW1mgxMPjVMT/G4UXzDNTDdJCw6tP9RC0v84554sbUpA0MZI+qqklxP9t5J0o3PuiRLmBISuu6S5ZlapQxfTHnTOPVbinIDQXSnpgcROMG9IuqTE+eQdWzcCAAAAgaINBgAAAAgUi3UAAAAgUCzWAQAAgECxWAcAAAACxWIdAAAACBSLdQAAACBQLNYBAACAQP0/mUO/+FBwhRsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x576 with 16 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_batch(inputs.squeeze().numpy(),classes.numpy(),(4,4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_weights(layer):\n",
    "    if type(layer) == torch.nn.Linear:\n",
    "        torch.nn.init.xavier_uniform_(layer.weight)\n",
    "#         torch.nn.init.normal_(layer.weight, 0.0, 0.1)\n",
    "        torch.nn.init.constant_(layer.bias.data, 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultilayerPerceptronZrc(torch.nn.Module):\n",
    "    def __init__(self, num_features, num_classes):\n",
    "        super(MultilayerPerceptronZrc,self).__init__()\n",
    "        \n",
    "        self.f1 = torch.nn.Linear(num_features,HIDDEN_NUMS_1)\n",
    "        self.f2 = torch.nn.Linear(HIDDEN_NUMS_1,HIDDEN_NUMS_2)\n",
    "        self.f_out = torch.nn.Linear(HIDDEN_NUMS_2, num_classes)\n",
    "        self.drop_1 = torch.nn.Dropout(p = DROP_OUT_PROB)\n",
    "        self.drop_2 = torch.nn.Dropout(p = DROP_OUT_PROB)\n",
    "    \n",
    "    def forward(self,x):\n",
    "        x = self.f1(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.drop_1(x)\n",
    "        \n",
    "        x = self.f2(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.drop_2(x)\n",
    "        \n",
    "        logits = self.f_out(x)\n",
    "        probas = F.softmax(logits, dim = 1)\n",
    "        \n",
    "        return logits, probas \n",
    "    \n",
    "    \n",
    "model = MultilayerPerceptronZrc(num_features= num_features, \n",
    "                              num_classes = num_classes)\n",
    "model.apply(init_weights)\n",
    "model = model.to(device)\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr = learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Linear-1                  [-1, 128]         100,480\n",
      "           Dropout-2                  [-1, 128]               0\n",
      "            Linear-3                  [-1, 256]          33,024\n",
      "           Dropout-4                  [-1, 256]               0\n",
      "            Linear-5                   [-1, 10]           2,570\n",
      "================================================================\n",
      "Total params: 136,074\n",
      "Trainable params: 136,074\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.00\n",
      "Forward/backward pass size (MB): 0.01\n",
      "Params size (MB): 0.52\n",
      "Estimated Total Size (MB): 0.53\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "summary(model, (28*28,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, data_loader, optimizer, num_epochs, metric_func, random_seed = 7):\n",
    "    # Manual seed for deterministic data loader\n",
    "    torch.manual_seed(random_seed)\n",
    "    for epoch in range(num_epochs):\n",
    "        # set training mode\n",
    "        model.train() \n",
    "        for batch_idx, (features, targets) in enumerate(data_loader[\"train\"]):\n",
    "            features = features.view(-1, num_features).to(device)\n",
    "            targets = targets.to(device)\n",
    "\n",
    "\n",
    "            ## forward pass\n",
    "            logits, probas = model(features)\n",
    "            loss = F.cross_entropy(logits,targets)\n",
    "\n",
    "            # backward pass\n",
    "            # clear the gradients of all tensors being optimized\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            ### Login\n",
    "            if not batch_idx % 50:\n",
    "                print ('Epoch: {0:03d}/{1:03d} | Batch {2:03d}/{3:03d} | Loss: {4:.2f}'.format(\n",
    "                    epoch+1, num_epochs, batch_idx, \n",
    "                         len(train_dataset)//batch_size, loss))\n",
    "\n",
    "        with torch.set_grad_enabled(False):\n",
    "            print('Epoch: {0:03d}/{1:03d} training accuracy: {2:.2f}'.format(\n",
    "                  epoch+1, num_epochs, \n",
    "                  metric_func(model, data_loader[\"train\"], device)))\n",
    "            \n",
    "            print('Epoch: {0:03d}/{1:03d} training accuracy: {2:.2f}'.format(\n",
    "                  epoch+1, num_epochs, \n",
    "                  metric_func(model, data_loader[\"val\"], device)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 001/003 | Batch 000/937 | Loss: 2.30\n",
      "Epoch: 001/003 | Batch 050/937 | Loss: 1.26\n",
      "Epoch: 001/003 | Batch 100/937 | Loss: 0.87\n",
      "Epoch: 001/003 | Batch 150/937 | Loss: 0.60\n",
      "Epoch: 001/003 | Batch 200/937 | Loss: 0.63\n",
      "Epoch: 001/003 | Batch 250/937 | Loss: 0.61\n",
      "Epoch: 001/003 | Batch 300/937 | Loss: 0.61\n",
      "Epoch: 001/003 | Batch 350/937 | Loss: 0.69\n",
      "Epoch: 001/003 | Batch 400/937 | Loss: 0.61\n",
      "Epoch: 001/003 | Batch 450/937 | Loss: 0.52\n",
      "Epoch: 001/003 | Batch 500/937 | Loss: 0.44\n",
      "Epoch: 001/003 | Batch 550/937 | Loss: 0.42\n",
      "Epoch: 001/003 | Batch 600/937 | Loss: 0.39\n",
      "Epoch: 001/003 | Batch 650/937 | Loss: 0.26\n",
      "Epoch: 001/003 | Batch 700/937 | Loss: 0.27\n",
      "Epoch: 001/003 | Batch 750/937 | Loss: 0.27\n",
      "Epoch: 001/003 | Batch 800/937 | Loss: 0.38\n",
      "Epoch: 001/003 | Batch 850/937 | Loss: 0.24\n",
      "Epoch: 001/003 | Batch 900/937 | Loss: 0.40\n",
      "Epoch: 001/003 training accuracy: 0.93\n",
      "Epoch: 001/003 training accuracy: 0.94\n",
      "Epoch: 002/003 | Batch 000/937 | Loss: 0.40\n",
      "Epoch: 002/003 | Batch 050/937 | Loss: 0.57\n",
      "Epoch: 002/003 | Batch 100/937 | Loss: 0.40\n",
      "Epoch: 002/003 | Batch 150/937 | Loss: 0.33\n",
      "Epoch: 002/003 | Batch 200/937 | Loss: 0.45\n",
      "Epoch: 002/003 | Batch 250/937 | Loss: 0.43\n",
      "Epoch: 002/003 | Batch 300/937 | Loss: 0.34\n",
      "Epoch: 002/003 | Batch 350/937 | Loss: 0.37\n",
      "Epoch: 002/003 | Batch 400/937 | Loss: 0.52\n",
      "Epoch: 002/003 | Batch 450/937 | Loss: 0.33\n",
      "Epoch: 002/003 | Batch 500/937 | Loss: 0.21\n",
      "Epoch: 002/003 | Batch 550/937 | Loss: 0.33\n",
      "Epoch: 002/003 | Batch 600/937 | Loss: 0.42\n",
      "Epoch: 002/003 | Batch 650/937 | Loss: 0.19\n",
      "Epoch: 002/003 | Batch 700/937 | Loss: 0.39\n",
      "Epoch: 002/003 | Batch 750/937 | Loss: 0.33\n",
      "Epoch: 002/003 | Batch 800/937 | Loss: 0.24\n",
      "Epoch: 002/003 | Batch 850/937 | Loss: 0.35\n",
      "Epoch: 002/003 | Batch 900/937 | Loss: 0.26\n",
      "Epoch: 002/003 training accuracy: 0.95\n",
      "Epoch: 002/003 training accuracy: 0.95\n",
      "Epoch: 003/003 | Batch 000/937 | Loss: 0.20\n",
      "Epoch: 003/003 | Batch 050/937 | Loss: 0.18\n",
      "Epoch: 003/003 | Batch 100/937 | Loss: 0.32\n",
      "Epoch: 003/003 | Batch 150/937 | Loss: 0.46\n",
      "Epoch: 003/003 | Batch 200/937 | Loss: 0.33\n",
      "Epoch: 003/003 | Batch 250/937 | Loss: 0.24\n",
      "Epoch: 003/003 | Batch 300/937 | Loss: 0.24\n",
      "Epoch: 003/003 | Batch 350/937 | Loss: 0.31\n",
      "Epoch: 003/003 | Batch 400/937 | Loss: 0.26\n",
      "Epoch: 003/003 | Batch 450/937 | Loss: 0.23\n",
      "Epoch: 003/003 | Batch 500/937 | Loss: 0.23\n",
      "Epoch: 003/003 | Batch 550/937 | Loss: 0.10\n",
      "Epoch: 003/003 | Batch 600/937 | Loss: 0.27\n",
      "Epoch: 003/003 | Batch 650/937 | Loss: 0.15\n",
      "Epoch: 003/003 | Batch 700/937 | Loss: 0.47\n",
      "Epoch: 003/003 | Batch 750/937 | Loss: 0.23\n",
      "Epoch: 003/003 | Batch 800/937 | Loss: 0.24\n",
      "Epoch: 003/003 | Batch 850/937 | Loss: 0.28\n",
      "Epoch: 003/003 | Batch 900/937 | Loss: 0.16\n",
      "Epoch: 003/003 training accuracy: 0.96\n",
      "Epoch: 003/003 training accuracy: 0.96\n"
     ]
    }
   ],
   "source": [
    "train_model(model, data_loader, optimizer, num_epochs, metric_func = compute_accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tryit",
   "language": "python",
   "name": "tryit"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}