Assignment_1_deep_learning_v2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Assignment_1_deep_learning_v2.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sanjeevrs2000/CS6910-Assignment/blob/main/Assignment_1_deep_learning_v2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ed3Fegt5GrCK"
      },
      "source": [
        "#**Deep Learning Assignment 1 - Fashion MNIST dataset**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "75MsIZ4JHGvO"
      },
      "source": [
        "#**Libraries**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sOx8Fw2N6xYA"
      },
      "source": [
        "#!pip install wandb"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZuK0pkO0-NEl"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import keras\n",
        "import math\n",
        "import matplotlib.pyplot as plt\n",
        "from keras.datasets import fashion_mnist\n",
        "import wandb\n",
        "#import tensorflow as tf\n",
        "#import tensorflow_datasets as tdfs\n",
        "from sklearn.metrics import accuracy_score"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bA_oP0S3Hq1J"
      },
      "source": [
        "**Visualizing Dataset**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jv_U2q2O_Ae1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 555
        },
        "outputId": "a5ad401b-b5d3-45d5-ed16-2e7bc2b54e5e"
      },
      "source": [
        "(trainx, trainy), (testx, testy)= fashion_mnist.load_data()\n",
        "classes = np.unique(trainy)\n",
        "index_mat = [0]*10\n",
        "for i in classes:\n",
        "  index_mat[i] = trainy.tolist().index(i)\n",
        "\n",
        "plt.figure(figsize=(10,10))\n",
        "\n",
        "for j, i in enumerate(index_mat):\n",
        "  plt.subplot(3,4,j+1)\n",
        "  plt.imshow(trainx[i], cmap=plt.get_cmap('gray'))\n",
        "  \n",
        "plt.show()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlAAAAIaCAYAAADvIXNPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdebRcVZn38d8DhjkhCRnIBGEIQ6RDhIjMMggCDgiogErTq7Xj0LbgohW0Wb64bIeltkOrraYF48DQ7VJaVAYxIoiMAWIIBJKAGcmcQEIYJGG/f+SCdz97555zqs69t+re72etLPJUzq7aVfXcczd1nnq2hRAEAACA8rbr7QkAAAC0GxZQAAAAFbGAAgAAqIgFFAAAQEUsoAAAACpiAQUAAFBRUwsoMzvNzB43swVmdlldk0L/Qh6hWeQQ6kAeoQprtA+UmW0vaZ6kUyQtlXS/pPNDCI/WNz30deQRmkUOoQ7kEap6TRNjj5C0IITwpCSZ2XWSzpS0zWQzs7bq2rnTTjtF8V577RXF69atS8Y899xzUewXqLkF68477xzFQ4YMieIXXnghGbNy5coo3rJlS3JMbwkhWIXD+1Qe7bLLLslte+yxRxRv3rw5il9++eVkjM+T17ym+Ef1r3/9axT7vBowYEAyxt/vvHnzCh+np1TIoz6VQ7n3euDAgVE8fPjwKPY5JaXnDZ9T22+/fTJmt912i+Jnn302ipctW5aMaeVmzP3pXLTDDjtEsc8ZSRo8eHAU+7xZu3ZtMsb/TvO/F/3vK0kaNGhQFPtzXO5x1qxZk9zWKraVR80soMZIWtIpXirpDU3cX8PM4udW1w/0+PHjo/jb3/52FP/sZz9Lxjz00ENR7H+pvfTSS8mYQw45JIrPOuusKH7iiSeSMV/5ylei+Omnn06OaRMtk0d1mDhxYnLbhRdeGMX+5LFx48ZkjD+xDRs2LIpzOb548eIoPvTQQ6N45MiRyRj/i/jEE09MjmkDvZJD/rwj1XPuGTp0aHLbSSedFMUf+MAHojj38z937two9uci/8tUko4++ugovueee6L405/+dDLm+eefT24r0l3n7Ca19blo9OjRUXzCCSckx5x55plR7M9FP/3pT5MxDz74YBQfdNBBUXzOOeckY04++eQo9ouw3ONMmzYtua3VNbOAKsXMpkqa2t2Pg76NPEKzyCHUgTzCK5qpgTpK0hUhhDd3xJ+SpBDCF7sYU/nB6vg/lcmTJye3nXfeeVGcW0X7y2K77rprFPtLJFJ6uaYR/jJK7hLPgQceGMX+kt4tt9ySjPnqV78axXPmzGl0il2q8rF5T+VRT/nEJz6R3HbGGWdEsX8/99lnn2SM//jdfwKVu3z8zDPPRLH/VCL3sfn+++9fOJfeUjaPWvlc5N+3iy66KDnmTW96UxTvuOOOyTGbNm3q8hj/qYCUv4TTWe7T8KVLl0bx8uXLozh3zvO5eMcdd0Txt771rWTM+vXru5xbXfrKuej000+P4o9//OPJMf6TQH9JT0ov6/oc8VdDpPST64ULF0Zx7vKxzxt/bsrl+JgxY6J4xowZUfyxj30sGdNTtpVHzXwL735JE8xsHzPbQdJ5km5o4v7QP5FHaBY5hDqQR6ik4Ut4IYTNZvZRSbdI2l7SVSGER2qbGfoF8gjNIodQB/IIVTVVAxVCuFHSjTXNBf0UeYRmkUOoA3mEKuhEDgAAUFHDReQNPVg3Fdz5nhM//vGPo3jSpEnJmO22i9eOua+S+4I7X3SZ673ke+3svvvuUeyLQaW0qLiR98T35sgVe/qiwj/+8Y/JMRdccEHlx/Yq9l6prJWLyK+44orktnHjxkWx/6JB7mvrua/Id+bf79yYMkXkxx57bBQfc8wxUewLRntSd+ZRdxWR77ffflH8q1/9Kor9lz2k4vOMlJ5rXnzxxSjOfanA93Qqug8pPUf4Nhe5HlV+jI/9V9gl6Xvf+14UX3/99ckxdWjXc5HPI39eyeWR70Hnf8dJ6e8aXwDuz1U5/j5yX3LyReP+cXI57nPYF5XnWnX867/+a9eTrUl3FJEDAAD0SyygAAAAKmIBBQAAUFGfqIH63e9+F8V77713FOdqP/x129y1fX/dtqguRUqvO/vtE3L7TxXdRyPKbDMxatSo5Jg3v/nNUfzYY49Vfux2rTuowzXXXJPc5utIfG2Kb9AqpfUMq1evjuJcIzpff+cb6/l9zaS0cZ6vKfD1hD2p1Wqgyvjf//3fKC7TANW/b7lzsq8Z8eevXD2Tv83XWuVyyNds+rk1cg7MNXT09/uOd7wjOSaXr1W167nov/7rv6LYv3e5uiN/XsnVSfrfab4+LdcU09cz+fvNzSWXW53l6of9Y/vnnGvy6c9Pv/nNb7p83EZRAwUAAFATFlAAAAAVsYACAACoqKlO5L3h8MMPT27zNU9r1qyJ4lx9k69Fyl0v9n0oyvTZ8LUK/rFz1359XYGvD8hdl/Z9q/wmoLkxXm4uH/jAB6K4p/ps9BW+5kVKN+z0NU++7kRKa2V8vuZyL1dL1VmuLsHf75AhQ7q8D/xNroZwzz33jGJfP5KrB/I/q/48I6XvrX//c3Uo/ufbx7lznn8cPyZ3XvHH+NolX8uSe5y3ve1tyTHXXnttclt/MX369Cj2mwf7mkgp7Q2V20w613+pM1+zK+XPaZ1t2LAhuc3XX5bhH9ufF5csWZKM6a6ap7L4BAoAAKAiFlAAAAAVsYACAACoqKkaKDNbKGmjpC2SNocQptQxKfQv5BHqQB6hWeQQqqijiPzEEMKa4sPqceKJJya3+eJYH+cKLH3xbK4R3aWXXhrFTz31VBT7wm1JGj16dBQvX748inPFv754zs/fN0iTpMMOOyyK/+Vf/iWKfSG9lBa0516Xd77znVHcg0XkPZpH3SVXEO5zwBfdvva1r03G+GLuXCGuV9SANbepq/8Cw8SJEwsfp8X1WB7lCu59Ebl/r3NF5L6gOleoXXROyzW4LGp6mWvq68eUeRz/HH3j2Ny5yL8Op5xySnJMLxaR9/q56L777oviu+++O4rf/va3J2PuvffeKM59ecp/QcE3mc4Vkfv3z5+Lcl968I/tC819juT4+73ssssKx/Q0LuEBAABU1OwCKkj6rZk9YGZT65gQ+iXyCHUgj9AscgilNXsJ79gQwjIzGyHpVjN7LIRwR+cDOpKQRERXyCPUocs8IodQAucilFbbZsJmdoWkZ0MIX+3imKYf7J577kluGzFiRBT7JpO567q+rsg3vJOkI488MopPPfXUKPaNNiXphz/8YRR/8IMfjOI5c+YkY3beeeco9rUJvkGaJM2aNSuK58+fH8X+NZDSxnm5WouDDjooiv0GjvPmzUvGeM1s4NlTeVQXX5vy0EMPJcf4+gXfzC7XqG7w4MFRPHbs2CjONc30dQa+oeGqVauSMRMmTIhi//7mai16SnfmUR05dN555yW3ff7zn49if+7J1an523L1br7+8oknnojihQsXJmM2bdrU5f36f5fS3PS1SpMmTUrGvPWtb+3ycXK1OP78++CDDybHnHvuucltVfXVc5F//yXp9ttvj+Jcs01f0+bPEbnfG57//ZRrzunfc98cOlc35etHb7vttij+1a9+VTi37lL7ZsJmtquZDXzl75JOlZSuDoAukEeoA3mEZpFDqKqZS3gjJV3f8a2M10i6JoRwcy2zQn9CHqEO5BGaRQ6hkoYXUCGEJyUdWuNc0A+RR6gDeYRmkUOoqu02Ez700DS//SaDvqYgt4mqN2jQoMJjbr45/p+RXA2B76Pj+yhdf/31yRi/kaa/fpyrD/CbKvt6plyNjO/XkusDtXjx4ig+6qijorhMDVR/MnTo0Cj2NQVS2kfF1zzl8tO/f/698nVzknTXXXd1OSZX8+brVYp6B+FvrrvuuuS2P/7xj1H83ve+N4p9TaEkfeELX4jixx57rPJccjUlPkd8nDtH+DpJf47L9Wb61Kc+FcX3339/FI8cOTIZ43uS7bvvvskx/Zn/HeB/do899thkjK+/y/Gvu7/f3HnFbwzs55arcfN9FYt61OWO6c2ap7LoAwUAAFARCygAAICKWEABAABUxAIKAACgopYvIvdFl7nmYL4Qzjf6yhXG+mI5v6limbnkNiAeNWpUFPvCvtxcfCMyf4wv5M7xjfZyTT7LFJH7gsHjjjsuin/0ox8VzqU/8RvK5nLCv86+OWFujM9hv+HwsmXLkjF77bVXFPvmirkGjb75Zq4pHvK+/OUvJ7f599o3A8w1WvVfYMkVkftzgn/fcuevp59+Oor9e5trouwfxzc3zG187Zs6+sL53Bcr/HxzPwP9We4LH535Dcql9H3YZ599kmP8OcA3zsz9TvBjfLF37v31mwX755MrKl+0aFFyW6vjEygAAICKWEABAABUxAIKAACgopavgbr00kujONfoy1+D9bU+uTH+um7umvOUKVOieI899ohi30RRSjdN9E3kcjUmfi6+RsZvLCulG236WhxfyySl9Qy5Y/xj+9cAMd940Deqy/E5MnDgwOQY33zT16v4+hYpzaO99947inN1Mj7v/dywbbfcckty28knnxzF55xzThT7DcmltK7wwx/+cHKMPwfsv//+Uew355XSnPF1df5nXUo3P/Y1MT/96U+TMb6Oxp+zc5u5r1+/PorPPvvs5Jijjz46itetW5ccg7/xdUW584p/P30TX19bJ6V54s8zuffXK6rpkvKbnbc6PoECAACoiAUUAABARSygAAAAKipcQJnZVWa2yszmdLptqJndambzO/47pKv7AMgj1IE8QrPIIdSlTBH5dEnflvTjTrddJmlGCOFLZnZZR3xpZmzT/A7ze+65Z3KML6j0jelyu47Pnz8/in3huSTdc889UewL8HJNx/z9+MLN3M7Vvnmdv49c0zFfuDlv3rwozu3O7ueSu1/fkPP//u//kmMaNF29mEfdxedArjDf86/7M888kxxz8MEHd3kfvghXSr9M4XPcN9qU0iJSn1ctaLpaJI++9KUvJbf5L4n4n6e5c+cmY972trdF8Wc+85nCx/aPk2tE6c8jvqg8V9jrzxH+SwW5YnWfi/fdd18Ur1ixIhnjG4z6XJW6tWh8ulokh8ry54zc756lS5dG8aRJkwrvx+dNrrmqzwGfV/6LNFJ6HvSF58OGDUvG5JoDd5b73VmmOL07FX4CFUK4Q5LP5DMlvfLVkR9JekfN80IfQx6hDuQRmkUOoS6NtjEYGUJ4pZf8Ckkjt3WgmU2VNLXBx0HfRh6hDqXyiBxCFzgXobKm+0CFEIKZpZ/7/e3fp0maJkldHYf+jTxCHbrKI3IIZXAuQlmNLqBWmtmoEMJyMxslqds6YH33u9/tMpbSJpITJkyI4lxjuje+8Y1RnLvePmfOnCj2zQtzTQd9DUEjfE1UrlbJX1P2TTJnz56djPGbfLaAHsuj7lKmNsEramYn5ZvgdeY3DpWkQw89NIp9XdymTZuSMT5vcrWAbaBX8ugXv/hFcptvpOkb0d50003JmBtuuCGKR4wYkRyzePHiKC6qVZLS2pRcDYnna0p8Y9hc00Rfc+obuF588cXJGH/MCSeckBzjN16eNWtWOuH6tP25yG8envu94Zti+t+d/j6kNCd8Q+lcPaYf42utcnPr7XqmRjTaxuAGSRd2/P1CSb+sZzroZ8gj1IE8QrPIIVRWpo3BtZLulnSgmS01s/dL+pKkU8xsvqQ3dcTANpFHqAN5hGaRQ6hL4We6IYTzt/FPJ2/jdiBBHqEO5BGaRQ6hLi2/mXAZRX1Icj1STjrppCjO9b/w14t9P6lcvVNRDYyvb8rdVqZGxtci+HoH3z8L3cO/V7nr+L6OxPdIyfVEKdqU2Nc3Senmq75ObuXKlcmY0aNHR3EdNXz9xcSJE5Pb/HvreyD53nKSdMwxx0TxIYcckhxTtDFwjs9Nfx9lzkX+cXLnN/8cr7nmmijO1S49+eSTUbxkyZLkmFyOY9t87jVSj5nLK/+7xY/J1UD5c1pRTafUnhuZs5ULAABARSygAAAAKmIBBQAAUBELKAAAgIrarog8V/joi898gXWuQHzDhg1RnCueK9qMs8z8yoxpRFERqW/6WfY+igpP0bXc6+UbGPri7lzxZNH798gjjxTOxRdy5n52Vq9eHcW83+Xtu+++yW3+vR47dmwU5zbW9V8YyH0RwW/y7BsRltkYuJEmqf6LM34TY0kaPnx4FPvnkysg9q/L4MGDk2P8xvG+8Lw/KVMQ7nPA/2xL6e/GXAG454/x97HzzjsnY1ativuQ+hzxG5+3Kz6BAgAAqIgFFAAAQEUsoAAAACpquxqoXI1G7rp8Z7mNV30NVG6jzdzGmUVzaaQGKlebUjSPoqZj/vnl5DZ0bNPNZHuNfw1981UpbUTn389cfUNRjcDMmTML51KmCaJv0lrUwBN/U2aTb//z5GuZJGmXXXaJ4tz75N9LH+fOIX5+Ps6N8Y9dJr/9XNasWZMc4w0dOjSKc+df3+S1P9dAldm03Nea+Y2CpfTn278POf799PnqNySXin935nLPbzDtteJmw3wCBQAAUBELKAAAgIoKF1BmdpWZrTKzOZ1uu8LMlpnZrI4/Z3TvNNHuyCM0ixxCHcgj1KVMDdR0Sd+W9GN3+9dDCF+tfUYN8NeHfd2B32RRSq/R5jbs9ddc/XX63HXcog07y9Qq+PvIbYbsr0P7+23B68XT1eJ51IgydSU+b3xtQm7Mo48+2uXjlunz1cgGtC3eB2q6WiiHcjVQRRuDr1u3Lhnj++jk6luKzhE5/pgymwn7elJ/XszVKvm5+V5Xvi5MSs/RudwsswFtg6arhfKojDJ9oHzfpzlz5iTH+E2b/e+R3Hs1cuTIKPa/OxcuXJiM8ffj66SWL1+ejPE1b+2g8BOoEMIdktKfeqAC8gjNIodQB/IIdWmmBuqjZja74+PQtNwfKIc8QrPIIdSBPEIljS6gvitpP0mTJS2X9B/bOtDMpprZTDNLv3eN/o48QrPIIdSBPEJlDS2gQggrQwhbQggvS/pvSUd0cey0EMKUEMKURieJvok8QrPIIdSBPEIjGmqkaWajQgivVIGdJSmtVutBRQWVuQK8MhsF+9tyRaNFj1WmcNcXc5YpGC1qeNdIkWlPa7U8aoRvLJjbWNM3ovPFkrkvMPhiTy/XkLHoSw+5/PVjihrgtZpWy6Gi5qUrV65MxuRypkhRsbpU/P6XKYIvU+ztlcmhoi/+lH2surRaHjXiuOOOi+Jc49FFixZFsS/2zjVgHjRoUBT7gvAyX9IaNWpUZsYxv3n0iBEjothvUCyVazDanQoXUGZ2raQTJA0zs6WS/p+kE8xssqQgaaGkD3bjHNEHkEdoFjmEOpBHqEvhAiqEcH7m5iu7YS7ow8gjNIscQh3II9SFTuQAAAAVtd1mwnUZM2ZMFK9fvz45xl+DL1MTVbQxcCNyj+Mb3vnH7cn6AfzNsGHDktv8Bp6+biq3MfSCBQsqP7avi/KPk6tV8Bsdb9q0qfLj9leNbBSeO8/4979MzWOZxrlFdZFl5u/vt8w5z9d05Zq++rzLKXNMX1VU2zNu3LhkzMSJE6M4VwM1ePDgKPbnq9x5Z9ddd43iffbZJ4pz76+vmyrDb6D+nve8J4q/8Y1vJGN6uubJ4xMoAACAilhAAQAAVMQCCgAAoKI+UQPVSD+jMpvt+hoS36skV+9UtHlwbq5FPV1yNTJ+g2F/v7kxXm/3geoLfE74zTklaezYsVHs3xufZ5L0+OOPV56L36jW1zv4GgOpsboY1MvX+uTqOop6xZWpvaxjA+Jcjyc/F18DlaurmTx5cuH9dkc9absoqu1585vfnNzmNyDP1ZD5Pk/jx4+P4mXLliVjDjrooC7ntnTp0mTMpEmTotj3P9tjjz2SMb4+0Ncp77///smYRmpF68QnUAAAABWxgAIAAKiIBRQAAEBFLKAAAAAq6hNF5I3wRdi5xpO+0Lxoo1ApLbr0Y3LFkn6M3wQ0V/zpmzN6voAYvcc3ovNyxbK5hotFfDHnwQcfHMU+56W0oL3dNhPuTbkNnf17XWYDcl90XeYcUaaBoB9T9AUXqbh5sG/gm7sf/5wXL16cjJkyZUoU53KTZsDb5ou0JWn27NlRnHv9/BdWchuZe0XvQy4X/W1+0+JcI1Bf4F5U8C5RRA4AANB2WEABAABUVLiAMrNxZnabmT1qZo+Y2UUdtw81s1vNbH7Hf4d0/3TRrsgjNIscQh3II9SlTA3UZkmXhBAeNLOBkh4ws1sl/YOkGSGEL5nZZZIuk3Rp9021Xo1sQlimKaZXR8O73Bh/jK/X8nUVZR6nm/XJPPJyNS++uaaPczUvjdRArVq1Kop9A7xcXZy/LddIr4X0ag75+pHcz49//30dR46vQ8vVGXn+sXPNWMs0/vV8/aW/jzJ1n/4+Fi5cmIzxz9k/Tu6YGrXducjX/yxfvjw5xjfOzDXO9e9NI783/JhcThTVVuVqeEeOHBnF/lw0fPjwwrn1tMJPoEIIy0MID3b8faOkuZLGSDpT0o86DvuRpHd01yTR/sgjNIscQh3II9SlUg2UmY2X9DpJ90oaGUJ4ZRm8QtLIbQwDIuQRmkUOoQ7kEZpRuo2Bme0m6eeSLg4hbOj8cXAIIZhZ9nqQmU2VNLXZiaJvII/QLHIIdSCP0KxSCygzG6CtiXZ1COEXHTevNLNRIYTlZjZK0qrc2BDCNEnTOu6nZXYqLdOfxWukZqiOGqjcXItqoHKb2va2vphHXm6Tav/++fcm10+okX5Ma9eu7fI+cnPztTNlNtnuTb2ZQ2U2XvY1JmVqyop6L0nFNZuNbGyeu09fi1Rmbn7MwIEDo3jevHnJGP86ldlAuU7tdi7aa6+9ojj3evnXNFcX5+uk/Hvn7yNnyJC4tj53zvD34+O//OUvyZgJEyZEsd+AePfdd0/GDB06NIr9hurdrcy38EzSlZLmhhC+1umfbpB0YcffL5T0y/qnh76CPEKzyCHUgTxCXcp8AnWMpAskPWxmszpu+7SkL0n6XzN7v6RFkt7dPVNEH0EeoVnkEOpAHqEWhQuoEMKdkrb1WerJ9U4HfRV5hGaRQ6gDeYS60IkcAACgoj6xmXAdDSEb2bgy97hFhY9lHqeRBp1FxZ/oHr4oc9OmTckxPid8EflTTz1Vy1x8w0LfiNBv6JlTpokjtirTSLNMEbkfk7tf/176Mbmf90YKz/1jN9KM0xf7PvLII8kxfv65L8p0ZxF5u/Hvb+718s0pc18kKto8vEyj1N122y2Kc0XkfnPoMWPGRPHMmTOTMccff3wU+2ahuQJ3X9DeckXkAAAAiLGAAgAAqIgFFAAAQEV9ogaqkU1+/bXfRhpP5q4X+2vV/vpwmbqDRjRSA9XDmwn3Sb6moEyzSh8//fTTtczFbyZcpvGjn38jm2z3V2VqoBYvXlx4P75eZPXq1ckxvtlqmYanRfVLZeqOfJzbJNbXAe66665RnKsDK9PUs0xTx/5i2LBhUZxrkunz5pBDDkmO8e+V3+w6d78+13yj1NwYX285adKkKP7Nb36TjPHnQX+/vt5J6v0c4RMoAACAilhAAQAAVMQCCgAAoCIWUAAAABVRpdchV1DZSBFmUYO4RnYdL1Os6tFIs2eUKSL3/Hv3/PPPF44p80UJX7jpvyjh81lKi0jLNNvsr4oKrHP865vjC7Nzhdq+wanfhT733pb5AotXdI7LPR9fND569OgozuWULxDOFQPnipP7K19Enjv/r127Nop9Q1MpfZ19s8rca75+/foo9s2Ci34X5Tz77LOFj+N/V+aaFI8aNSqKH3/88cpzaQafQAEAAFTEAgoAAKCiwgWUmY0zs9vM7FEze8TMLuq4/QozW2Zmszr+nNH900W7Io/QLHIIdSCPUJcyNVCbJV0SQnjQzAZKesDMbu34t6+HEL7afdMrp5GGkH4D1wMOOCA5xtcQ+GuyuXqmosaEZTZr9PUMZZqF+ftowUaaLZ9HdfDX8XP8tfwyNVBFm0dL0po1a6K4KH+lNAdavAaqV3PI/0z5GjMpfc3L1If8/Oc/j+JBgwYlx/gmqf6cUKb2zo/J1UQVNbjMPc4zzzwTxbmNYj1/P7n7baS2pqS2Oxf5DXz9xsFSvtGk5xtp+hzO/a4ZPnx4FPuGnb4GLjfG13Dtt99+yRifa2Xqh31Tz55W+Js5hLBc0vKOv280s7mSxnQ9CoiRR2gWOYQ6kEeoS6UlvpmNl/Q6Sfd23PRRM5ttZleZWXb5a2ZTzWymmRX/bwn6BfIIzSKHUAfyCM0ovYAys90k/VzSxSGEDZK+K2k/SZO1dTX/H7lxIYRpIYQpIYQpNcwXbY48QrPIIdSBPEKzSvWBMrMB2ppoV4cQfiFJIYSVnf79vyX9ultm2E0GDx4cxbnruP56cJleHP42XxNVRpmNgZcsWRLFfjPk3DVmLzf/7txMti/mkb/W72Mp7c/i6xDK1B2VqYHydSS+n1Cu5s33ffG1Fq2mN3No55139nNJjvHvkz/P5Hzxi19sbmJtyOdi7lxU5rVrVLudiyZMmBDFf/nLX5Jj/Hklx7/O/vdG7lx01113RfF73vOeKM7VTc2YMaPLxy3zfvta0dxzvu2225LbelKZb+GZpCslzQ0hfK3T7Z07WJ0laU7900NfQR6hWeQQ6kAeoS5lPoE6RtIFkh42s1kdt31a0vlmNllSkLRQ0ge7ZYboK8gjNIscQh3II9SizLfw7pSU2wPgxvqng76KPEKzyCHUgTxCXehEDgAAUFGf2Ey4zEar3kMPPRTFjz76aHLM008/HcVlCsJ9cZzfNDE3Nz//Mg0QfQM030TtvvvuK5xrdxaM9xezZ8+O4l/96lfJMT5v1q1bF8VlCiHLvFcrVqyI4vnz50dxrtGeb9A4Zw5lH9vi37d58+YlxyxdujSK77333uQYr8wmvz3c9LbbXX311VG87777Jsc8+OCDPTWdlveRj3wkiss0Hv2f//mf5Bj/5aJFixZF8dixY5MxCxcujOIyjVI93yw252c/+1nl++1tfAIFAABQEQsoAACAilhAAQAAVGQ9eW3dzFZLWiRpmKQ1BYe3itPucFIAACAASURBVHaaq9T78907hJB2k6wRedTtWmGu3ZpHbZpDUnvNt7fnyrkoj7lWs8086tEF1KsPajazXdrgt9NcpfabbzPa6bky19bUbs+1nebbTnNtVjs9V+ZaHy7hAQAAVMQCCgAAoKLeWkBN66XHbUQ7zVVqv/k2o52eK3NtTe32XNtpvu0012a103NlrjXplRooAACAdsYlPAAAgIp6fAFlZqeZ2eNmtsDMLuvpx++KmV1lZqvMbE6n24aa2a1mNr/jv+l+GL3AzMaZ2W1m9qiZPWJmF3Xc3pLzrVMr55BEHrWLVs4jcqg9tHIOSeRRd+vRBZSZbS/pO5JOlzRR0vlmNrEn51BguqTT3G2XSZoRQpggaUZH3Ao2S7okhDBR0pGS/rnjtWzV+daiDXJIIo9aXhvk0XSRQy2tDXJIIo+6Vwihx/5IOkrSLZ3iT0n6VE/OocQcx0ua0yl+XNKojr+PkvR4b89xG/P+paRT2mW+TTzPls+hjnmRRy38px3yiBxq7T/tkEMd8yKPuulPT1/CGyNpSad4acdtrWxkCGF5x99XSBrZm5PJMbPxkl4n6V61wXyb1I45JLXB+0IetXwetfx7Qg61fA5JbfC+tEseUUReQdi6BG6pry2a2W6Sfi7p4hDChs7/1orzRWu+L+RRe2nF94Qcaj+t+L60Ux719AJqmaRxneKxHbe1spVmNkqSOv67qpfn8yozG6CtiXZ1COEXHTe37Hxr0o45JLXw+0IeSWqPPGrZ94QcktQeOSS18PvSbnnU0wuo+yVNMLN9zGwHSedJuqGH51DVDZIu7Pj7hdp6XbbXmZlJulLS3BDC1zr9U0vOt0btmENSi74v5FFb5VFLvifkUFvlkNSi70tb5lEvFIadIWmepCck/VtvF4G5uV0rabmkl7T1evb7Je2hrZX/8yX9TtLQ3p5nx1yP1daPMmdLmtXx54xWnW9/ySHyqH3+tHIekUPt8aeVc4g86v4/dCIHAACoiCJyAACAilhAAQAAVMQCCgAAoCIWUAAAABWxgAIAAKiIBRQAAEBFLKAAAAAqYgEFAABQEQsoAACAilhAAQAAVMQCCgAAoCIWUAAAABWxgAIAAKiIBRQAAEBFLKAAAAAqYgEFAABQEQsoAACAilhAAQAAVMQCCgAAoCIWUAAAABWxgAIAAKiIBRQAAEBFLKAAAAAqYgEFAABQEQsoAACAilhAAQAAVMQCCgAAoCIWUAAAABWxgAIAAKiIBRQAAEBFLKAAAAAqYgEFAABQEQsoAACAilhAAQAAVMQCCgAAoCIWUAAAABWxgAIAAKiIBRQAAEBFLKAAAAAqYgEFAABQEQsoAACAilhAAQAAVMQCCgAAoCIWUAAAABWxgAIAAKiIBRQAAEBFLKAAAAAqYgEFAABQEQsoAACAilhAAQAAVMQCCgAAoCIWUAAAABWxgAIAAKioqQWUmZ1mZo+b2QIzu6yuSaF/IY/QLHIIdSCPUIWFEBobaLa9pHmSTpG0VNL9ks4PITxa3/TQ15FHaBY5hDqQR6jqNU2MPULSghDCk5JkZtdJOlPSNpPNzBpbrfWS7baLP6Dbeeedo3innXZKxjz//PNR/PLLL0fxDjvskIx56aWXuryPdhNCsAqH9/k8QmMq5FFb55BZ/DRz5wh/rtm0aVMUb968uf6JZeaSO+dt2LChWx67DpyLUIdt5VEzC6gxkpZ0ipdKekMT99dt/EJIShc2ObvssksUv/a1r43iiRMnJmMefvjhKH7hhReiePTo0cmYlStXRvGf//znwrn5k26jnyS2gLbJI7SsHskhfx7JnUPKHOMNGDAgiseNG5cc48899957bxSvWLGi8HEaMWrUqCjOnfNuvvnmKG7kXNToObpmnItQSTMLqFLMbKqkqd39OOjbyCM0ixxCHcgjvKKZBdQySZ3/V2lsx22REMI0SdOk3vu4s8z/yRx44IHJbQMHDoziAw44IIoPPfTQZIz/OHvdunVRPHjw4GSM/1jcf7o0a9asZEwbf+LktU0eoWX1SA75n7lGPjX5/ve/n9y24447RvGLL76YHDNy5Mgo/tjHPtbl3KT08ttDDz0Uxb4kQUrLCfwnXxs3bkzGnHbaaVHsz3E33HBDMubnP/95FNf1aV6TOBehkma+hXe/pAlmto+Z7SDpPEnpTwrQNfIIzSKHUAfyCJU0/AlUCGGzmX1U0i2Stpd0VQjhkdpmhn6BPEKzyCHUgTxCVU3VQIUQbpR0Y01zQT9FHqFZ5BDqQB6hCjqRAwAAVNTt38JrVfvtt18Ujx07Njlm0aJFUey/0uuLP6W0JcHChQujOFcIuXbt2ij2RZhTpkxJxsycOTO5De3Jf2lAKi6gLfMlgtz9enV8GeHoo49Obrvrrrui2H9JY968ed0yl+7kX88yRc1f/OIXo3jIkCHJMU899VQU5/pALVmyJIp33333KPbnJkm69tpro/h73/teFN99993JGH/+8nNbs2ZNMuY1r4l/jTz33HNR/O53vzsZs9dee0Xx17/+9eSYMvkL9CY+gQIAAKiIBRQAAEBFLKAAAAAq6rc1UL7OKLcVgm9o5+sQLrjggmTMWWedFcW/+c1vovh3v/tdMmbu3LlR7OsQ9t5772SMb4LX7vvnoWuN1AfVUVN0wgknJLf93d/9XRRPmDAhOeYLX/hCFPt6llNPPTUZk2sg2UrKNHbcd999o/iQQw6J4sWLFydjfC1l7n3zj7VsWdzfMVeP6c8b73rXu6LY1ypJ0urVq6PYN87cfvvtC+e2ZcuWKPZ1VFL6uuTu19+PP8b/O9DT+AQKAACgIhZQAAAAFbGAAgAAqKhP1kD5WgVflyBJu+22WxRPnjw5OcbXPPlr+b6XlJRuxul7uowZMyYZ4/vo+B4pucdZunRpFPueL/7f0TNyvWuKapFy/95Ifcff//3fR/E999wTxccdd1wyxm9K63N80qRJyZj58+dH8YMPPpgcc/HFF0dxbkPsdrN58+bCY04++eQo9vVBu+66azLmhRdeiGLfVynHn7+WL1+eHDNs2LAoftvb3hbFfnNhKd1A3dda5uq+/DnPn39zPxP+vJjLzT/84Q+F9wP0Jj6BAgAAqIgFFAAAQEUsoAAAACpqqgbKzBZK2ihpi6TNIYR00zagAHmEOpBHaBY5hCrqKCI/MYSQ7jDZi3zR+Lhx45JjfOPJBQsWJMf4Atr77rsvin3DS0kaP358FB9//PFRfP/99ydjjjjiiCj2xeu///3vkzG+yPiYY46J4scffzwZ0+KFvC2XR63koIMOiuJcobFveuk3oc5tZDt9+vQovuOOO6I4VyB++OGHR/HrX//65Ji//vWvUbz//vtHce7nrSa9mkcTJ06MYl/4nCsi969VmS8i+GLuAQMGJGN8Y9JNmzZFcW7TYj/G32/uyw2+CN5vdLzTTjslY/zz8Y01pbSIvEwRf004F6EULuEBAABU1OwCKkj6rZk9YGZTcweY2VQzm2lmM5t8LPRd5BHq0GUekUMogXMRSmv2Et6xIYRlZjZC0q1m9lgIIboGEEKYJmmaJJlZ85tzoS8ij1CHLvOIHEIJnItQWlMLqBDCso7/rjKz6yUdIemOrkd1P79R8KpVqwqPyTUz/O1vfxvFGzZsiGLfmE6Sbrnllij2TeVmzJiRjCnaNHOPPfZIxvh6Bl+rMGrUqGSMrzt59tlnk2N6Q6vmUSMa2cB3l112SW7zzVX9Ztc+FyXpyiuvjOKPf/zjUZzb1PXrX/96FI8YMSKKc8/H19f5mihJOuWUU6LY18l0Rw1UK+SRb3rr63ZytUq+WaV/raS0WaU/Z+Tqpvx5pKjJb+5+/fxzdUh+I2Nfn+WfX26+w4cPT47pDa2QQ2gfDV/CM7NdzWzgK3+XdKqkOXVNDP0DeYQ6kEdoFjmEqpr5BGqkpOs7/k/iNZKuCSHcXMus0J+QR6gDeYRmkUOopOEFVAjhSUmH1jgX9EPkEepAHqFZ5BCq6hObCftr7P7afu66va8hytWh+Ovyvp/JokWLkjG+xuHee++N4lwdiu8d4+fr66iktIbA9wXKjRk7dmwUP/bYY8kxaI6vO5HSmhBfV+Q3hpXSOhjfJ8f3fJKkD37wg1F82mmnRbGvz8vJ1Qt6vk5q3bp1yTF+0+x//Md/jOI//elPyZg5c9rrakmunsnXFfrNeX0dkpS+Vr4PnJTmg//5zuWd52uVcvy5M7d5cNXHGTp0aHKMf465Dd+BVkcfKAAAgIpYQAEAAFTEAgoAAKAiFlAAAAAV9Yki8mHDhkWxL7DONabzBeG5QlhfDOmL1X0zTkn6wAc+0OX9jhw5Mhnj5+s39MxtHOsLzX2hpt+gNPfYFJHXL1d0W9Rc029sLaVFwieddFIU//SnP03GfOhDHyozxab5xq6DBg1Kjpk5M97lwud0rqC58/0+/fTTzUyxR+Sa1fovo5T5woD/2c1tBO7zoUwRuc9Ff0wuL3MNObu6Tyl9bw877LAo9l/YkdIC/Ny5FM0pei+lNAca+RJMmd9PZficbuQLDDk+1/zcGml+/Ao+gQIAAKiIBRQAAEBFLKAAAAAq6hM1UEW1Srm6A19jseuuuybH+I01fS3Vc889l4x5+9vfHsW33357FC9cuDAZ46//+2vKuevSvtbC12PMmjUrGbPnnnsmt6FejVxP37hxY3LbHXfc0WWcU7QpbZm5+bqJ3Bifa7n6Qf+cbrrppigePXp0Mmbvvfd+9e+5urBW42t9pLTewr+euY11/c93rn7E36+vD2mk9i7Hj/H3m6urKdqAePfdd0/G+M2x165dmxwzfvz4KM6dO7Ftjbz/ufe36H4aqXf68Ic/nNx2+eWXR7FvMNuoXPPauvAJFAAAQEUsoAAAACoqXECZ2VVmtsrM5nS6baiZ3Wpm8zv+O6R7p4l2Rx6hDuQRmkUOoS5lPoGaLuk0d9tlkmaEECZImtERA12ZLvIIzZsu8gjNmS5yCDUoLCIPIdxhZuPdzWdKOqHj7z+S9AdJl9Y4r0p8EXbRjui5Y3wzOClttunlCkJnzJgRxX7X8dx9+mJff0yuKaYvnPcF7WWeTyMFg41qhzxqJb6w2Bfz+qZzOf4YX+zbqOHDh0ex/1mS0tzyzyf3xY4yxaitlEeNNMXNfZFjw4YNUewLxqW0ENa/nrmfZf/++5/tXD74Y/zj5h7Hz9c/53333TcZM2/evML7nTx5chTXVUTeSjnU04q+JNJIQfj555+f3Pa6170uit/1rndFce5LImvWrInia6+9tvBxythhhx2i+JOf/GQU//u//3tD9ys1XgM1MoSwvOPvKySlZxKgGHmEOpBHaBY5hMqabmMQQghmts2PLcxsqqSpzT4O+jbyCHXoKo/IIZTBuQhlNfoJ1EozGyVJHf9dta0DQwjTQghTQghTGnws9F3kEepQKo/IIXSBcxEqa/QTqBskXSjpSx3//WVtMyrgG0hKaY2Tv26fuwbva4Zym5cW1QPlahV8A8Eym37623wjzVyTPF9/5TdUzs3dv3Z+U1gpvQ7dzXotj1pdUb1S7t99XUEu17wyjTM933T2wgsvTI759a9/HcXXXHNNFOfqpjr/TFbcSLRX8mi//fZLbvPnBF/fmPuZ8/VAueeeO9d0lquJK9ootsxms0UbEkvpe+mPyY3xc8vN/8ADDyycX43a/lxU5me56Od7//33T27z9UtHH310FJ966qnJmCeeeCKKly5dGsW+7k9KG6eeccYZXc61rPPOOy+K3/CGN9Ryv1K5NgbXSrpb0oFmttTM3q+tSXaKmc2X9KaOGNgm8gh1II/QLHIIdSnzLbxtlb6fXPNc0IeRR6gDeYRmkUOoC53IAQAAKmq7zYRz13D9dXpfozFo0KBkTK5PUhFfm5Sbi69NKrMpqu+J469l5zZDPOCAA6LYb7yYq5nwdV+5PjY9XAPVVnqyb1YdfJ1UmZqoMr2ifI489NBDyTFTpsT1td///vejOFc/dNddd73691Z+XV/hN1WW0l5rvrYyV8Pp66T8eUYqfj3K9AXz+dtIz5/cedP32Vm/fn0U585Ffr65zdxzr29f4J97rubNv6a5XoBemZ8Z3zPx85//fBSfe+65yRj/e2P58uVRfN999yVj/Hvufy8+9thjyZixY8dG8ec+97nkGG/EiBFRnJv/1772tSg+6KCDovjwww9PxjzwwAOFjy3xCRQAAEBlLKAAAAAqYgEFAABQEQsoAACAitquiDxXkLhp06Yuj/FFmpK0du3aKM41uPNFeb64M1dU7JvK+SLyMhuF5opIPV906Qt7c41BfYFrbjNkbFs7FDZ3pZHNhP2GrpL05z//OYqvu+665Ji3vvWtUfzmN785in2RrBRvvJ374kSryZ0zihpe5gq3y3zRxBceF8U5/nyVm6svaPbnztz75n8uck1SPf/YuS/6jB49uvB+2oF/3cs0MC1TNO6dfHLcheGcc85JjnnPe94Txf734KOPPpqM8Tnr36vcz4HPaV+I7r9kIkkrVqzocq6f+MQnCh/n4YcfTo7Zcccdo9j/HvTNr6vgEygAAICKWEABAABUxAIKAACgorargco1oitqcJm7Ruuvi+bqW4oaz+UaE/pahKFDh0axr9eS0uZ0/jnmHsfPxTfF3HPPPZMx/nq3vxaMvsXnTZkaqEsvvTSKff5K0ne/+90ovuCCC5JjfK7deOONUbz33nsnYxqp++hNuRpCf87wP2N+028pPSeUaXjq5ZoxFm1knqsN9fyYMhsd+3qX3PvqX5dcbVWZuq524H+3NFKL+LGPfSyKP/ShDyXH+N8BfgNfKa0R8nPJNVf2fA7kfncWNQtdvXp1MiZXB9dZ50a7rzjrrLO6HCNJl19+eRR/5CMfieLFixcnY973vve9+vfOtZle38hQAACAHsQCCgAAoKLCBZSZXWVmq8xsTqfbrjCzZWY2q+PPGd07TbQ78gjNIodQB/IIdSlTAzVd0rcl/djd/vUQwldrn1GBXD2Tv37ur9vn6o68XH8Wf23fXy8usyGxvxacqzsYOHBgl4+T69fi+z75eoxc/YB/7HHjxmVm3G2mq4XyqD/weTR+/PjkmCuuuCKK/c9Orlbhne98ZxTPnz8/OcbXJfqePg32eZquXswhXzeZ42t7hg8fHsWzZs1Kxviebbk6FH+u8TUlZeoxG+k35/vs5Mb412XlypVRnDv/+lqwXG+kovNgE73Cpqub8uiwww5LbjvllFOi+MADD4ziXC2q/3nxG87n+vwtW7YsinfffffkGP9YPs7VM/maNv8+lHnvfN7kaul8rvnfV0cccUQy5qmnnopi/zpJaS2YP1/l6qr/6Z/+6dW//+d//mfy768o/AQqhHCHpHVFxwFdIY/QLHIIdSCPUJdmaqA+amazOz4OHVLbjNDfkEdoFjmEOpBHqKTRBdR3Je0nabKk5ZL+Y1sHmtlUM5tpZjMbfCz0XeQRmkUOoQ7kESpraAEVQlgZQtgSQnhZ0n9LSi9O/u3YaSGEKSGEdPMb9GvkEZpFDqEO5BEa0VAjTTMbFUJY3hGeJWlOV8fXKVdQ7YvIJ0yYEMW5Aku/ceEhhxySHOM3xSzTeDJXHNdZrvDcFwyuX78+il//+tcnY5555pko9oWbuUJUX+yXa+rXk3ozj3I50UiDu+7i5+ffu1zjQV/sedBBB0XxV77ylWSML6j0Xyy45JJLkjFlNlX2mxDvu+++UXz33XcX3kcZPZlDQ4YUX9Xxhdv+CyK580OZYm6fm/49yBXyltm0tuhx/PPJ5Z0/p/mNznNF5AcccEAU54rr/WONGDEiin3RdDMazaPhw4fr3HPPfTU+++yzk2P8F3z8+5JrNFrUnDT33voC6lyu+ffCF6PnctHfj/89mJuL/2KBP5/lmtD6+/WvwYYNG5Ix/stf/ndn7hj/2P5ntIrCn1wzu1bSCZKGmdlSSf9P0glmNllSkLRQ0gcbngH6BfIIzSKHUAfyCHUpXECFEM7P3HxlN8wFfRh5hGaRQ6gDeYS60IkcAACgorbbTDjX8NJfX/XXzv3GprljctdkfQ2Ul2va5a9n+2Nyzc38GH9dOtcA8dFHH43ie++9N4pPP/30ZIzfSDJ37drXzTz22GPJMX1BmXqnMjUkZeqBGuHn53Pc10RI0pgxY6LY1y/9/ve/T8YceeSRUfyud72r0jy3xb8uZebf6gYPHhzFucaavmbI1wMtWrQoGeNrP8qc4xrZ0NUfk2u2W1TDmXsc/zr4mqhHHnkkGbPXXntFca4OyD9n/1q2gnXr1uknP/nJq/H999+fHHP00UdHsa+3zW2s7etyfP1drlapqH5NShu7+jj3/hf9fs3NpWgj6NzvVl+f5XMi93Ph55JrVO2P8Y+Tq0v+zW9+8+rffb1xZ3wCBQAAUBELKAAAgIpYQAEAAFTUdjVQuV5M/tqvP+aPf/xjMsZf683VZOR6BXWWuybr77dMjxd/TdbXWixYsKDwPnydV67uy19TztUz9HZvqFbSXfVNXq7Wyj92mZotvzGw32jz0EMPTcZ07mFTJz9fn1e5mpdW52spcpvZ+jodXx908803J2P8+5K736Kaktx5xp8H/WueG1NUa5U7//r5+uec22za19qVqSfNbfraCjr//M6Zk7aP8vWpXq6Wbp999oni/fffP4pzdbG+n2DuvfLnGp9XuRoov3G9r1/K/a7xdbxFsZRuJlymTtL/TJapW/XPJ9enrOy5n0+gAAAAKmIBBQAAUBELKAAAgIpYQAEAAFTUdkXkuQJLX9Tmm2nlir2LijJzfLFfrhDOz88X8uWaco0dO7bLuT355JPJGF8wuHr16ijONZ3zBXdLlixJjilqHtpXlCnc9sX8uQ2aR40aFcV/+MMfKs+lkWL1z372s8ltPs8nTZoUxWeddVblxynzJYjcz5cf1xe+nJB7np7PKz8mVzzvX6t169YlxxQV++beJ18c6wv7c8XLuds68wW4Upq/fkPqO++8Mxnjz4O5TeL9uSjXhLi3bdmyJfo9kDvv+nNEmUJnnwP+vFKmmD/Hf0mgTHNV/1hFjTWlNB/9feS+NOCbeg4aNCiKcznin3Pu58B/+WDjxo1d3ocUN7zt6ktcfAIFAABQEQsoAACAigoXUGY2zsxuM7NHzewRM7uo4/ahZnarmc3v+O+QovtC/0UeoVnkEOpAHqEuZWqgNku6JITwoJkNlPSAmd0q6R8kzQghfMnMLpN0maRLu2+qW+WuH/trshs2bIjiXF2Pv1ada1RYVM+Qu97qb/O1CrnmnH6Mr63K1SWMGDEiiv116Pvuuy8Z45+zb1wmdWsNVEvlUZm6o4kTJ0axr+2Q0lzLNfurY+Ncv1Gw36BUSusMjjvuuKYfN/c6FW04mxvnN49tUK/mkH9vc7UTvv7S/+yW2ex0zz33TI7xNTF+8/M99tgjGbNq1aoo9hvS5ubv60P8/ebeR3++8ueZXA755+g3OpfS1yq34XuDui2Pck0Zc7cV8c/V1//kfl/5uqLc741cHVFnud9Pvi6qTC1gURNqn2dS2vjX//7N/b71z6dMPaY/Jnd+9nPZlsJPoEIIy0MID3b8faOkuZLGSDpT0o86DvuRpHeUekT0S+QRmkUOoQ7kEepS6Vt4ZjZe0usk3StpZAhhecc/rZCUfkVp65ipkqY2PkX0NeQRmkUOoQ7kEZpRuojczHaT9HNJF4cQousWYetntNlrIiGEaSGEKSGEKU3NFH0CeYRmkUOoA3mEZpX6BMrMBmhrol0dQvhFx80rzWxUCGG5mY2StGrb91CfMtc4fU1BrnfJlCnVc//FF1+M4tx13qJNUgcOHJjc5q/153qJeL5WydfnzJs3Lxlz/PHHR7F/PlLa+6hOdeZR5+vjjfRRKtMH6q677qp8v91l2rRpUXzAAQckx7zlLW+p/XHL1AaWGXfQQQfVMp/ePBf580qu7sj3KvKvQ+7n3+ddrnbFn/d8/VKutsX31fHvwT333JOMKaqbyvUJ8s/JP58VK1YkY5YvXx7Fjz32WHLMhAkTojjXb6hRrfQ7LcfXp+bqVb3169d313SwDWW+hWeSrpQ0N4TwtU7/dIOkCzv+fqGkX9Y/PfQV5BGaRQ6hDuQR6lLmE6hjJF0g6WEzm9Vx26clfUnS/5rZ+yUtkvTu7pki+gjyCM0ih1AH8gi1KFxAhRDulLStz+xPrnc66KvIIzSLHEIdyCPUhU7kAAAAFbXdZsJl5JrVeb4oL1eEWdSAK1dg62/z95trXucb9Pki8twGxL6Bo3+c3EbHRRtJSuVeu1bQSOF41fG+WPrGG29MjvENLr/4xS8mx1x77bUVZyd95jOfieLTTjstir/5zW8mY+bMmVP5cbqL/9nxxcjtyDcqzG2I6vmfyze84Q3JMX4j8FzDVv/llDKNgP3Pu/8yTa5prn9OZTY6fu1rXxvF/txzyimnJGN8oXwuP/yXXHKbeQO9iU+gAAAAKmIBBQAAUBELKAAAgIr6RA2Uv56+ePHiKB40aFAyxl+3nz17dnKM35zV1xTkNjf0x/iap1zzSr9xpD8mV9/g79fPtcyGj2Wakrai3XbbLWqEmmte6mvEfJO53Aaf/nX39WC5+rD99tsvii+55JLkmBkzZkSxb1Z46qmnJmM+9rGPRfHtt98exZdddlkypqeUqR/zDRfbpbauK74x5YIFC5JjfCNN32Qy11TS/+yWOUf4n91cc1N/v77mKXdeKdqcNVeP6eum/PxzDTD9z1+u0ap/7GbrHoG68QkUAABARSygAAAAKmIBBQAAUBELKAAAgIpav2LYyRWE+8Zzs2bNiuK99torGTN+/Pgo/vOf/5wcU9RI0xeMS2lh5lNPPRXFuR3c/RhfYOkLUyXpueeei+IRI0ZEca7g0heeDxs2rHAurWjHHXeM3j//Xkppwa/Pm1xDU98k8OWXX47ieyG7LwAAIABJREFUJUuWJGOuvvrqKM59GeHkk+PdIY4++ugonjRpUjLmT3/6UxT74vRc4bz/MkWuGLmn+Pz87W9/20szqY8vhs4VR/v3xRdy534u/Wvl30epsSL8wYMHR/Ff/vKXwjG+GN3PLXfO81+K8M8x17Bz48aNUZw77/j8LfPFGKAn8QkUAABARYULKDMbZ2a3mdmjZvaImV3UcfsVZrbMzGZ1/Dmj+6eLdkUeoVnkEOpAHqEuZS7hbZZ0SQjhQTMbKOkBM7u149++HkL4avdND30IeYRmkUOoA3mEWhQuoEIIyyUt7/j7RjObK2lM16O6T27DVH9t3zd7y9Ud/fKXv4xi36gup8w1eH/d3se+LkFK6wHKbBTqN0P2zzFXn3H99ddHsW/yJ+Vrg+pQZx6tXbtW06dPb2o+uZwYO3ZsFA8dOrTLf5fSmpG99947OcbXPPnXPbdJ8TXXXBPFuforrzdrnjxfs/Pxj388ij/3uc9Vvs/ePhf5eqBcPebChQuj2Ncv+to8KW1EmWvy6scV1U1Kae2Rr60qc87zzzE3xtc8+ThXg+rPpbnzjm9+W6aGq4zeziP0HZVqoMxsvKTXSbq346aPmtlsM7vKzNp/u3X0CPIIzSKHUAfyCM0ovYAys90k/VzSxSGEDZK+K2k/SZO1dTX/H9sYN9XMZprZzBrmizZHHqFZ5BDqQB6hWaUWUGY2QFsT7eoQwi8kKYSwMoSwJYTwsqT/lnREbmwIYVoIYUoIYUru39F/kEdoFjmEOpBHqENhDZRtLfK4UtLcEMLXOt0+quNasiSdJSktTuoGfpPYbd3W2WGHHVZ4v2X6rPjapBxfm+DrXXxvodz9+t4xOb5uwvesytUd+M1Pfe1Vd2q1PFq7dm2p29AYXwv0ne98p+n77O0ceuSRR6LY10RJaU+vf/u3f4viXB2lr8dbs2ZNcoyvPZowYUIUv/3tb0/G+PfAn3sOOOCAZIzvheY3F8718/IbR/u6r9zz8cccfvjhyTFPP/10FPveaI3q7TxC31HmW3jHSLpA0sNm9kqHyk9LOt/MJksKkhZK+mC3zBB9BXmEZpFDqAN5hFqU+RbenZIs80/pV4eAbSCP0CxyCHUgj1AXOpEDAABUxAIKAACgIsttbtltD2bWIw/mC6onT56cHOOLxnOF2/4Y/1rlCkL9Y/tGi7kxfiNgXxTvC8Zz9+ML0R9++OFkjN/00xd/Svki96pCCLmPx2vTU3mE3tWdeVRHDp1++unJbccee2wUf/azn43i3CbQ/ZEvIv/mN7+ZHHPnnXdG8Q9+8IPKj8O5CHXYVh7xCRQAAEBFLKAAAAAqYgEFAABQUU/XQK2WtEjSMElpd7XW1E5zlXp/vnuHENIdU2tEHnW7Vphrt+ZRm+aQ1F7z7e25ci7KY67VbDOPenQB9eqDms1slzb47TRXqf3m24x2eq7MtTW123Ntp/m201yb1U7PlbnWh0t4AAAAFbGAAgAAqKi3FlDTeulxG9FOc5Xab77NaKfnylxbU7s913aabzvNtVnt9FyZa016pQYKAACgnXEJDwAAoKIeX0CZ2Wlm9riZLTCzy3r68btiZleZ2Sozm9PptqFmdquZze/475DenOMrzGycmd1mZo+a2SNmdlHH7S053zq1cg5J5FG7aOU8IofaQyvnkEQedbceXUCZ2faSviPpdEkTJZ1vZhN7cg4Fpks6zd12maQZIYQJkmZ0xK1gs6RLQggTJR0p6Z87XstWnW8t2iCHJPKo5bVBHk0XOdTS2iCHJPKoe4UQeuyPpKMk3dIp/pSkT/XkHErMcbykOZ3ixyWN6vj7KEmP9/YctzHvX0o6pV3m28TzbPkc6pgXedTCf9ohj8ih1v7TDjnUMS/yqJv+9PQlvDGSlnSKl3bc1spGhhCWd/x9haSRvTmZHDMbL+l1ku5VG8y3Se2YQ1IbvC/kUcvnUcu/J+RQy+eQ1AbvS7vkEUXkFYStS+CW+tqime0m6eeSLg4hbOj8b604X7Tm+0IetZdWfE/IofbTiu9LO+VRTy+glkka1yke23FbK1tpZqMkqeO/q3p5Pq8yswHammhXhxB+0XFzy863Ju2YQ1ILvy/kkaT2yKOWfU/IIUntkUNSC78v7ZZHPb2Aul/SBDPbx8x2kHSepBt6eA5V3SDpwo6/X6it12V7nZmZpCslzQ0hfK3TP7XkfGvUjjkktej7Qh61VR615HtCDrVVDkkt+r60ZR71QmHYGZLmSXpC0r/1dhGYm9u1kpZLeklbr2e/X9Ie2lr5P1/S7yQN7e15dsz1WG39KHO2pFkdf85o1fn2lxwij9rnTyvnETnUHn9aOYfIo+7/QydyAACAiigiBwAAqIgFFAAAQEUsoAAAACpiAQUAAFARCygAAICKWEABAABUxAIKAACgIhZQAAAAFbGAAgAAqIgFFAAAQEUsoAAAACpiAQUAAFARCygAAICKWEABAABUxAIKAACgIhZQAAAAFbGAAgAAqIgFFAAAQEUsoAAAACpiAQUAAFARCygAAICKWEABAABUxAIKAACgIhZQAAAAFbGAAgAAqIgFFAAAQEUsoAAAACpiAQUAAFARCygAAICKWEABAABUxAIKAACgIhZQAAAAFbGAAgAAqIgFFAAAQEUsoAAAACpiAQUAAFARCygAAICKWEABAABUxAIKAACgIhZQAAAAFbGAAgAAqIgFFAAAQEUsoAAAACpiAQUAAFARCygAAICKWEABAABUxAIKAACgIhZQAAAAFbGAAgAAqIgFFAAAQEUsoAAAACpiAQUAAFARCygAAICKmlpAmdlpZva4mS0ws8vqmhT6F/IIANBuLITQ2ECz7SXNk3SKpKWS7pd0fgjh0S7GNPZgaCshBCt7LHmEbamSRwDQ05r5BOoISQtCCE+GEP4q6TpJZ9YzLfQj5BEAoO00s4AaI2lJp3hpx21AFeQRAKDtvKa7H8DMpkqa2t2Pg76NPAIAtJJmFlDLJI3rFI/tuC0SQpgmaZpE7QqyyCMAQNtp5hLe/ZImmNk+ZraDpPMk3VDPtNCPkEcAgLbT8CdQIYTNZvZRSbdI2l7SVSGER2qbGfoF8ggA0I4abmPQ0INx6aVf6O6vn5NH/QNtDAC0MjqRAwAAVMQCCgAAoCIWUAAAABWxgAIAAKiIBRQAAEBF3d6J3DP72xdrevIbgEU6z+sVfn7+mNz8t99++yh++eWXC8cU3W9dc0NrKvP+egMHDkxuO/bYY6P4pptuqvzYPn83b95ceB9l5J6jR84CaCd8AgUAAFARCygAAICKWEABAABU1OM1UEV1DoccckgU+xqM3XbbLRkzc+bMbp9X2WO2bNlS+2PXNTe0pu22S/8/xufR/vvvH8Uf+MAHkjHPP/98FG/atCmKX3jhhWTMfffdF8Vlap58PVNu/v6YMvfbuf6qkZ8jAOhJfAIFAABQEQsoAACAilhAAQAAVNRUDZSZLZS0UdIWSZtDCFPqmBT6F/IIANBu6igiPzGEsKbMgdttt5122mmnV+N3v/vdyTFvf/vbo3j27NlR7BtTStJxxx0XxUuWLIniwYMHJ2N8I8IFCxYkxwwbNiyK16wpfpr+sV588cUozs3fNy/0j/v0008XjvGPk+MLzQcMGJAc42/bcccdu5ybJP3whz989e+rV68unMc2lM6jvsa/l1JaRH3SSSdF8Zve9KZkzNKlS6PYv3e77LJLMuaUU06J4h/84AdRvHLlymSMz6MyBd/+yx+5n4Pnnnuu8H4AoFVwCQ8AAKCiZhdQQdJvzewBM5uaO8DMpprZTDObyVftsQ2V8qiH5wYAQKLZS3jHhhCWmdkISbea2WMhhDs6HxBCmCZpmiRtv/32rKCQUymPzIw8AgD0qqYWUCGEZR3/XWVm10s6QtId2zp+991316mnnvpqPHny5OSYyy+/PIp9fdNpp52WjPENAmfNmhXF++yzTzLmpZdeiuIjjzwyOcbXPO25555RvMceeyRjfDNDXxN04IEHJmPWrVvX5ZhJkyYVPo6vk8rVRB1//PFRnJu/f+3mzp0bxblGphMmTHj17xs2bEj+vUjVPOpr/vrXvxYe8/rXvz6Kx48fnxzja6l8g8tbbrklGfO6170uir/85S9Hca5J7cMPPxzFPkck6YgjjohiP/+77rorGXP33Xe/+vdnn302+XcAaCUNX8Izs13NbOArf5d0qqQ5dU0M/QN5BABoR818AjVS0vUdWza8RtI1IYSba5kV+hPyCADQdhpeQIUQnpR0aI1zQT9EHgEA2lGPbib80ksvadmyZa/GuQ1Gp0yJeyj62olnnnkmGeNve+Mb3xjFt99+ezJm9OjRUXzBBRckx9x8c/xBiK87yfWyue6666J4xIgRUbzrrrsmY3wt0s477xzFBx98cDKmc72IJK1duzaKDzjggGTMkCFDotjXgUlpDZOf/7HHHpuM6dwHKnefiPmNdnPfTvX9mfzPxcaNG5MxPrd8DuRy4v77749i3w8tV/N21FFHRfHZZ5+dHOPzwD9ObjPkznV7dWwQDgDdiT5QAAAAFbGAAgAAqIgFFAAAQEUsoAAAACrq0SLynXbaSQcddNCr8dixY5Nj9tprryieMyduCbTffvslY3xxt288edtttyVjRo0aFcVPPPFEcozfOHfTpk1RvGjRomSM55sk+o2OpbRI3L8uuU1gPb/p69ve9rbCY/xrK0n7779/FPvi5UGDBiVjOhe9++aN/Y0vEG/U5z73uSj2+Zrj88R/SSPXsNN/KcC/37kvSjz44INRnNuI2z/2P//zP0fxvvvum4x55zvfmdwGAK2qf/+2AwAAaAALKAAAgIpYQAEAAFTUozVQmzdvjho+Dh8+PDlmxYoVUexrnnI1Nv5+fK1Prt7izDPPjOIHHnggOcbXIs2ePTuKTzrppGSM37jY1xn5xqBSurGqbwTqNwqWpMMOOyyKt2zZEsW518nXiuVef9/E0z927n4HDBjw6t/rqgFqV7mmmI1Yv359FPsaKL+ZtCTtuOOOUfya18Q/3rmmmH4jbv/+52qg/AbfRx99dHKMzxPfkNU3qQWAdsMnUAAAABWxgAIAAKiocAFlZleZ2Sozm9PptqFmdquZze/475Cu7gMgjwAAfUmZT6CmSzrN3XaZpBkhhAmSZnTEQFemizwCAPQRhUXkIYQ7zGy8u/lMSSd0/P1Hkv4g6dKi+9puu+2iQta//OUvyTF33nlnFJ92Wvw71xe5StJjjz0WxRs2bIjiPffcMxnzzW9+M4pPPPHE5BhfZH3yySd3OdfcbWPGjIniG2+8MRnjG3/6xprXXXddMsYX4foCcV/wLklHHnlkFA8dOjQ5xnv00Uej2L/WUly0/9JLL2Xvp8486g98U0xflJ0r5n/uueei+Jlnnonizl/geIXPG18En/tSgH/sXKNX/6UGX4w+bty4ZAwAtJNGa6BGhhCWd/x9haSRNc0H/Qt5BABoS00XkYet/8u6ze9um9lUM5tpZjNffPHFZh8OfVSVPOrBaQEAkNXoAmqlmY2SpI7/rtrWgSGEaSGEKSGEKb5PDfq9hvKox2YHAMA2NNpI8wZJF0r6Usd/f1lm0IABAzRy5N+u0qxbty45ZvLkyVHsN6/N1dj4Yzo/hiQdeuihyZgZM2ZEsd/8VJIOPPDAKL7kkkui2NecSNL73ve+KPbNOH/4wx8mY26//fYo9vVYjz/+eDLG14L5jVgHDx6cjJk/f34U5xa0vmbLP46viZKkgQMHvvr31atXJ//ehYbyqJWVqRny9UG5BpejR4+OYv/pbe7TXP9++s2Dc/nq88TXSeXqm3bYYYco3rhxY3LM7rvvHsW+Ji/3nDtvZJzLMwBoJWXaGFwr6W5JB5rZUjN7v7b+wjvFzOZLelNHDGwTeQQA6EvKfAvv/G3808nbuB1IkEcAgL6ETuQAAAAV9ehmws8991y0ae873vGO5JgFCxZE8fLly6PYb7Qrpf2afI8nXxMlSZ/85CejOFdT8olPfCKK/SbFF110UTJmjz32iGJfs3XUUUclY2644YYo/ta3vhXFJ5xwQjLG97b685//HMW5uqm3vvWtUbzXXnslx/jNjztvFCzl68nuvvvuV/++/fbbJ//en+Q2E/avia+BOvfcc5Mx/v31tWW5fmi+19Kuu+4axbneS75OytdR5WoO/SbFubn4n4PvfOc7UexrHf399vdNqQG0Pj6BAgAAqIgFFAAAQEUsoAAAACpiAQUAAFBRjxaRb9myRZs2bXo1Pv3005NjHnnkkSi+9tpro9gXp0rpprhLliyJ4ve85z3JGN98M1dQfe+990bxE088EcU/+clPkjFnn312FPsmig8++GAyZt99941iX8g7ZMiQZIwvGPavy0MPPZSM8a9T7n5vuummKP6Hf/iHKM4VDHcu+O3vxb++wFpKC7U9X7gvpV9q8MX8uWJ9X5w+YsSIKH7hhReSMb5xpn+cnXbaKRnji9PXr1+fHLN06dIo9j+DX/nKV5Ix99xzT3IbALQqPoECAACoiAUUAABARSygAAAAKurRGqiddtop2qA3Vw/k6zgmTpwYxX/84x+TMb7u5Jhjjoliv5GpJG3YsCGKDz744OSYxYsXR/F73/veKPabDUvSr3/96yj29SLHHntsMsY3K5w1a1YUP//888kY31jRbxT7lre8JRkzb968KP7GN76RHHPAAQdEsX9tfe2VFDdobJVNYH0tlq8Z8rVpuTH+fck9dy+3KXWRG2+8Mbmtc62glOaA39BXSpt4+hzJ1U35Gqdc40yvzOviH2vSpElR/MwzzxQ+DgC0Mj6BAgAAqIgFFAAAQEWFCygzu8rMVpnZnE63XWFmy8xsVsefM7p3mmh35BEAoC8pUwM1XdK3Jf3Y3f71EMJXqzzYiy++qPnz578a53oKrVixIor9prgXXHBBMsbX3cydOzeKL7/88mRM5w1wpXTzVkk644z497nftDjXO2q33XaLYt97J9eTym8m7Dc/zm0Cu3HjxigeNWpUl/cppf2lzjrrrOQY3/uq8+bPknTmmWcmYzrXVuV6DXWYrpryyCvTE6mR2qRGHH/88clt55xzThT7Gj1fvyal/Zl8zVOu35R/zv5+c6+TzwlfE5XbHDk3X8/P99lnn41i3y9Nkn71q18V3i8AtIrCT6BCCHdIWtcDc0EfRh4BAPqSZmqgPmpmszsuzaQtrYFyyCMAQNtpdAH1XUn7SZosabmk/9jWgWY21cxmmtnMMl+RRr/SUB711OQAANiWhhZQIYSVIYQtIYSXJf23pCO6OHZaCGFKCGGK32cL/VujedRzMwQAIK+hRppmNiqEsLwjPEtSuhtqflxU/JpriumLWk888cQoPvzww5MxTz31VBT7QuYnn3wyGZNrgun5Atrf//73UTxw4MBkjC8095vC5jaOve+++6LYF9f71yR3my8Q9hsqS9KECROiOFdE7uf/i1/8Iopzhb6dx3T+kkCRRvPI88XTZfiNlSVp9OjRUexfL//vUloM7RuRSmkO+CaeuaJsvzl0UY5LaeG230w4t6nxLrvsEsV33XVXFPsvRUhpoXyukaZvlOk/fT7yyCOTMQDQTgoXUGZ2raQTJA0zs6WS/p+kE8xssqQgaaGkD3bjHNEHkEcAgL6kcAEVQjg/c/OV3TAX9GHkEQCgL6ETOQAAQEU9upnwgAED/n979x9qdX3Hcfz1nmsoVDdLKvFnZFJRoWBtlX+sgnAR5V9iwVAaNWKDHAVZ/2wQo/7Y2qCNgVJMKDaEbPVHUSLFWqHNzHR6M3NpU67eRNPMwLT3/rjHy/m8P597z/127z33nHOfDwjP+3vO95zPOecj59Pxdd6fJEcSN/SV8s13v/jii6QuZYji/cRmm7ExpZQ3Kixt2HvTTTcldWzGGJtOSnkGKOaZnnnmmeycmOuK+Ze4ubCUZ5Vmz56d1Lfeemt2zmuvvZbUsUmmJF1wwQVJPZRsVdyEt9lKeZonnngiqePrFZ+nlGep4nOPc1HK50RscCrl2aP4epXmXswiLVmyJKk3b85/jBgzeTF7FedIybXXXjvofUr5HChluOK8j1mqWbNmNRwLALQyvoECAACoiAUUAABARSygAAAAKrLSZqGjpaury+tzRVdddVV2m7gpbsx6lHrxXH755YM+bsyCSHkepHSbI0fSrdtin5133nknOyf2F4r3UcpjxZ5UsWdOKS9y9OjRpI5ZnFLGJ2Z4ShvSvvrqq4OOLeazJGn16tX9l3ft2qWTJ0+OaijKzLw+nxQ3hpbyeRTzTaXeUY02yS1txlvKLzXS1dWV1FOmTMlus3z58qS+/fbbk/rBBx/MzmnUK+rTTz/Nzok90mLvq9L7HTNdpQa5MTsVb1PqHRXnubuPbbgOAAbBN1AAAAAVsYACAACoiAUUAABARSygAAAAKmpqI00pDY9OnDgxu/7GG29M6hhqjRuxSnlg+qWXXkrqvXv3ZufEJpmlBp3bt29P6riB7/3335+dEwO2sWFnbBQqSa+//npSx+D8o48+mp1zzTXXJPWqVauS+sMPP8zOeeyxx5K6FMg///zzk3r69OlJXdosuD4UXQpaj7SLLrpId911V39dCtnv2bMnqWMjx9ImuaUNhuuVwtIxEF5qNBrD3XED30OHDmXnrFmzJqkXL16c1KVNneMPI+JzLG3EHTfrjn+/ShsQx78H8ccVJTG0X3otZ8yY0X/54MGDDe8TAMYS30ABAABUxAIKAACgooYLKDObYWZvmtlOM9thZg/Vjl9oZuvNbHftz8mjP1y0K+YRAKCTDCUDdVrSw+6+xczOk/S+ma2XtFzSBnd/ysxWSlopKQ/r1Pnmm2+SvEepCWF3d3dSx7xFqUFkbP741ltvJfX8+fOzczZu3JjUMTMj5VmVOJZStio2yoyZp3gfUr7Rbcw3lfJZMVt16aWXJnVpU9vYNLGUV4oZqNjw8MSJE9k5hw8f7r8cG3rWGbF5dPr0afX29vbXpdxRo411S+fEzFDM9sTXRsobpe7bt6/h/cZ5HxteSvnrGHN9MZ8n5RmomOkq5Zlic9XYxLX0fsY5UcozxdvEDZRLuam5c+cOOC4AaDUNv4Fy9x5331K7/KWkbknTJN0t6WzSdY2kxeV7AJhHAIDOUulXeGY2W9J8SZskXeLuPbWrDkrK9yjpO+cBSQ9I5f9Txfgz3Hk0adKk0R8kAACDGHKI3MzOlfSipBXufrz+Ou/bUK+4qZ67r3L3Be6+oLT3GsaXkZhHQ/nZPAAAo2lIKxozO0d9H3ovuPu62uFDZjbV3XvMbKqk3oHvoc/EiROTzWmXLl2a3Sb2zIlZns8//zw75957703quLlwKS9y2WWXJXXsdyRJb7zxRlLHLFVpo9VSRqje5Ml5RnrOnDlJHfNNMRNVepx4zrx587JzrrvuuqQ+fvx4dpuY2Yr9e2JfLint3VXqaXTWSM2jU6dO6cCBA/11aUPs/fv3J3V8XqUNfGPupj7bJZXnXvyfglLGLX7zGvufxbyWlPdjimMpbcT91VdfJXXMecUNqEvjjY8TM1FSnosq3SZ+SxgzeseOHcvOqZ+zpdwfALSSofwKzyQ9K6nb3Z+uu+oVSctql5dJennkh4dOwTwCAHSSoXwDdbOkn0rabmZba8cel/SUpLVm9jNJ+yQtGZ0hokMwjwAAHaPhAsrd/yXJBrj6tpEdDjoV8wgA0EnoRA4AAFBRU38Wd+bMmSQUHkPaUh6wjQHq2BBRkjZt2jTobWJDTCkP7pYaBsbNV2PIuLQxcBTD3jt27MhuE4PIU6dObXi/sWFnbKJYapL52WefJXVp89z42sVmoaXmoR999FH/5VJTyJH29ddfa+vWrf31unXrstvcd999SR1/nBCbikr52GMDzFIbjhiWLv1CML4X8TWOQX0pD8afPHkyqXt6ehTFc+L9ln4F2+g5D6X5ZqnpZaOGnPFHHFL6A4RSMB0AWgnfQAEAAFTEAgoAAKAiFlAAAAAVNTUDNWHChGRD1qE0JrzttvQHWh988EF2znvvvZfUsRngwoULs3NiE8lSTio2vYwbusaMlCTNnDkzqeOmqjGLUxpLvI9S7ivmcWIOJWZZJGnXrl1JXWpmuGjRoqTesGFDUpcyPvV5lp07d2bXj7Ynn3wyO1afkZKkRx55JKljZkzK5018TWOjSinPN5Venzin4zlxo10pzzPF97uUx4qPHW9Tepwo3qbUGDXOrVKWLs772Ehz27Zt2TnPP/98w/EBQKvgGygAAICKWEABAABUxAIKAACgIhZQAAAAFVlpJ/vRMmnSJJ8zZ05/ff3112e3ic3/pk2bltSlBpExxBqbDJaeY29vb1Lfeeed2W1igHb37t1JXR+IH+ixjhw5ktSl8G9skhibF37ve/k6NzbSjK9bqTHo9OnTk/rjjz/ObjN37tykPnr0aFKXGiCuXbu2//LGjRt17NixxmnlYTAzr39NYmB5KG655ZbsWAyjX3zxxUnd1dWVnRPfm9L8jCHyUuPMKM7POK8OHDiQnRNfh9jEtTS2KD5OqaFlnK+l+bl+/fqk7u7uTup33313KGMZ1XkEAMPBN1AAAAAVNVxAmdkMM3vTzHaa2Q4ze6h2/DdmdsDMttb+u2P0h4t2xTwCAHSSofSBOi3pYXffYmbnSXrfzM5+P/8Hd//d6A0PHYR5BADoGJUzUGb2sqQ/SbpZ0okqH3xm1rzAFcbMULIr7TiPrrzyyuzYlClTkrq0sW7MnsUNmUs5oz179nyHEXYWMlAAWlmlDJSZzZY0X9Km2qFfmtk2M3vOzCYPeCJQh3kEAGh3Q15Amdm5kl6UtMLdj0v6i6TLJc2T1CPp9wOc94CZbTazzSMwXrQ55hEAoBMMaQFlZueo70PvBXdfJ0nufsjdz7j7t5JWS7qhdK67r3L3Be6+YKQGjfbEPAIAdIr4JHvQAAADNklEQVSGGSjr2110jaQj7r6i7vhUd++pXf6VpB+6+9IG90UGahwoZVeYR6iKDBSAVjaUBdRCSW9L2i7pbKe+xyXdo75/dnFJeyX9/OwH4SD3xQffODDAAop5hEpYQAFoZU3tRM4H3/gw2h98zKPxgQUUgFZGJ3IAAICKWEABAABUxAIKAACgIhZQAAAAFbGAAgAAqIgFFAAAQEXfb/LjHZa0T9KU2uV20E5jlcZ+vLOa8BjMo9HVCmNtxjwCgO+sqX2g+h/UbHO7bMnRTmOV2m+8w9FOz5WxAkBn4Z/wAAAAKmIBBQAAUNFYLaBWjdHjfhftNFap/cY7HO30XBkrAHSQMclAAQAAtDP+CQ8AAKCipi+gzGyRme0ys0/MbGWzH38wZvacmfWa2X/qjl1oZuvNbHftz8ljOcazzGyGmb1pZjvNbIeZPVQ73pLjHUmtPIck5hEAjAdNXUCZ2QRJf5b0E0lXS7rHzK5u5hga+KukReHYSkkb3P0KSRtqdSs4Lelhd79a0o8k/aL2WrbqeEdEG8whiXkEAB2v2d9A3SDpE3f/r7ufkvR3SXc3eQwDcvd/SjoSDt8taU3t8hpJi5s6qAG4e4+7b6ld/lJSt6RpatHxjqCWnkMS8wgAxoNmL6CmSfpfXb2/dqyVXeLuPbXLByVdMpaDKTGz2ZLmS9qkNhjvMLXjHJLa4H0ZZ/MIAIaFEHkF3veTxZb62aKZnSvpRUkr3P14/XWtOF605vvCPAKAapq9gDogaUZdPb12rJUdMrOpklT7s3eMx9PPzM5R34feC+6+rna4Zcc7QtpxDkkt/L6M03kEAMPS7AXUvyVdYWaXmdkPJC2V9EqTx1DVK5KW1S4vk/TyGI6ln5mZpGcldbv703VXteR4R1A7ziGpRd+XcTyPAGBYmt5I08zukPRHSRMkPefuv23qAAZhZn+T9GP17UZ/SNKvJf1D0lpJMyXtk7TE3WNAuOnMbKGktyVtl/Rt7fDj6suvtNx4R1IrzyGJeQQA4wGdyAEAACoiRA4AAFARCygAAICKWEABAABUxAIKAACgIhZQAAAAFbGAAgAAqIgFFAAAQEUsoAAAACr6P0VXowIQARH9AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 720x720 with 10 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        },
        "id": "63jtDyXR-8Wb",
        "outputId": "2c192ebf-b803-4a3d-ec00-6edcee14d3e6"
      },
      "source": [
        "#display images with names in wandb \r\n",
        "class_names=['T-shirt/top','Trouser','Pullover','Dress','Coat','Sandal','Shirt','Sneaker','Bag','Ankle boot']\r\n",
        "\r\n",
        "wandb.init(entity='sanjeevrs',project='assignment1')\r\n",
        "    \r\n",
        "wandb.log({'sample':[wandb.Image(trainx[i],caption=class_names[trainy[i]]) for i in index_mat]})"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33msanjeevrs\u001b[0m (use `wandb login --relogin` to force relogin)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Tracking run with wandb version 0.10.22<br/>\n",
              "                Syncing run <strong style=\"color:#cdcd00\">rosy-blaze-89</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://wandb.ai/sanjeevrs/assignment1\" target=\"_blank\">https://wandb.ai/sanjeevrs/assignment1</a><br/>\n",
              "                Run page: <a href=\"https://wandb.ai/sanjeevrs/assignment1/runs/du47f1xd\" target=\"_blank\">https://wandb.ai/sanjeevrs/assignment1/runs/du47f1xd</a><br/>\n",
              "                Run data is saved locally in <code>/content/wandb/run-20210311_191348-du47f1xd</code><br/><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zcRfsy9P--ak"
      },
      "source": [
        "x_train = trainx.reshape(60000,784)/255.0\n",
        "#one hot encoding y \n",
        "temp = np.zeros((60000,10))\n",
        "temp[np.arange(60000),trainy] = 1\n",
        "y_train=temp"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O12WuMf5dpfS"
      },
      "source": [
        "#**TRAIN FUNCTION**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nIo-diz97IzO"
      },
      "source": [
        "sweep_config = {\n",
        "  'name': 'sweep2',  \n",
        "  'method': 'grid',\n",
        "  'parameters': {\n",
        "        'epochs': {\n",
        "            'values': [5]\n",
        "        },\n",
        "        'batch_size':{\n",
        "            'values':[16,32]\n",
        "        },\n",
        "        'hidden_layers':{\n",
        "            'values':[4,5]\n",
        "        },\n",
        "        'eta': {\n",
        "            'values': [1e-3,1e-4]\n",
        "        },\n",
        "        'size':{\n",
        "            'values':[32]\n",
        "        },\n",
        "        'optimizer': {\n",
        "            'values': ['adam','sgd']\n",
        "        },\n",
        "        'activation': {\n",
        "            'values': ['sigmoid']\n",
        "        },\n",
        "        'wt_decay': {\n",
        "            'values': [0.005,0.0005]\n",
        "        }\n",
        "    }\n",
        "}"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SJpDL67pg-0Q"
      },
      "source": [
        "def Train():            #define inputs\n",
        "  \n",
        "  X = x_train[:50000]\n",
        "  Y = y_train[:50000]\n",
        "\n",
        "  x_val = x_train[50000:]\n",
        "  y_val = y_train[50000:]\n",
        "  \n",
        "  config_defaults = {\n",
        "        'epochs': 5,\n",
        "        'batch_size':32,\n",
        "        'hidden_layers':3,\n",
        "        'eta': 1e-2,\n",
        "        'size':32,\n",
        "        'activation': 'tanh',\n",
        "        'optimizer': 'adam',\n",
        "        'wt_decay':0\n",
        "        }\n",
        "  config=config_defaults  #for testing\n",
        "\n",
        "  #wandb.init(config=config_defaults)\n",
        "\n",
        "  #config = wandb.config\n",
        "\n",
        "  eta = config['eta']\n",
        "  epochs = config['epochs']\n",
        "  batch_size=config['batch_size']\n",
        "  activation=config['activation']\n",
        "  optimization=config['optimizer']\n",
        "  num_hidden_layers = config['hidden_layers'] \n",
        "  size=config['size']        \n",
        "  wt_decay=config['wt_decay']   \n",
        "\n",
        "  input_layer_size = 784  #input number of inputs here\n",
        "\n",
        "  output_layer_size = 10  #input number of output nodes here\n",
        "\n",
        "  num_nodes = [input_layer_size]\n",
        "\n",
        "  for i in range(num_hidden_layers):\n",
        "    num_nodes.append(size)\n",
        "  num_nodes.append(output_layer_size)\n",
        "\n",
        "  num_weight_mat = num_hidden_layers+1\n",
        "\n",
        "\n",
        "  def init_random():\n",
        "    weights={}\n",
        "    bias = {}\n",
        "    for i in  range(1,num_weight_mat+1):\n",
        "      weights['W%s'% i] = np.random.randn(int(num_nodes[i]),int(num_nodes[i-1]))\n",
        "      bias['b%s'% i] = np.random.randn(int(num_nodes[i]))\n",
        "    \n",
        "    return weights, bias\n",
        "\n",
        "  def init_xavier():\n",
        "    weights={}\n",
        "    bias = {}\n",
        "    for i in  range(1,num_weight_mat+1):\n",
        "      weights['W%s'%i] = np.random.normal(0, 1/input_layer_size ,size=([int(num_nodes[i]),int(num_nodes[i-1])]))\n",
        "      bias['b%s'%i] = np.ones(int(num_nodes[i]))\n",
        "    \n",
        "    return weights, bias\n",
        "\n",
        "  def init_zeros():\n",
        "    weights={}\n",
        "    bias = {}\n",
        "    for i in  range(1,num_weight_mat+1):\n",
        "      weights['W%s'% i] = np.zeros([int(num_nodes[i]),int(num_nodes[i-1])])\n",
        "      bias['b%s'% i] = np.zeros(int(num_nodes[i]))\n",
        "\n",
        "    return weights,bias\n",
        "  \n",
        "  \n",
        "  #Setting weight initialization function\n",
        "  initialization='random'\n",
        "  if initialization == 'random':\n",
        "    wt_init=init_random()\n",
        "  elif initialization == 'xavier':\n",
        "    wt_init = init_xavier()\n",
        "  else:\n",
        "    print('weight initialization given is not available')\n",
        "\n",
        "  #Setting weights\n",
        "  weights = {}\n",
        "  bias = {}\n",
        "  weights, bias = wt_init\n",
        "\n",
        "\n",
        "  #Defining activation functions and their derivatives\n",
        "\n",
        "  def sigmoid(x):\n",
        "    return 1/(1+np.exp(-x))\n",
        "\n",
        "  def tanh(x):\n",
        "    return (np.exp(x)-np.exp(-x))/(np.exp(x)+np.exp(-x))\n",
        "\n",
        "  def relu(x):\n",
        "    if x<0:\n",
        "      return 0\n",
        "    else:\n",
        "      return x\n",
        "\n",
        "  def grad_tanh(x):\n",
        "    return 1-(tanh(x))**2\n",
        "\n",
        "  def grad_relu(x):\n",
        "    if x<0:\n",
        "      return 0\n",
        "    else:\n",
        "      return 1\n",
        "\n",
        "  def grad_sigmoid(x):\n",
        "    return (sigmoid(x)*(1-sigmoid(x)))\n",
        "\n",
        "  def softmax(x):\n",
        "    # Numerically stable with large exponentials\n",
        "    exps = np.exp(x )\n",
        "    return exps / np.sum(exps)\n",
        "  \n",
        "\n",
        "  #Setting activation function and its derivative\n",
        "  if activation == 'sigmoid':\n",
        "    activ = sigmoid\n",
        "    der_activ = grad_sigmoid\n",
        "  elif activation == 'tanh':\n",
        "    activ = tanh\n",
        "    der_activ = grad_tanh\n",
        "  elif activation == 'relu':\n",
        "    activ = relu\n",
        "    der_activ = grad_relu\n",
        "\n",
        "  #Setting output function\n",
        "  output = softmax \n",
        "  \n",
        "  activation = activ\n",
        "  der_activ = der_activ\n",
        "\n",
        "  #Forward prop\n",
        "  def forward_prop(weights, bias, x):\n",
        "    Acti = {}\n",
        "    H = {}\n",
        "    \n",
        "    H['H0'] = x\n",
        "    \n",
        "    for i in range(1,num_hidden_layers+1):\n",
        "      Acti[('A%s'% i)] = np.matmul(weights['W%s'% i],H[('H%s'% (i-1))])+bias['b%s'% i]\n",
        "      H['H%s'% i] = activation(Acti['A%s'% i])\n",
        "    \n",
        "    Acti['A%s'%(num_hidden_layers+1)]= np.matmul(weights['W%s'% (num_hidden_layers+1)],H['H%s'% (num_hidden_layers)])+bias['b%s'% (num_hidden_layers+1)]\n",
        "    y_hat = output(Acti['A%s'% (num_hidden_layers+1)])  \n",
        "    \n",
        "    return y_hat, Acti, H\n",
        "\n",
        "\n",
        "  #Backward prop\n",
        "  def back_prop(w, b, x, y):\n",
        "    grad_A = {}\n",
        "    grad_W = {}\n",
        "    grad_b = {}\n",
        "    grad_H = {}  \n",
        "\n",
        "    y_hat, A, H = forward_prop(w, b, x)\n",
        "\n",
        "    grad_A['A%s'% (num_hidden_layers+1)] = -(y-y_hat)\n",
        "\n",
        "    for i in range(num_hidden_layers+1, 1, -1):\n",
        "      \n",
        "      grad_W['W%s'%i] = np.matmul(grad_A['A%s'%i].reshape(-1,1), H['H%s'%(i-1)].reshape(1,-1))\n",
        "\n",
        "      grad_b['b%s'%i] = grad_A['A%s'%i]\n",
        "\n",
        "      grad_H['H%s'%(i-1)] = np.dot(np.transpose(w['W%s'%i]), grad_A['A%s'%i]) #check once if this is logically correct (Looks good)\n",
        "\n",
        "      grad_A['A%s'%(i-1)] = np.multiply(grad_H['H%s'%(i-1)], der_activ(A['A%s'%(i-1)]))\n",
        "\n",
        "    grad_W['W1'] = np.multiply(grad_A['A1'].reshape(-1,1), H['H0'].reshape(1,-1))\n",
        "\n",
        "    grad_b['b1'] = grad_A['A1']\n",
        "\n",
        "    return grad_W, grad_b\n",
        "    \n",
        "\n",
        "   #defining sgd\n",
        "  def sgd():\n",
        "\n",
        "    for k in range(epochs):\n",
        "      for x,y in zip(X,Y):\n",
        "        grad_w,grad_b=back_prop(weights, bias, x,y)\n",
        "        for j in range(1,num_hidden_layers+2):\n",
        "          \n",
        "          weights['W%s'%j]-=(grad_w['W%s'%j])*eta\n",
        "          bias['b%s'%j]-=(grad_b['b%s'%j])*eta\n",
        "\n",
        "   #defining momentum\n",
        "  def momentum_gd():\n",
        "\n",
        "    gamma=0.9\n",
        "    prev_dw,prev_db=init_zeros()\n",
        "\n",
        "    for k in range(epochs):\n",
        "      count=0   \n",
        "    \n",
        "      dw,db=init_zeros()\n",
        "      \n",
        "      for x,y in zip(X,Y):\n",
        "        \n",
        "        if count==0:\n",
        "          dw,db=init_zeros()\n",
        "        \n",
        "        grad_w,grad_b=back_prop(weights, bias, x, y)\n",
        "\n",
        "        for j in range(1,num_hidden_layers+2):\n",
        "          dw['W%s'%j]+=(grad_w['W%s'%j])\n",
        "          db['b%s'%j]+=(grad_b['b%s'%j])\n",
        "\n",
        "\n",
        "\n",
        "        count+=1\n",
        "\n",
        "        if count%batch_size==0:\n",
        "          \n",
        "          weights['W%s'%j]-=((prev_dw['W%s'%j])*gamma+(dw['W%s'%j])*eta+(weights['W%s'%j])*wt_decay*eta)\n",
        "          bias['b%s'%j]-=((prev_db['b%s'%j])*gamma+(db['b%s'%j])*eta+(bias['b%s'%j])*wt_decay*eta)\n",
        "\n",
        "          prev_dw['W%s'%j]=(prev_dw['W%s'%j])*gamma+(dw['W%s'%j])*eta\n",
        "          prev_db['b%s'%j]=(prev_db['b%s'%j])*gamma+(db['b%s'%j])*eta\n",
        "\n",
        "          count=0\n",
        "\n",
        "   #defining nesterov\n",
        "  def nesterov():\n",
        "\n",
        "    gamma=0.9\n",
        "    prev_vw,prev_vb=init_zeros()\n",
        "    temp_w,temp_b=init_zeros()\n",
        "    dw,db=init_zeros()\n",
        "    v_w,v_b=init_zeros()\n",
        "\n",
        "    for k in range(epochs):\n",
        "      count=0\n",
        "\n",
        "      if count==0:\n",
        "        for j in range(1,num_hidden_layers+2):\n",
        "            v_w['W%s'%j]=prev_vw['W%s'%j]*gamma \n",
        "            v_b['b%s'%j]=prev_vb['b%s'%j]*gamma \n",
        "\n",
        "\n",
        "      for x,y in zip(X,Y):\n",
        "        if count==0:\n",
        "          dw,db=init_zeros()\n",
        "\n",
        "        for j in range(1,num_hidden_layers+2):\n",
        "          temp_w['W%s'%j]=weights['W%s'%j]-v_w['W%s'%j]\n",
        "          temp_b['b%s'%j]=bias['b%s'%j]-v_b['b%s'%j]\n",
        "        \n",
        "        grad_w,grad_b=back_prop(temp_w,temp_b,x,y)      \n",
        "        for j in range(1,num_hidden_layers+2):\n",
        "          dw['W%s'%j]+=(grad_w['W%s'%j])\n",
        "          db['b%s'%j]+=(grad_b['b%s'%j])\n",
        "        count+=1\n",
        "\n",
        "        if count%batch_size==0:\n",
        "          \n",
        "          for j in range(1,num_hidden_layers+2):\n",
        "            v_w['W%s'%j]=(prev_vw['W%s'%j])*gamma + (dw['W%s'%j])*eta\n",
        "            v_b['b%s'%j]=(prev_vb['b%s'%j])*gamma + (db['b%s'%j])*eta\n",
        "            weights['W%s'%j]-=(v_w['W%s'%j]+(weights['W%s'%j])*wt_decay*eta)\n",
        "            bias['b%s'%j]-=(v_b['b%s'%j]+(bias['b%s'%j])*wt_decay*eta)\n",
        "            prev_vw['W%s'%j]=v_w['W%s'%j]\n",
        "            prev_vb['b%s'%j]=v_b['b%s'%j]\n",
        "\n",
        "          count=0\n",
        "\n",
        "   #defining adagrad\n",
        "  def adagrad():\n",
        "  \n",
        "    eps = 1e-8\n",
        "\n",
        "    v_w,v_b=init_zeros()\n",
        "\n",
        "    for k in range(epochs):\n",
        "      count=0\n",
        "      dw,db=init_zeros()\n",
        "      \n",
        "      for x,y in zip(X,Y):\n",
        "\n",
        "        if count==0:\n",
        "          dw,db=init_zeros()\n",
        "        \n",
        "        grad_w,grad_b=back_prop( weights,bias,x,y)\n",
        "        for j in range(1,num_hidden_layers+2):\n",
        "          dw['W%s'%j]+=(grad_w['W%s'%j])\n",
        "          db['b%s'%j]+=(grad_b['b%s'%j])\n",
        "        count+=1\n",
        "\n",
        "        if count%batch_size==0:\n",
        "\n",
        "          for j in range(1,num_hidden_layers+2):\n",
        "            v_w['W%s'%j] += np.square(dw['W%s'%j])\n",
        "            v_b['b%s'%j] += np.square(db['b%s'%j]) \n",
        "            weights['W%s'%j] -= ((dw['W%s'%j])*(eta/np.sqrt(v_w['W%s'%j] + eps))+(weights['W%s'%j])*wt_decay*eta)\n",
        "            bias['b%s'%j] -= ((db['b%s'%j])*(eta/np.sqrt(v_b['b%s'%j] + eps))+(bias['b%s'%j])*wt_decay*eta)\n",
        "\n",
        "            count=0\n",
        "\n",
        "   #defining rmsprop\n",
        "  def rmsprop():\n",
        "  \n",
        "    eps = 1e-8\n",
        "    beta1 = 0.9\n",
        "\n",
        "    v_w,v_b=init_zeros()\n",
        "\n",
        "    for k in range(epochs):\n",
        "      count=0\n",
        "      dw,db=init_zeros()\n",
        "      \n",
        "      for x,y in zip(X,Y):\n",
        "\n",
        "        if count==0:\n",
        "          dw,db=init_zeros()\n",
        "        \n",
        "        grad_w,grad_b=back_prop( weights,bias,x,y)\n",
        "        for j in range(1,num_hidden_layers+2):\n",
        "          dw['W%s'%j]+=(grad_w['W%s'%j])\n",
        "          db['b%s'%j]+=(grad_b['b%s'%j])\n",
        "        count+=1\n",
        "\n",
        "        if count%batch_size==0:\n",
        "\n",
        "          for j in range(1,num_hidden_layers+2):\n",
        "            v_w['W%s'%j] = beta1*v_w['W%s'%j] + (1-beta1)*np.square(dw['W%s'%j])\n",
        "            v_b['b%s'%j] = beta1*v_b['b%s'%j] + (1-beta1)*np.square(db['b%s'%j]) \n",
        "            weights['W%s'%j] -= ((dw['W%s'%j])*(eta/np.sqrt(v_w['W%s'%j] + eps))+(weights['W%s'%j])*wt_decay*eta)\n",
        "            bias['b%s'%j] -= ((db['b%s'%j])*(eta/np.sqrt(v_b['b%s'%j] + eps))+(bias['b%s'%j])*wt_decay*eta)\n",
        "\n",
        "            count=0\n",
        "\n",
        "   #defining adam \n",
        "  def adam():\n",
        "  \n",
        "    eps = 1e-8\n",
        "    beta1 = 0.9\n",
        "    beta2 = 0.999\n",
        "\n",
        "    v_w,v_b=init_zeros()\n",
        "    v_w_hat,v_b_hat=init_zeros()\n",
        "\n",
        "    m_w,m_b=init_zeros()\n",
        "    m_w_hat,m_b_hat=init_zeros()\n",
        "\n",
        "    for k in range(epochs):\n",
        "      \n",
        "      count=0\n",
        "      dw,db=init_zeros()\n",
        "      \n",
        "      for x,y in zip(X,Y):\n",
        "\n",
        "        if count==0:\n",
        "          dw,db=init_zeros()\n",
        "        \n",
        "        grad_w,grad_b=back_prop( weights,bias,x,y)\n",
        "        for j in range(1,num_hidden_layers+2):\n",
        "          dw['W%s'%j]+=(grad_w['W%s'%j])\n",
        "          db['b%s'%j]+=(grad_b['b%s'%j])\n",
        "        count+=1\n",
        "\n",
        "        if count%batch_size==0:\n",
        "\n",
        "          for j in range(1,num_hidden_layers+2):\n",
        "            \n",
        "            v_w['W%s'%j] = beta2*v_w['W%s'%j] + (1-beta2)*np.square(dw['W%s'%j])\n",
        "            v_b['b%s'%j] = beta2*v_b['b%s'%j] + (1-beta2)*np.square(db['b%s'%j]) \n",
        "\n",
        "            v_w_hat['W%s'%j] = v_w['W%s'%j]*1/(1-math.pow(beta2,k+1))\n",
        "            v_b_hat['b%s'%j] = v_b['b%s'%j]*1/(1-math.pow(beta2,k+1))\n",
        "\n",
        "            m_w['W%s'%j] = beta1*m_w['W%s'%j] + (1-beta1)*(dw['W%s'%j])\n",
        "            m_b['b%s'%j] = beta1*m_b['b%s'%j] + (1-beta1)*(db['b%s'%j]) \n",
        "\n",
        "            m_w_hat['W%s'%j] = m_w['W%s'%j]*1/(1-math.pow(beta1,k+1))\n",
        "            m_b_hat['b%s'%j] = m_b['b%s'%j]*1/(1-math.pow(beta1,k+1))\n",
        "            \n",
        "            weights['W%s'%j] -= ((m_w_hat['W%s'%j])*(eta/np.sqrt(v_w_hat['W%s'%j] + eps))+(weights['W%s'%j])*wt_decay*eta) \n",
        "            bias['b%s'%j] -= ((m_b_hat['b%s'%j])*(eta/np.sqrt(v_b_hat['b%s'%j] + eps))+(bias['b%s'%j])*wt_decay*eta)\n",
        "\n",
        "            count=0\n",
        "  \n",
        "  #Error and validation functions\n",
        "\n",
        "  def error(y, y_hat):\n",
        "    return -np.log(np.dot(y, y_hat))\n",
        "\n",
        "  def val_accuracy():\n",
        "    y_pred=np.zeros(10000)\n",
        "    fx=np.zeros([10000,10])\n",
        "    val_loss=0\n",
        "    for i in range(10000):\n",
        "      fx[i]=forward_prop(weights,bias,x_val[i])[0]\n",
        "      val_loss+=error(y_val[i],fx[i])\n",
        "    y_pred=np.argmax(fx,axis=1)   \n",
        "    y_val_=np.argmax(y_val,axis=1)\n",
        "\n",
        "    val_accuracy=accuracy_score(y_pred,y_val_)\n",
        "    return val_loss,val_accuracy\n",
        "\n",
        "\n",
        "  #Setting optimization function\n",
        "  if optimization == 'sgd':\n",
        "    sgd()\n",
        "  elif optimization == 'momentum':\n",
        "    momentum_gd()\n",
        "  elif optimization == 'nesterov':\n",
        "    nesterov()\n",
        "  elif optimization == 'rmsprop':\n",
        "    rmsprop()\n",
        "  elif optimization =='adam':\n",
        "    adam()\n",
        "  elif optimization =='adagrad':\n",
        "    adagrad()  \n",
        "  else:\n",
        "    print('Optimization algo given is not available')\n",
        "\n",
        "  val_loss,val_acc = val_accuracy()\n",
        "  print(val_loss)\n",
        "\n",
        "  params={'epochs':epochs,'learning rate':eta,'no of hidden layers':num_hidden_layers,'layer size':size,'batch size':batch_size,'optimizer':optimization,'validation accuracy':val_acc}\n",
        "  #wandb.log(params)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6cQ06i3Dsoc6"
      },
      "source": [
        "#**Run wandb sweep**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wsGcGNQAm5HC",
        "outputId": "925b3e74-2c2c-42cd-80f0-e6bd12ea6bea"
      },
      "source": [
        "Train()"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "6609.484042070051\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p0aucX-F6myg"
      },
      "source": [
        "#sweep_id = wandb.sweep(sweep_config,entity='sanjeevrs',project='assignment1')"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hYPwyi-_qcvi"
      },
      "source": [
        "#wandb.agent(sweep_id, Train)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lqyoBceqAJbo"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}