10_introduction_to_artificial_neural_networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Chapter 10 – Introduction to Artificial Neural Networks**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_This notebook contains all the sample code and solutions to the exercises in chapter 10._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To support both python 2 and python 3\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "def reset_graph(seed=42):\n",
    "    tf.reset_default_graph()\n",
    "    tf.set_random_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"ann\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perceptrons"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Note**: we set `max_iter` and `tol` explicitly to avoid warnings about the fact that their default value will change in future versions of Scikit-Learn."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.linear_model import Perceptron\n",
    "\n",
    "iris = load_iris()\n",
    "X = iris.data[:, (2, 3)]  # petal length, petal width\n",
    "y = (iris.target == 0).astype(np.int)\n",
    "\n",
    "per_clf = Perceptron(max_iter=100, tol=-np.infty, random_state=42)\n",
    "per_clf.fit(X, y)\n",
    "\n",
    "y_pred = per_clf.predict([[2, 0.5]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure perceptron_iris_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsgAAAEYCAYAAABBfQDEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3Xd4FdXWx/HvSiHUSA0qEJAmikgL5YoKFhQUFEVRpFgvgmB5rVhQKSYBEhSVIipWBEFBxAIovUVEUbmgYkFQ6R1CDdnvHyccQ0g3OZOE3+d55vGcNXtm1gkQVyZ79jLnHCIiIiIi4hPkdQIiIiIiIgWJCmQRERERkVRUIIuIiIiIpKICWUREREQkFRXIIiIiIiKpqEAWEREREUlFBbKIiIiISCoBK5DNLMzMXjez9Wa2z8y+M7P2mYz/PzPbbGZ7zWy8mYWl2lfDzOaZ2QEz+8nMLg/MpxARERGRoi6Qd5BDgD+B1sBpwFPAZDOrkXagmV0J9AcuA6oDNYGBqYZMBFYCFYAngQ/MrFI+5i4iIiIipwjzspOemf0ADHTOfZgm/h7wh3PuiZT3lwETnHOnm1ldYBVQ0Tm3L2X/opT9YwP7CURERESkqAnx6sJmVhmoC6xOZ3d9YHqq998Dlc2sQsq+348Xx6n218/gOr2AXgClShVvevbZVfIg+8Jv69bd/PXXjhNi5cpVo0yZCI8yEhEREcnchg0Z74uMzM4xf+DcdsvqOp4UyGYWCkwA3nLO/ZTOkNLAnlTvj78uk86+4/vTrXydc+OAcQBNm9Z2CQnx/yLzomX+/FXccstwtm/fC8CuXX9y9tmXcsstYyhWrITH2YmIiIicqHfvjPc98UR2jonK1nUCvoqFmQUB7wBHgH4ZDNsPhKd6f/z1vnT2Hd+/D8mRNm0akJAQT5MmtfyxhIS3iIu7iJ07M/kRTURERKQIC2iBbGYGvA5UBjo7545mMHQ10DDV+4bAFufcjpR9Nc2sTJr96U3VkCxERlZi3rxoevS4xB/bsOEboqOb8vPP8zzMTERERORE4WlvkWYRz2pfRgL6kJ6ZjQUaAZc75/ZnMq4d8CZwKbARmAosd871T9mfACzGtxJGe+ANoI5zbltm19cUi4w55xg79nMeeuh1kpKOARAUFEznznFceun9+H62ERERESm8eve2b5xzWc6zCOQ6yNWBu/EVyJvNbH/K1s3MIlNeRwI452YCw4B5wAZgPfBMqtPdjG8SyS4gFrghq+JYMmdm9OlzFbNnDyIi4jQAkpOPMWXK/zF+fHeOHDngcYYiIiIigeHpMm+BpjvI2fP33zu46aahLF++1h+rVq0Rd989lYoVz/IwMxEREZHcK3B3kKXwqFKlAnPmPMcdd7T1x/788ztiYqJYs+YLDzMTERERyX8qkCVdYWGhjB3bl9Gj+xAa6lsNMDFxJy+/3I5Zs4ZxKv3mQURERE4tmmIhWUpI+ImbbhrKpk27/LGmTbvQs+d4wsJKeZiZiIiISOYefRT27j3+LgrnVmS58oDuIEuWWrasR0JCPBdccI4/9s03kxk6tCVbt/7qYWYiIiIimfunOM4+FciSLWecUZ7ZswfRu3d7f2zjxv8RG9uM//3vcw8zExEREclbKpAl24oVC+XFF+9m3Lh+FCvmm5d84MBuRo26ms8+e07zkkVERKRIUIEsOXbbbZczb140VatWAHxNRj7++CleeaUzhw6p47eIiIgUbiqQJVeaNatLQkI8F19c3x/77rtpxMa2YPPmnz3MTEREROTfUYEsuRYRUZbPPx/Ivfd28Mc2b/6R2Njm/PDDDA8zExEREfEJD8/5MVrmTfLEhAnz6dNnNIcOHfHHrr76Ga6++mmCgvRzmIiIiHhPnfQkoLp1a8OCBTFUr17JH/v004GMGXMtBw/u8TAzERERkZxRgSx5pnHjWixbFs+ll57vj61a9Qmxsc3ZuHGNh5mJiIiIZJ8KZMlTFSuG88knz/Dgg538sS1b1jJ0aAtWrpzqYWYiIiIi2RPidQJS9ISEBBMbexuNG9eiV6+XOHjwCIcP7+eVVzrTrt0TXHPNIIKCgr1OU0RERDzUu3fG+8aOTT/epw+k9/icGYwZkzd5ge4gSz666aaLWLRoGDVrVvbHZs6MZtSoDiQm7vIwMxERESmMMlpbIq/XnFCBLPnq/PNrsHRpHFdc0dgfW716JjExUfz99yoPMxMRERFJnwpkyXfly5dh+vSnePTRzv7Y9u2/M3RoS1asmOxhZiIiIiInU4EsAREcHMyQIT2YNOlRSpUqDsCRIwd47bWb+PDDRzl2LMnjDEVERER8Alogm1k/M1thZofN7M1Mxo01s/2ptsNmti/V/vlmdijVfvU2LiSuv/4CFi8eRu3aZ/pjX3wxnJdeas/+/Ts8zExERETEJ9B3kDcCQ4DxmQ1yzvV2zpU+vgETgSlphvVLNebsfMpX8kH9+pEsXTqMq676p5HNTz99SUxMFBs2rPQwMxERESnIzHIWz/V1vGg1bWZDgKrOuduyMbYUsBno4JxbkBKbD7zrnHstJ9dVq+mCJTk5mcGD3+e55973x0JDi9O9+2u0aNHNw8xERESkKCpKraY7A9uAhWniMWa23cyWmFmbjA42s14p0zpWbN++Nz/zlBwKCgrimWe68sEHj1OmTAkAjh49xBtvdGfy5P/j2LGjHmcoIiIip6LCUCDfCrztTrzV/RhQE6gCjANmmFmt9A52zo1zzkU556IqVgzP/2wlx665pgVLlgzn7LOr+mNz577AyJFXsHfvVg8zExERkVNRgS6QzSwSaAO8nTrunPvKObfPOXfYOfcWsAS4yoMUJY/Uq1eVJUuGcc01LfyxtWvnExMTxfr1KzzMTERERE41Bb3VdA9giXPu9yzGOSCPp2dLoIWHl2Ty5MeIjf2AgQMn4pxj164/GTHiAm66aRwXXHCb1ymKiIgUOo8+CnvTmWUaHg7DhgU+n0A78fM3bZqdYwK9zFuImRUHgoFgMytuZpkV6T2BN9Oco6yZXXn8WDPrBlwMzMy3xCVggoKCeOKJLnz00ZOcdlpJAA4fPsrbb9/OxIn9SEo64nGGIiIihUt6xXFm8aImN58z0FMsngIOAv2B7imvnzKzyJT1jCOPDzSz/wBVOXl5t1B8S8VtA7YD9wKdnHNrA5C/BEj79lEsXRrHuef6/0qwYMEoXnjhMvbs2exhZiIiIlLUBbRAds4965yzNNuzzrkNKesZb0g1dplzrpRzbl+ac2xzzjVzzpVxzpV1zrV0zn0RyM8hgVGnzpksXjyU66+/wB/79dfFREc35fffEzzMTERERIqyAv2Qnkjp0iWYOPERoqN7EhTk++u6Z89GRoxozaJFr3qcnYiIiBRFKpClwDMzHn74embMGEC5cqUBSEo6woQJvZgw4W6OHj3scYYiIiJSlKhAlkKjbdvGLFsWR4MGNfyxRYvGMWJEG3bv3uhdYiIiIgVYeAZtIDKKFzW5+ZyetJr2ilpNFw2JiYfo3XsU77+/yB8LD69Mr14fULv2hR5mJiIiIgVZUWo1LXKCUqWK8/bbDzJ8+B0EB/v+Cu/du4URIy5h/vzRnEo/9ImIiEjeU4EshZKZcf/91/DZZ89yvIV4cnISkyb15Z137uTo0UMeZygiIiKFlQpkKdQuueR8li2Lo3Hjmv7Y0qVvEBd3ETt3/ulhZiIiIlJYFfRW0yJZql49gvnzY+jbdyzvvjsPgPXrVxAT05S77prM2We38TZBERERDwWi1XRRa2etO8hSJJQoEcbrr9/H88/fRUhIMAD79m3jxRcvY86ckZqXLCIip6xAtJouau2sVSBLkWFm9O3bgVmzBhERcRoAx44lM2XKA7z5Zk+OHDngcYYiIiJSGKhAliLnoovqk5AQT7Nmdfyxr756l+HDL2T79j+8S0xEREQKBRXIUiRVrVqROXOe4/bbL/fH/vxzJTExUfz445ceZiYiIiIFnQpkKbKKFy/G2LF9GTWqD6GhvudRExN38OKLVzJ7dpzmJYuIiEi6VCBLkWZm/Pe/V/Lll0M4/fRyADiXzNSpj/D66105fDjR4wxFRETyVyBaTRe1dtZqNS2njE2bdnLzzcNYtuwnf6xKlQb07j2NSpVqeZiZiIiIBIJaTYukccYZ5fnii8HcfXc7f+zvv1cRExPF6tUzPcxMREREChIVyHJKKVYslJde6s0rr/SlWDHfvOQDB3bz8stX8fnn0ZqXLCIiIiqQ5dR0++1tmTcvmipVKgDgnGP69CcZN+4GDh3a53F2IiIi4qWAzkE2s37AbUADYKJz7rYMxt0GvA4cTBXu4Jybn7K/BvAG0ALYAPRzzmW5dpfmIEtaW7bspmvXYSxevMYfO+OMc+ndexqVK9f1MDMRESlMCnKr5d69M943duzJsdx8lkB9/j59IL3S1QzGjMlOblE4t8Kyuk6g7yBvBIYA47MxdplzrnSqbX6qfROBlUAF4EngAzOrlOfZSpFXuXJZZs0aRL9+HfyxTZvWEB/fmB9++MTDzEREpDApSq2Wc/NZAvX5M7qvm9n93tzkENAC2Tk31Tn3EbAjt+cws7pAE+AZ59xB59yHwCqgcx6lKaeY0NAQRoy4i9dfv5/ixYsBsHfvAUaP7sinnw4iOTnZ4wxFREQkkAryHOTGZrbdzNaa2QAzC0mJ1wd+d86lnij6fUr8JGbWy8xWmNmK7dsL4Y9xEjA9elzC/PnRREb+88uIGTOeYezY6zh4cI+HmYmIiEggFdQCeSFwHhCB785wV+CRlH2lgbTVyh6gTHoncs6Nc85FOeeiKlYspKtVS8A0aVKbZcviuOSSBv7YDz98TGxsczZt+tHDzERERCRQCmSB7Jz73Tm3zjmX7JxbBQwCbkjZvR9IW+mGA1p6QPJEpUqn8emnz/J//3etP7Zly1piY5uzcuU0DzMTERGRQCiQBXI6HHD8icPVQE0zS33HuGFKXCRPhIQEM3To7bz99oOUKOGbl3z48H5eeeV6Pv54AMnJxzzOUERECpKi1Go5N58lUJ/fMlh/IqN4bnMI9DJvIUAI8AxQFfgvkOScS0ozrj3wrXNui5nVAz4ApjjnBqbsTwAWA08B7fEt+VbHObcts+trmTfJje+/X0eXLrGsW7fFH6tfvz133DGBUqXKeZiZiIiI5ERBbTX9FL61jfsD3VNeP2VmkWa238wiU8ZdBvxgZonAZ8BUIDrVeW4GooBdQCxwQ1bFsUhuNWx4FsuWxdG2bSN/bPXqz4mNbcbff//Pw8xEREQkPwT0DrLXdAdZ/o1jx47x9NPvMXz4h/5YWFgpevZ8g6ZNb/QwMxEREcmOgnoHWaTQCg4O5rnnevDee49QqlRxAA4fTuTVV7swdepjmpcsIiJSRIRkPUREUrvhhlbUq1eVLl1i+fXXTQDMnj2MP/9cyZ13TqR06QoeZygiUjQV5HbOgZKbVss5lZuv879vAZ296wSK7iCL5MJ551Vn6dLhtG/f1B/78ccviImJ4s8/v/MwMxGRoqsotXPOrdy0Ws6p3Hyd87IFdEH481SBLJJLZcuWZtq0J3niiS7+2I4dfzBs2AUsX/6eh5mJiIjIv6ECWeRfCAoK4tlnb2HKlP6UKVMCgKNHDzJ+fDemTHmQY8eSsjiDiIiIFDQqkEXywLXXtmTx4mHUrVvFH5sz53lefPEK9u3TCoQiIiKFiQpkkTxyzjnVWLp0OB07NvfHfv55HtHRTVm//hsPMxMREZGcUIEskofCw0syZUp/nnmmK5bS93LXrj8ZPrwVy5a95XF2IiKFW1Fq55xbuWm1nFO5+TrnZQvogvDnqUYhIvnks89WcOutI9iz54A/1qZNP268cQTBwaEeZiYiInJqUqMQEY9ddVUUS5fGcc451fyx+fNf5vnnL2Pv3i0eZiYiIiKZUYEsko/q1DmTxYuHcd11//HHfv11EdHRTVm37isPMxMREZGMqEAWyWdlypRg0qRHGTKkh39e8u7dfxMffzFLlrzucXYiIiKSVrZbTZtZSaAREEGawto5NzWP8xIpUsyMRx/tTKNGNenRI55du/aTlHSEd965iz/++JouXUYSGhrmdZoiIiJCNgtkM7scmAhUSGe3A4LzMimRouqKKxqzbFkcN94Yy6pVfwCwaNEr/P33D/Tq9QFly57pbYIiIsCjj6bf7jc8HIYNC3w+/0bv3hnvGzs2/XifPum3SDaDMWO8PSY3fzY5PaYo/fnnVnanWIwEPgWqOueC0mwqjkVyoGbN01m4MJYuXS7yx37/fRnR0U359dclHmYmIuKTXnGUWbyoyWiBr8wW/grUMbn5s8npMaf6nz9kv0CuAQx2zm3Mx1xEThmlShXnnXceZNiw2wkK8v0z3Lt3MyNHtmHBgjGcSssvioiIFDTZLZCXAGfnZyIipxoz44EHruWzz56hQoUyABw9msTEiffwzjt3cfToIY8zFBEROTVlWCCbWZPjGzAWiDOzu8ysRep9KftFJJcuvbQhCQnxNGpU0x9bunQ8cXEXs3Pnnx5mJiIicmrK7A7yCuDrlP9+ANQDxgHLUmIrUo3JFjPrZ2YrzOywmb2ZybhbzewbM9trZn+Z2TAzC0m1f76ZHTKz/Snbz9nNQaQgql49ggULYujWrY0/tn7918TENGXt2gXeJSYiInIKyqxAPguomfLfzLaaGZ0gHRuBIcD4LMaVBB4AKgItgMuAh9OM6eecK52yafqHFHolSoQxfvz9PP/8XQQH+/5p7tu3jRdeuIy5c1/UvGQRCZjw8JzFi5qUJeuzHQ/kMbn5s8npMaf6nz+AZed/umZ2MbDUOZeUJh4CXOCcW5iji5oNwbcixm3ZHP8gcIlzrmPK+/nAu86513Jy3aZNa7uEhPicHCLiiYUL/0fXrsPZtm2PP9aiRQ+6dXuFYsVKeJiZiIhI4dW7t33jnIvKalx2H9KbB5RPJ35ayr78djGwOk0sxsy2m9kSM2uT0YFm1itlWseK7dtPofVJpFC7+OLzSEiIIyqqjj/21VfvMHx4K3bsWO9hZiIiIkVfdgtkw9cQJK0KQGLepZPOhc3uAKKAuFThx/BN7aiCb170DDOrld7xzrlxzrko51xUxYqn0O8GpNCrVq0Sc+c+x623XuaP/fnnSqKjm/LTT3M9zExERKRoy7RANrOPzexjfMXxu8ffp2yfAl8AS/MrOTPrBMQA7Z1z24/HnXNfOef2OecOO+fewrcM3VX5lYeIV4oXL8a4cf146aW7CQ31PaeamLiDkSPb8sUX8ZqXLCIikg+yajW9I+W/BuwCDqbadwRYDLyaD3lhZu1Szn21c25VFsNdSo4iRY6Zcffd7WnQoAY33zyMzZt34VwyH374MOvXr6BHj9cICyvldZoiIvkuEG2WA5lbQb5OThXUvHIr0wLZOXc7gJn9AcQ55/7VdIqUh/pCgGAg2MyKA0npPPx3KTABuM45tzzNvrL4VrZYACQBN+Gbo3z/v8lNpKC74IJzSEiI5+abh5KQ4FvZcMWKSWzatIbevadRqVJOFpQRESl8AtFmObeK2nVyqqDmlVvZmoPsnBv4b4vjFE/huwvdH+ie8vopM4tMWc84MmXcAHwPAH6Waq3jz1P2heJbKm4bsB24F+jknFubB/mJFGhnnlmeL74Ywn//e6U/9vffPxATE8Xq1bM8zExERKToyPAOspmtI/0H807inMvWrSvn3LPAsxnsLp1q3CWZnGMb0Cw71xMpisLCQhk1qg9Nm9bmvvte4ciRJA4c2MXLL7fn2mujufLKx7DMFtEUERGRTGV2B/llYFTK9ha+FSt+A95N2X5Lib2ZvymKSHruuKMtc+dGU6VKBQCcc3z00eO8+moXDh3a53F2IiIihVeGd5Cdc/6OGiltoYc656JTjzGzx4H6+ZadiGSqefO6JCTE07XrMBYvXgPAt99+kDIv+SMqV66TxRlEREQkreyug3w9MDmd+BTgmrxLR0RyqnLlssyaNYi+fa/2xzZtWkNsbDNWrfrUw8xERPJWINos51ZRu05OFdS8ciu7raY3AQPStnY2s7uAIc650/MpvzylVtNS1L399lz69h3D4cNHAd8ScR06DKR9+ycJCsruz8MiIiJFU163mn4eGGVmY83stpRtLPBSyj4RKQB69ryUBQtiqFatIuCblzxjxtO88sr1HDxYSNfaERERCbDsLvM2DOgBNABGpGwNgFudc0PzLz0RyakmTWqTkBBP69bn+WPffz+d2NjmbN78k4eZiYiIFA7Z/p2rc26yc66Vc658ytbKOZfevGQR8VilSqfx+ecDeeCBfx4R2LLlZ2Jjm/Pddx95mJmIiEjBl1WraREppEJCghk27A4aN65F796jOHjwCIcO7WPs2Ou46qoBdOjwrOYlyymhqLXALSoKcttokQz/72hme82sYsrrfSnv090Cl66I5FTXrq1ZsCCWGjUi/LHPPhvM6NEdOXBgt4eZiQRGUWuBW1QU5LbRIpndQb4X2Jfqdba66olIwdOoUU2WLYujR494vvzyewD+97/PiIlpRp8+H3HmmVrOXERE5LjMGoW8ler1mwHJRkTyTYUK4cyY8TQDBkwgLm4qANu2/Up8fBS33PIOTZve4HGGIiIiBUO2JiCa2RNm9h8z05xlkUIsODiY6OieTJjwMKVKFQcgMfEQr756I9OmPU5y8jGPMxQREfFedp/QaQ/MA3aZ2eyUgvkCFcwihdONN17IokVDqVXrnx4/s2bF8vLLV5GYuNPDzERERLyX3XWQLwLKAdcBX+ErmOfgK5hn5V96IpJfzjuvOkuXxtGuXRN/bM2a2cTERPHXX997mJlI3ipqLXCLioLcNlokW62mTzjArDJwKXA10AVIcs6VzIfc8pxaTYuc7NixYwwaNImYmCn+WGhoCXr2fJ1mzbp6mJmIiEjeytNW02bWxcxGm9mPwO/Af4FfgLb47iyLSCEVHBzMwIHdmDy5P6VL++YlHz16kNdfv4UPPniIY8eSPM5QREQksLI7B3kS0BkYD1Ryzl3qnBvonFvgnDucf+mJSKB06tSSJUuGU7duFX/syy9H8OKLV7Jv3zYPMxMREQms7BbIvYDZ+NZD3mhmM8zsITNrYmaWf+mJSCCdc041liwZRocOzf2xn3+eS0xMFBs2fOthZiIiIoGTmznItYA2+KZXXAfsd85VyOax/YDbgAbAROfcbZmM/T/gMaAk8AHQ5/jdajOrAbwBtAA2AP2cc19mdX3NQRbJnuTkZKKjpzBo0ER/LDS0ON26jaNlyx4eZiZSdPTpA+n9L9gMxowpfNcpqG2g1dJaUsvTOcgAZhZkZi2AG/A9nNcBMGBtDvLaCAzBN1Ujs2tdCfQHLgOqAzWBgamGTARWAhWAJ4EPzKxSDvIQkUwEBQXx1FM3MW3ak4SH+57BPXr0EG++2ZP337+fY8eOepyhSOGX0f2pHN63KjDXKahtoNXSWnIjuw/pfQ7sAhYBnYBv8c1JLuec+092L+acm+qc+wjYkcXQW4HXnXOrnXO7gMH47jxjZnWBJsAzzrmDzrkPgVUp+YhIHrr66mYsXTqcevWq+mPz5r3ICy9czt69WzzMTEREJP9k9w7yd/juGpdzzv3HOfe4c26Wcy4xn/KqD6ReiPV7oLKZVUjZ97tzbl+a/fXTO5GZ9TKzFWa2Yvt2/egnklN161ZhyZLhdOrU0h/75ZeFREc3Zd265R5mJiIikj+y2ygkvwvitEoDe1K9P/66TDr7ju8vk96JnHPjnHNRzrmoihW1krhIbpQpU4L333+MwYO7c/y53N27/yY+/iKWLHnd4+xERETyVrbnIAfYfiB1NXv89b509h3fvw8RyTdmxmOP3cDHHw+gbNlSACQlHeGdd+7ivff6kJR0xOMMRURE8kZBLZBXAw1TvW8IbHHO7UjZV9PMyqTZvzqA+Ymcsq68sgnLlsVx3nnV/bGFC8cyYsQl7NmzycPMRAqXjBZJzevFUwN1nYLaBlotrSU3crzM27+6mFkIEAI8A1TF15EvyTmXlGZcO+BNfC2tNwJTgeXOuf4p+xOAxcBTQHt8S77Vcc5l2s1Ay7yJ5J3ExEP06vUyU6Ys9sdOO+0MevX6gFq1LvAwMxERkfTl+TJveeQp4CC+Jdy6p7x+yswizWy/mUUCOOdmAsOAefjWOV6Pr6g+7mYgCt/KGrHADVkVxyKSt0qVKs677z5EbOxtBAX5vpXs2bOJESPasHDhKwTyh28REZG8FNA7yF7THWSR/DFnzvd07x7Hjh3/PArQqtWd3Hzzy4SGFvcwMxERkX/86zvIZrbPzPZmZ8vb1EWksLnssoYsWxZHw4Zn+WNLlrxOfHxrdu36y8PMREREci4kk339ApaFiBR6NWpUZsGCWPr0Gc3EiQsA+OOP5URHN6VXrynUqXOxxxmKiIhkT4YFsnPurUAmIiKFX8mSYbz55gNERdXm0Uff4NixZPbt28rzz1/GjTc+T5s2ff3rKIuIiBRUBXWZNxEppMyMe+/tyMyZA6lU6TQAkpOTeP/9e3nrrds4cuSgxxmKiIhkLlsFspkVM7OBZrbWzA6Z2bHUW34nKSKFT+vWDUhIiKNp09r+WELC28TFXciOHes9zExERCRz2b2DPBi4FYgHkoFHgFHADuCe/ElNRAq7atUqMW9eNLfeepk/tmHDt8TERPHzz/M8zExERCRj2S2QuwC9nXOvAMeA6c65+/CtTdw2v5ITkcKvePFijBvXjxdf7EVISDAA+/dvZ+TItnz55QitlywiIgVOdgvkysCalNf7gbIpr2cCV+R1UiJStJgZvXtfxRdfDKZyZd+3j+TkY3zwwUOMH9+NI0cOeJyhiIjIP7JbIG8Azkx5/StwZcrr/+DrhicikqVWrc4lISGeFi3O9se+/noiw4ZdwPbt6zzMTERE5B/ZLZCnAccnEY4EBprZOuBN4LV8yEtEiqgqVSrw5ZdDuOuuf3759Ndf3xMTE8WaNbM9zExERMQnWwWyc+5x59xzKa8/AC4EXgKud849mY/5iUgRFBYWyujR9zBmzD0UK+Zbjj0xcScvvdSeWbOGal6yiIh4KrvLvF1sZv6mIs65r5yywB9UAAAgAElEQVRzI4CZZqb2WCKSK3feeQVz5jzHmWeWB8C5ZKZN68+rr97EoUP7Pc5OREROVdmdYjEPKJ9O/LSUfSIiudKixdkkJMTTqtU5/ti3305h2LCWbN36q4eZiYjIqSq7BbIB6f3OswKQmHfpiMip6PTTyzFr1iD69LnKH9u4cTWxsc1YteozDzMTEZFTUaYFspl9bGYf4yuO3z3+PmX7FPgCWBqIREWkaCtWLJSRI3vx2mv3EhYWCsCBA7sZPboDn302hOTkZI8zFBGRU0VWd5B3pGwG7Er1fgfwFzAW6J6fCYrIqaVnz8uYPz+GatUqAuCc4+OPB/DKK505eHCvx9mJiMipwLLztLiZPQPEOecK9XSKpk1ru4SEeK/TEJFs2Lp1N7fcMpyFC1f7Y6efXo/evT/i9NPPzuRIERGR9PXubd8456KyGpfdZd4GOucSzSzKzG4ys1IAZlYq9eoWIiJ5JSKiLJ9/PpD77uvoj23e/BOxsc34/vuPPcxMRESKuuwu81bZzBKA5cB7+FpPA4wAsn1L1szKm9k0M0s0s/VmdksG4z43s/2ptiNmtirV/j/M7GCq/eouIFIEhYaGEBd3J2+++X+UKFEMgEOH9jFmzLXMmPGM5iWLiEi+yO4qFs8DW/CtWnEgVXwKcEW6R6RvFHAEX4HdDRhjZvXTDnLOtXfOlT6+4XsQcEqaYR1TjclJDiJSyNxyS2vmz4+levVK/tinnw5izJhrOXBgt4eZiYhIUZTdAvky4Enn3K408d+AyOycIGVaRmdggHNuv3NuMfAx0COL42oAFwFvZzNXESmCGjeuybJl8Vx2WUN/bNWqT4iNbc7GjWs8zExERIqa7BbIJfDd+U2rEnAom+eoCyQ559amin0PnHQHOY2ewCLn3B9p4hPMbJuZzTazhukcB4CZ9TKzFWa2Yvt2PQEvUphVrBjOjBlP8+CDnfyxrVt/YejQFnz77YceZiYiIkVJdgvkhcBtqd47MwsGHgPmZPMcpYG0FeoeoEwWx/UE3kwT6wbUAKrj6+Q3y8zKpnewc26ccy7KORdVsWJ4NlMVkYIqJCSY2NjbePfdhylZMgyAw4f3M27cDXz00RMkJx/zOEMRESnsslsgPwr818y+AMLwPZi3BmgFPJ7Nc+wH0lao4cC+jA4wswuB04EPUsedc0uccwedcwecczHAbnzTMETkFNGly4UsWjSUmjUr+2MzZ8bw8stXk5i408PMRESksMvuMm9rgPOBZcBsoDi+h+YaO+d+y+a11gIhZlYnVawhsDqD8QC3AlOdc/uzShFfMxMROYU0aFCDZcviufLKJv7YmjWziIlpxl9//eBhZiIiUphl9w4yzrlNzrmnnXMdnHNXOeeecs5tysHxicBUYFDK+smtgGuBd9Ibb2YlgC6kmV5hZpFm1srMiplZcTN7BKgILMluLiJSdJQrV5qPPnqSxx67wR/bvv13hg37D19/PcnDzEREpLDKtEA2s5JmNsrM/jazrWb2nplV/BfXuwffA39bgYlAH+fcajO7yMzS3iXuhG/qxLw08TLAGHytr/8G2gHtnXM7/kVeIlKIBQcHM3hwd95//zFKly4OwJEjB3j99a58+OEjHDuW5HGGIiJSmGTaatrMhuMraifgW62iKzDfOXdjYNLLW2o1LVL0rVnzJzfcEMOvv270x+rVu4y77ppE6dL/5ud7EREp7PKq1fT1wJ3OuV7OufuAq4FOKStYiIgUOOeeW41ly4Zz9dXN/LGffppDdHQUGzas9DAzEREpLLIqkKsBi46/cc4tB5KAM/MzKRGRf+O000rx4YePM2DAzf7Yzp3rGT78AhIS0n3sQURExC+rAjmYkxuEJAEh+ZOOiEjeCAoKYsCAm/nwwycIDy8JwNGjh3jzzZ5MnvwAx44d9ThDEREpqLIqkA1418w+Pr7hW+Lt1TQxEZECqWPH5ixZMpx69ar6Y3PnjuSFF9qyd+9WDzMTEZGCKqsC+S1gI7Aj1fYu8GeamIhIgXX22VVYsmQ4117b0h/75ZcFxMQ05Y8/vvYwMxERKYgyXcWiqNEqFiKntuTkZIYN+5BnnnmP49/7QkLCuOWWMVxwwe0eZyciIvktr1axEBEpMoKCgujf/0amT3+KsmVLAZCUdJi3376DiRP7kpSU9pELERE5FalAFpF0bd26gBUr/suSJdexYsV/2bp1gdcp5Zl27ZqydGkc9etH+mMLFozm+ecvZc+ezR5mJiIiBYEKZBE5ydatC/jtt9EcPrwNcBw+vI3ffhtdpIrk2rXPYNGiodxwQyt/7LfflhAd3YTff1/mYWYiIuI1FcgicpING94lOfnwCbHk5MNs2PCuRxnlj9KlSzBhwsPExNxKUJDv2+GePZuIj2/NokXjPM5ORES8ogJZRE5y+PD2HMULMzPjoYeu45NPnqZ8+TIAHDt2lAkT7ubdd3tx9OjhLM4gIiJFjQpkETlJWFjFHMWLgssvb8SyZXE0bHiWP7Z48auMGNGaXbv+9jAzEREJNBXIInKSyMjuBAWFnRALCgojMrK7RxkFxllnVWbBgli6dm3tj61b9xUxMU355ZdFHmYmIiKBpAJZRE4SEdGaWrXuISysEmCEhVWiVq17iIhoneWxhV3JkmG8+eYDxMXdQXCw71vk3r1beP75S5k/fxSn0trxIiKnqhCvExCRgikiovUpURCnx8y4775raNjwLLp2Hc727XtJTk5i0qR+rF+/gltuGUNoaHGv0xQRkXyiO8giIhlo3boBCQnxNGlSyx9btuxN4uIuYufODR5mJiIi+UkFsohIJiIjKzFvXjQ9elzij61fv4Lo6Kb8/PM8DzMTEZH8ogJZRCQLJUqE8dpr9zFyZC9CQoIB2L9/OyNHtmXOnBc0L1lEpIgJaIFsZuXNbJqZJZrZejO7JYNxz5rZUTPbn2qrmWp/IzP7xswOpPy3UeA+hYikpyi3pgbfvOQ+fa5i9uxBVK5cFoDk5GNMmfJ/jB/fnSNHDnicoYiI5JVA30EeBRwBKgPdgDFmVj+Dse8750qn2n4HMLNiwHTgXaAc8BYwPSUuIh44FVpTH3fhhfVJSIinefO6/tjXX7/H8OGt2L59nYeZiYhIXglYgWxmpYDOwADn3H7n3GLgY6BHDk/VBt/qGy845w47514EDLg0L/MVkew7VVpTH1elSgXmzHmOO+9s64/9+ed3xMREsWbNFx5mJiIieSGQd5DrAknOubWpYt8DGd1B7mhmO81stZn1SRWvD/zgTpz090NG5zGzXma2wsxWbN++99/kLyIZOJVaUx8XFhbKmDF9GT26D6GhvhUzExN38tJL7Zg1a5jmJYuIFGKBLJBLA2kr1D1AmXTGTgbOASoB/wWeNrOuqc6zJ5vnwTk3zjkX5ZyLqlgxPLe5i0gmTsXW1MfdddeVzJkzhDPOKAeAc8lMm/YYr712M4cPJ3qcnYiI5EYgC+T9QNoKNRzYl3agc26Nc26jc+6Yc24pMBK4IafnEZHAOFVbUx/XsmU9EhLiueCCc/yxb76ZzNChLdm69VcPMxMRkdwIZCe9tUCImdVxzv2SEmsIrM7GsQ7fPGNSxj9kZpZqmsX5+B4AFBEPHO+4t2HDuxw+vJ2wsIpERnY/pTrxnXFGeWbPHsTDD49n7NjPAdi48X/Exjbjjjve47zz2nucoUjBYJZMRMR2KlfeTXDwMa/TkSLk2LFgtmwpy9atFXHu390DtkDOkzOzSfiK3buARsBnwAXOudVpxl0LLAR2A82AacATzrm3Ular+AUYAYzFNwXjEaCOc+5IZtdv2rS2S0iIz9sPJSKSxltvzaFfv7EcPnwU8C0Rd801Q2jX7nHMLIujRYq2WrU2cMYZRvnylQkODtW/CckTzjmOHTvKzp1b2LTJ8dtvkemO693bvnHORWV1vkAv83YPUALYCkwE+jjnVpvZRWa2P9W4m4Ff8U2beBsY6px7CyClCO4E9MRXQN8BdMqqOBYRCZRbb72MefOiqVq1AuD7xj19+pO88kpnDh3SbDA5tYWHJ1KpUhVCQoqpOJY8Y2aEhBSjUqUqhIf/++c/AlogO+d2Ouc6OedKOecinXPvpcQXOedKpxrX1TlXIWX943opS7mlPs9K51xT51wJ51wT59zKQH4OEZGsREXVISEhnosv/meBne++m0ZsbAs2b/7Zw8xEvGemRr6SP/Lq75b+hoqI5JOIiLJ8/vlA7r23gz+2efOPxMY254cfZniYmYiIZCaQD+mJSBpbty4IyINtq1Y9zd69P/jfh4efT4MGg/I0t0B9lkBdJ6+EhoYQH38XTZrUpk+f0Rw6dIRDh/YyevQ1dOjwLFddNYCgIN2rEBEpSPRdWcQjgWrPnLY4Bti79wdWrXo6z3IL1GcpzC2tu3Vrw4IFMVSvXskf++STZxk7thMHD6Zd2l1EJPs6dWpD//79vE6jSFGBLOKRQLVnTlscZxXPTW6B+iyFvaV148a1WLYsnksvPd8f++GHGcTGNmfjxjUeZiYiWbn33tuIiDDi4wefEF+yZD4REcaOHdnvHJrdgvbee2+jW7cOWY57442pPPVUTLavn9aBAwd47rknaN68NtWqFadevYpcfXUrpk6dmO1zbNjwBxERxnffrch1HgWJCmQRjxTk9sw5zS1Qn6Ugf82yq2LFcD755BkefLCTP7Zly1qGDm3BypVTPcxMpHCoXx8iIk7e6tfP+th/q3jx4owaNZzt27fl/8Wy4cgR3wJe5cqVp3TpdBsKZ8sjj/Tmo4/eZ8iQF1iy5CemTPmCG27ozq5dO/Mq1UJHBbKIRwpye+ac5haoz1KQv2Y5ERISTGzsbbzzzkOULOnrQHj48H5eeaUzH330JMnJap4gkpFtGdSmGcXzUqtWl1CtWg1GjBic6bhlyxbSrl0LqlUrzrnnVmbAgP/zF7P33nsbS5cuYPz4UUREGBERxoYNf2Tr+sfvKL/44lAaNqxKo0ZVgZPvSH/yyVRatz6fyMgS1K1bnmuvbc3WrVsyPO+sWR9z//2Pc8UVHYiMrEGDBo25/fY+3HlnX/8Y5xwvvTSMZs1qERlZgtatGzBlyj+/vYuKOguAK65oRkSE0alTGwCSk5OJjx9Mo0bVqFo1jNatG/D559NPuH5c3CCaNKlO1aph1K9/On379vTvmzt3Jh07XkSdOuWoW7c8Xbpcydq1P2br6/VvqEAW8Uig2jOHh5+fo3hucgvUZylqLa1vuukiFi4cSs2alf2xmTOjGTWqA4mJuzzMTETSExQUxIABsbz11ljWrfst3TGbNv1N167tOe+8xsyZs5IXXnidqVMnMmTI4wA899xIoqL+Q9eut7Nq1SZWrdpElSrVsp3D0qULWLPmByZNmskHH8w5af+WLZu5++6buemmW1m8+EemT1/IjTf2yPScERGnM3fuTPbuzfh5iJiYp3jvvdcZOnQUixat4b77HueRR+7miy8+BWDWrOUATJo0k1WrNvHGG77fiI0bN5JRo4YzYMBQFixYRfv213H77dezatV3AMyY8SGjR8cxdOhoEhJ+YcKET2jSpLn/uomJifTq9QCzZi1n2rT5hIefRvfuHf0/cOQXrWIh4pFAtWdu0GBQjlexyGlugfosRbGl9fnn12Dp0jh69hzB7Nm+Jd1Xr55JbGwzeveeRpUqDTzOUERSu/zyq2jevBUxMU8ybtykk/a/8cZoKlc+k2HDRhMUFETduucwYEAsDz98N/37DyY8/DSKFStGiRIlqVz59Bxfv3jx4owcOZ6wsLB092/ZspGjR4/SseMNVKtWHYBzzjkv03PGx4+jT59u1KtXkXPOaUCzZhfQrt21tGnTFvAVqWPHjmDy5Nm0bHkRANWrn8XKlcsZP34UbdteTYUKvgeQy5evcMLnGj06jnvueZjOnW8BoH//QSQkLGT06DjGjHmXv/5aT+XKZ9CmzRWEhoZStWokjRr90+iuY8fOJ+Q6cuQb1KoVzrffLqdlywtz8qXLERXIIh6KiGgdkOIuqyXd0pPT3AL1WQJ1nUAqX74M06c/xTPPvMewYR8CsG3bbwwd2pKePd8gKqqLxxmKSGoDBgzlqqv+Q9++j5y0b+3aH2natOUJyzc2b34hR44cYd26X6lfP+Pf3mVHvXrnZVgcA9Sv35CLL76ciy8+jzZtruDiiy+nY8cbqFixEn/9tYELLzzXP/aBB57ggQee4D//uZivv/6db75JYPnyJSxaNJcuXa6gR49exMe/wtq1azh06BA339wO+Kf7YVLSUapVq5FhLvv27WXz5o00b97qhHiLFhfy5ZefAXDNNTfy6qsjiYo6i0suuZJLL23HlVde4/+M69b9xtChA/jmm6/YsWMbycnJJCcn8/ffG3Lx1cs+TbEQESkAgoODGTKkB5MmPUqpUsUBOHLkAK+9dhMffvgox44leZyhiBzXpElzOnTozKBBj+bouLxorV2yZKlM9wcHBzNlymwmT57Nueeez3vvvU7LlnX43/++5/TTz2Tu3O/826239vYfFxoaSsuWF3Hfff2ZMmU2/fsP5p13xrFhwx8kJycD8M47M044fuHC1UyePDtXn+P416JKlWosXfozcXGvUKZMOM888xBt2zYlMdHXLrp79w5s376NuLhXmDnzK+bOXUlISAhHj+bvFAsVyCIiBcj111/A4sXDqF37TH/siy+G89JL7dm/f4eHmYkUDJUq5SyeX554IpqEhEXMnTvzhHjduufwzTcJ/qISYPnyxRQrVowaNWoBEBpajGPH8u9hXDOjWbP/8MgjzzB79tecfvqZTJ/+PiEhIdSsWdu/lStXPsNz1K3ru9OcmLifs88+l7CwMP76a/0Jx9esWds/jaNYsWIAJ3yuMmXCOf30M1m+fMkJ5/7qq8X+84Nv2kjbtlczePDzzJr1NT/9tJrly5ewc+cOfvnlJx544Alat76cunXPYf/+fSQl5f8NA02xEBEpYOrXj2Tp0mHcdtsLfPaZb03Rn376kpiYKHr3nka1ao08zlDEO6tXe52BT82atenRoxevvjryhPjtt9/DuHEv8Oij99Cr1/2sX/87gwf35447+lGyZEkAIiNrsHLlcjZs+INSpUpTrlz5POuouWJFAgsXfskll1xJpUqVWbVqJX///ecJBWlanTq14brrutKoURTlylVg7do1REc/QZ069ahb9xyCg4O5556HefbZh3HO0bLlxSQm7uebbxIICgqiZ89eVKwYQYkSJZg3bxbVqtWgePHihIefRt++jzB06NPUrFmHhg2bMmXKuyQkLOLLL78FYNKkN0lKSqJJkxaUKlWa6dPfJzQ0lJo161C2bDkqVKjIu+++yplnVmPz5r8ZOPARQkLyv3xVgSzioV9/HcuWLbOBZCCIypWvoHbt3pkeE4i20blR2FpAF3Rly5Zm6tQnGDz4fZ577n0Aduz4g2HDLqB791dp0aKbxxmKyEMPPc377791QuyMM6owceLnDBz4CJde2ojw8LJ07nwLTz4Z7R9zzz0P06/frVx00bkcPHiQFSvWERlZI09yCg8/jeXLl/Daay+xd+9uzjyzGg8+OIAbb8x4tZ9LLrmSKVPeISbmSRIT9xMRcTqtW7floYeeJjg4GID+/QdTqVJlRo+O49FH+1CmTDj16zeiXz/fNJOQkBCee+5F4uMHERc3kJYtL+Kjj+bz3//ex/79+xg06FG2bdtC7dpnM378h5x3XsOUfMvy0ktDefbZh0lKOkrduufyxhtTqV7dt2zcuHHv8+ST99G69XmcdVZtnn02njvu6Jz+B8lD5pzL94sUFE2b1nYJCfFepyECHC+OZ54Ur1y5XYZFcnptoyHzIvl4e+bUHeiCgsKoVeuePCtgA3GNU9mMGcu57bbn2bfvoD926aUP0LnzMIKDQz3MTCTnGjf+kbPOOsfrNKQIW7fuR1auTP/vWO/e9o1zLirdnaloDrKIR3x3jrMfh8C0jc6Nwt4CuqDr2LE5S5YM5+yzq/pjc+e+wMiRV7B371YPMxMRKZpUIIt4JjmH8dwJRHvmotACuqCrV68qS5YM45prWvhja9fOJyYmivXrV3iYmYhI0aMCWcQzGf3zy9t/loFoz1xUWkAXdOHhJZk8+TEGDuzmXyJp164/GT78QpYufdPb5EREihAVyCIeqVz5ihzFITBto3OjqLWALsiCgoJ4/PEb+eijJzntNN8T8UlJh3n77duZOLEfSUn5uzaoiMipIKAFspmVN7NpZpZoZuvN7JYMxj1iZv8zs31mts7MHkmz/w8zO2hm+1O23K1SLeKh2rV7U7lyO/75ZxiU6QN64OuIl7YYzk7b6Fq17iEsrBJghIVVyvOH5wJxDTlR+/ZRLFsWz7nnRvpjCxaM4oUXLmPPns0eZiYiUvgFdBULM5uIrxq4E2gEfApc4JxbnWbco8CXwA9ALWA28JhzblLK/j+Au5xzX+bk+lrFQkSKmv37D3LXXS8xdepSf+y0087k7rs/pGbNlh5mJpI+rWIh+a1QrWJhZqWAzsAA59x+59xi4GOgR9qxzrlhzrlvnXNJzrmfgelAq7TjREROdaVLl2DixEeIju7pbzSwZ89GRoxozaJFr3qcnYhI4RTIKRZ1gSTn3NpUse+B+pkdZL4nUS4C0vbOmWBm28xstpk1zOT4Xma2wsxWbN++N7e5i4gUWGbGww9fz4wZAyhXrjQASUlHmDChFxMm3M3Ro4ezOIOIiKQWyAK5NJC2Qt0DlMniuGfx5flGqlg3oAZQHZgHzDKzsukd7Jwb55yLcs5FVawYnou0RUQKh7ZtG7NsWRznn1/DH1u0aBwjRrRh9+6N3iUmIlLIBLLV9H4gbYUaDuzL6AAz6wf0BC5yzvlvgTjnlqQaFmNmt+K7yzwj79KVoiJQLZBz0zb6m2/u5dChP/3vixevRtOmL2V6zJIlnYFjqSLBtGr1YRbHdAFSr25QjFatJmd6zFdf3UFS0k7/+5CQ8rRoMT7D8YH6OquldeZq1jydhQuHcvfdL/P++4sAWLcugejoJvTq9QG1a1/ocYYiRVenTm2oV+88YmNf9joV+ZcCeQd5LRBiZnVSxRpy8tQJAMzsDqA/cJlz7q8szu0Ay5MspUg53gL58OFtgOPw4W389ttotm5dkKfX+adt9PEmH8ls2TKTX38dm+ExaYtjgEOH/uSbb+7N8JiTi2OAYynxjI5JWxwDHEmJpy9tcQyQlLSTr766I93xgfo6B+o6hV3JkmG8/faDDB9+B8HBvm/ze/duYcSIS5g/fzSBfDhbpKi4997b6NatQ6Zj3nhjKk89FZPraxw4cIDnnnuC5s1rU61acerVq8jVV7di6tSJ2T7Hhg1/EBFhfPedGgj9GwErkJ1zicBUYJCZlTKzVsC1wDtpx5pZNyAaaOuc+z3Nvkgza2VmxcyseMoScBWBJWnPIxKoFsi5aRudtjjOKu6TtjjOKg4nF8dZxTmpOM4qHqivs1paZ5+Zcf/91/D55wM5Pr0sOTmJSZP68s47d3L06CGPMxTJvd27J7B2bQ1Wrw5i7doa7N49wdN8jhzxfT8tV648pUtnNXM0Y4880puPPnqfIUNeYMmSn5gy5QtuuKE7u3al/71X8k+gG4XcA5QAtgITgT7OudVmdpGZ7U81bghQAfg61VrHx2/FlQHGALuAv4F2QHvn3I6AfQopNALXAjkwbaMLqkB9ndXSOufatGlAQkI8jRvX9MeWLn2DuLiL2Lkzsx/GRAqm3bsnsHFjL44eXQ84jh5dz8aNvQJaJB+/m/zii0Np2LAqjRpVBXxTLPr37+cf98knU2nd+nwiI0tQt255rr22NVu3bsnwvLNmfcz99z/OFVd0IDKyBg0aNOb22/tw5519/WOcc7z00jCaNatFZGQJWrduwJQp/9wkiIo6C4ArrmhGRITRqVMbAJKTk4mPH0yjRtWoWjWM1q0b8Pnn00+4flzcIJo0qU7VqmHUr386ffv29O+bO3cmHTteRJ065ahbtzxdulzJ2rU/5v6LWMAFtEB2zu10znVyzpVyzkU6595LiS9yzpVONe4s51yoc650qq13yr7VzrnzU85RwTl3mXNOv0eQdAWuBXJg2kYXVIH6Oqulde5ERlZi/vwYune/xB9bv34FMTFNWbtW01OkcNm69UmcO3BCzLkDbN36ZEDzWLp0AWvW/MCkSTP54IM5J+3fsmUzd999MzfddCuLF//I9OkLufHGk1a2PUFExOnMnTuTvXv3ZDgmJuYp3nvvdYYOHcWiRWu4777HeeSRu/nii08BmDVrOQCTJs1k1apNvPHGVADGjRvJqFHDGTBgKAsWrKJ9++u4/fbrWbXqOwBmzPiQ0aPjGDp0NAkJvzBhwic0adLcf93ExER69XqAWbOWM23afMLDT6N7947+u+dFTSAf0hMJuMjI7vz22+gTfi2fHy2QK1e+ImUO8snxjBQvXi3d6RTFi1fL5ErBpD+dIjiTY4qR/nSKYhkeERJSPt3pFCEh5dMdH6ivc6CuUxSVKBHG66/fR1RUbR5+eDxJScfYt28bL7xwGZ07x3PppffhW1VTpGA7enRDjuL5pXjx4owcOZ6wsLB092/ZspGjR4/SseMNVKtWHYBzzjkv03PGx4+jT59u1KtXkXPOaUCzZhfQrt21tGnTFvAVqWPHjmDy5Nm0bHkRANWrn8XKlcv5//buPbyq4l7j+PclCQGBiAiJgHIHL3AKCig+FtBzRK0WL8ejUkHrpSoiVY54t1RAEfEEtVpRUZEq9VKeomCtSC0q1YqA0mrxgqVq5CaoVUiAEMycP9YibGISIMBekLyf51kPYdbM5Leyd3Z+e/asmUmT7qdfv1PYf/9mADRpsj95eQeU9T1hQj5DhlzDmWdGmxjfcMNo5s6dw4QJ+TzwwBSWLv2MvLzmHHvsCWRlZXHgga3o1m3LfnKA/qwAABF0SURBVBr9+299r8uvfvUY7dvn8M478+jVq+bd/Fs7hres1krXFsjV2Ta6e/f7vpcMb2sVi2i1ivLJcNWrWESrVZRPhqtexeKooyZ9LxmuahWLdP2cvaX1zpHEkCGn8NJLo8nN3ReA0tLvmDp1GJMnn8/Gjeu20YNZ8rKyWu1Q+e5yyCFdKk2OATp37kqfPsfTp08XLrzwTB577AG+/HI1AEuXFtCmTcOy4557bgfg6KP7MH/+v5g2bTannXY2S5Ys5uyzT2D48MsAWLz4fTZs2MCAASdt1X7y5Af49NMllcaydu0aVq5czpFHbr3n2lFH/ZDFi98H4NRTz6K4eAM9erRl2LCLmTFjKsXFWwYjPvlkCYMHn0vPnu1p1y6Hzp3zKC0tZdmy9L4xSRePIFuNl5vbNy0JVIcOg7e5rFt521rSrSLbWtKt4jZVL+lWkaqWdKtIun7O6fo+NVnv3p2ZO3c855wzjvnzPwbgrbemsHz5Ii67bBpNm7ZJNkCzKuTmjmH58ku3mmYh7UNu7pi0xrHPPg2qPJ+RkcHUqbNYsGAur746iyeffJQxY27kuede45BDOjN79t/K6u6335YBiaysLHr16k2vXr258sobuOuu27jjjhFcddWNlJZG97U88cTztGy59RuCrKysal3H5k+OWrY8iL/+9SP+8pc/M2fOy9xyy3Dy80fx4otv0aBBAwYN+jHNmx9Ifv5DNG/ekszMTH74w8MoKamZUyw8gmxmVgsdeGBTZs++nQsvPL6s7PPPFzJ2bA8++ODlBCMzq1rjxgNp0WIiWVmtAZGV1ZoWLSbSuPHApEP7Hkn07Hk01157C7NmzeeAA1owffozZGZm0q5dh7IjNUEur1OnwwAoKirk4IMPIzs7m6VLP9uqfbt2HcqmcdStG31i+N13W6bjNWqUwwEHtGDevK0X/HrrrdfL+odo2ki/fqdw661389JL8/nww0XMm/cGX3/9FR9//CHDht1E377H06nToRQWrmXTpk277Ge1p/EIsplZLZWdncWDD15Bjx4dGTbsYUpKNlFU9BX33nsiZ5wxjn79hntesu2RGjceuEcmxKkWLJjLnDkvc9xxJ9KsWR7vvbeQZcs+3yohLe/004/ljDN+QrduPdhvv/1ZvPh9br/9Jjp2PIROnQ4lIyODIUOuYeTIawgh0KtXH4qKCnn77bnUqVOH88+/lKZNc6lfvz6vvPISBx3Uhnr16pGTsy9XXHEt48b9knbtOtK1a3emTp3C3Ll/4eWX3wHg6acns2nTJo444igaNGjI9OnPkJWVRbt2HWnceD/2378pU6Y8TIsWB7Fy5TJGjbqWzMyam0bW3CszM7NtksQll5xIly6tOeeccaxc+W9CKGXatGspKFjAeec9SnZ21R8lm9n35eTsy7x5b/DII/exZs03tGhxEFdfPYKzzqr8puLjjjuRqVOfYOzYmykqKiQ39wD69u3H8OG/JCMjuv/khhtupVmzPCZMyOe66y6nUaMcOnfuxtCh1wGQmZnJmDH3Mn78aPLzR9GrV2+ee+5VLrnkSgoL1zJ69HWsXv0FHToczKRJv6dLl65xvI25775xjBx5DZs2ldCp02E89tg0WreOlo2bOPEZbr75Svr27ULbth0YOXI8F11U+SZVezvVph2VunfvEObOHZ90GGZme6QVK75mwIA7efPND8vKWrb8DwYPfpZmzdonGJnVJIcf/gFt2x6adBhWg33yyQcsXFjxc2zwYL0dQuhR4ckUHkE2q8CqVa9RUDCF4uIvyc5uSqtWg/aYG8Oiba1nEW1CUoe8vBO2eXNgddpY7dO8eRP+9KdbGT78UR56KFq2cNmy9xg7tgcXX/wUnTuflHCEZmbp4Zv0zMpZteo1liyZQHHxaiBQXLyaJUsmsGpV8hsqRInuTLbs0FfKF1/M5J//fHCXtrHaq27dLO67bzAPPXQFdetGYyjr1n3Dr399MjNnjqU2fepoZrWXE2SzcgoKpmy1EQVAaWkxBQVTKmmRPtEo8PaXV7eN2YUX9uOVV26nZcv9gWh72+eeu4mJE89iw4a1CUdnZrZ7OUE2K6e4+MsdKk+v0h0sr24bM+jZsxNz546nd+/OZWULF/6eceN68cUXixOMzMxs93KCbFZOdnbTHSpPr8p+Zav6Va5OG7NIXl5jZs4cxdChPy4rW7HifcaO7cm77/4hwchsb+apOra77Krnlv9CmpXTqtUg6tTZevvQOnWyadWq8qV50iUv74QdKq9uG7NUWVmZ3HXXz5g06Srq1Ys2IdiwYQ0TJvTnhRdGl+3uZbY9SkqyKClZn3QYVkOVlKynpKR6uwqmcoJsVk5ubl/atx9CdnYzQGRnN6N9+yF7xCoWHToMJi/vJLb86tYhL++kKlekqE4bs4oMGnQcr702llatmpWVPf/8LTz44BmsX/9tgpHZ3qSgIJcVK5axceM6jyTbLhNCYOPGdaxYsYyCgtyd7s/rIJuZ2Q5ZvfpbBg3K55VX3isry8s7mMGDn6V5c69va9uWk7OGVq1WkZVVknQoVoOUlGRRUJDLmjU5ldbZ3nWQnSCbmdkO27TpO26++XHuvnt6WVl2dkMuuOBxDj/8jAQjMzOr3PYmyJ5iYWZmOywzM4Nx4y7k8cevpn79aF5ycXEhDz3038yYMYLS0u8SjtDMrPqcIJuZWbUNGNCHOXPG0bZtXlnZH/94G/ff35+ion8nGJmZWfWlNUGW1ETSs5KKJH0m6dxK6knSOElfxcc4SUo5303S25LWxf92S99VmJlZqq5d2/Lmm/n067flpXjRohe5446eLFv2jwQjMzOrnnSPIN8PbATygIHAA5I6V1DvUuB0oCvwA6A/cBmApLrAdGAKsB/wG2B6XG5mZglo0qQRM2aM4NprzywrW716CXfe2Yu3356aYGRmZjsubQmypAbAmcCIEEJhCOF1YAZwXgXVfwqMDyEsDSEsA8YDF8TnjgUygXtCCMUhhHsBAf+5my/BzMyqkJGRwZgx5/HUU9fRoEE9AIqLi3j44bOZNu16z0s2s71GZhq/VydgUwghdX/SvwMVLS7bOT6XWq9zyrl3w9bLb7wbl88s35GkS4lGpAGK69Y93Z/31V5NgT1hv2hLjp8DCZk1605mzboz6TDAz4Hazo+/Hbw9ldKZIDcE1pQr+xZoVEndb8vVaxjPQy5/rqp+CCFMBCYCSFqwPUt7WM3kx9/8HDA/B2o3P/4macH21EvnHORCoPzKzTnA2u2omwMUxqPGO9KPmZmZmdkOSWeCvBjIlNQxpawrsKiCuovicxXVWwT8IHVVC6Ib+Srqx8zMzMxsh6QtQQ4hFAHTgNGSGkg6BjgNeKKC6o8DV0tqKakFMByYHJ97FfgOuFJStqShcfns7Qhj4k5cgu39/PibnwPm50Dt5sfftus5kNatpiU1ASYB/YCvgBtCCE9K6g28GEJoGNcTMA74Wdz0EeD6zTfmSTo8LjsM+AC4OISwMG0XYmZmZmY1VloTZDMzMzOzPZ23mjYzMzMzS+EE2czMzMwsRa1IkCU1kfSspCJJn0k6N+mYLH0kDZW0QFKxpMlJx2PpFd/M+2j8u79W0t8k/SjpuCy9JE2RtELSGkmLJf1s262sppHUUdIGSVOSjsXSS9Kr8WNfGB8fVVW/ViTIwP3ARiAPGAg8IKlz1U2sBlkO3EZ0g6jVPpnA50S7du4L/AL4naQ2CcZk6TcWaBNCyAFOBW6T1D3hmCz97gfmJx2EJWZoCKFhfFS5o16NT5AlNQDOBEaEEApDCK8DM4Dzko3M0iWEMC2E8BzRyilWy4QQikIII0MIn4YQSkMIfwA+AZwc1SIhhEUhhOLN/42P9gmGZGkmaQDwDfDnpGOxPV+NT5CBTsCmEMLilLK/Ax5BNquFJOURvS54c6FaRtIESeuAD4EVwB8TDsnSRFIOMBq4OulYLFFjJX0p6Q1Jx1ZVsTYkyA2BNeXKvgUaJRCLmSVIUhbwW+A3IYQPk47H0iuEMITotb830cZVxVW3sBrkVuDREMLSpAOxxFwPtANaEm0W8rykSj9Fqg0JciGQU64sB1ibQCxmlhBJdYh27twIDN1GdauhQgjfxVPtDgQuTzoe2/0kdQOOB+5OOhZLTgjhrRDC2hBCcQjhN8AbwMmV1c9MX2iJWQxkSuoYQvg4LuuKP141qzXi3TkfJbpR9+QQQknCIVnyMvEc5NriWKANUBC9FNAQyJB0WAjhiATjsmQFQJWdrPEjyCGEIqKP0kZLaiDpGOA0opEkqwUkZUqqB2QQvSjWk1Qb3hzaFg8AhwL9Qwjrkw7G0ktSrqQBkhpKypB0IvATfLNWbTGR6M1Qt/h4EHgBODHJoCx9JDWWdOLmv/+SBgJ9gJmVtanxCXJsCFAfWAU8BVweQvAIcu3xC2A9cAMwKP76F4lGZGkjqTVwGdEfxpUpa2AOTDg0S59ANJ1iKfBvIB8YFkKYkWhUlhYhhHUhhJWbD6KplxtCCKuTjs3SJotoudfVwJfAz4HTyy3gsBWFENIUm5mZmZnZnq+2jCCbmZmZmW0XJ8hmZmZmZimcIJuZmZmZpXCCbGZmZmaWwgmymZmZmVkKJ8hmZmZmZimcIJuZ7eUkXSCpcBt1PpV0TbpiqoqkNpKCpB5Jx2JmVhEnyGZmu4CkyXHSFySVSPqXpHxJDXawjz/szjjTrSZek5nVfN5u18xs13kZOI9o16bewCNAA6Jd3MzMbC/hEWQzs12nON7O9vMQwpPAb4HTN5+UdJikFyStlbRK0lOSDojPjQR+CpySMhJ9bHzuDkkfSVofT5W4U1K9nQlU0r6SJsZxrJX0WuqUh83TNiT9l6R/SCqS9IqktuX6uVHSF3HdxyXdIunTbV1TrLWkP0laJ+l9Sf125prMzHYVJ8hmZrvPeqLRZCQ1B+YA/wCOBI4HGgLTJdUB8oHfEY1CN4+Pv8b9FAEXAYcCQ4ABwM3VDUqSgBeAlsCPgcPj2GbHcW6WDdwYf++jgcbAgyn9DABuiWM5AvgAuDqlfVXXBDAGuBfoCswHnpbUsLrXZWa2q3iKhZnZbiDpSOBc4M9x0eXA30MI16fUOR/4GugRQpgnaT3xKHRqXyGEW1P++6mk24FrgBHVDO84oBvQLISwPi4bIak/0RSRO+OyTOCKEMJHcbz5wCRJCiEE4Cpgcgjhkbj+WEnHAZ3iuAsruqYoPwfg7hDC83HZTcD5cVyvV/O6zMx2CSfIZma7zknxahKZRCPH04Gfx+e6A30qWW2iPTCvsk4l/Q8wDOhANOqcER/V1R3YB1idkqwC1Itj2ax4c3IcWw7UBfYjSuwPAR4u1/dbxAnydni3XN8AudvZ1sxst3GCbGa268wBLgVKgOUhhJKUc3WIpjVUtNTaF5V1KKkX8DQwCvhf4BvgVKLpC9VVJ/6evSs4tybl603lzoWU9rtC2c8nhBDiZN1T/8wscU6Qzcx2nXUhhH9Wcu4d4Gzgs3KJc6qNfH9k+BhgWeo0C0mtdzLOd4A8oDSE8K+d6OdDoCcwKaXsyHJ1KromM7M9mt+pm5mlx/3AvsAzko6S1E7S8fFKEo3iOp8CXSQdLKmppCxgMdBS0sC4zeXAT3YylpeBN4huEPyRpLaSjpY0SlJFo8qV+RVwgaSLJHWUdB1wFFtGmiu7JjOzPZoTZDOzNAghLCcaDS4FZgKLiJLm4viAaD7vB8ACYDVwTHwT2/8B9xDN2e0H/HInYwnAycDs+Ht+RLTaxMFsmQu8Pf08DdwK3AEsBLoQrXKxIaXa965pZ2I3M0sHRa+TZmZmO0/Ss0BmCKF/0rGYmVWX5yCbmVm1SNqHaPm6mUQ39J0JnBb/a2a21/IIspmZVYuk+sDzRBuN1Ac+BsbFuwiame21nCCbmZmZmaXwTXpmZmZmZimcIJuZmZmZpXCCbGZmZmaWwgmymZmZmVkKJ8hmZmZmZin+Hxztdxu0fLjJAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = -per_clf.coef_[0][0] / per_clf.coef_[0][1]\n",
    "b = -per_clf.intercept_ / per_clf.coef_[0][1]\n",
    "\n",
    "axes = [0, 5, 0, 2]\n",
    "\n",
    "x0, x1 = np.meshgrid(\n",
    "        np.linspace(axes[0], axes[1], 500).reshape(-1, 1),\n",
    "        np.linspace(axes[2], axes[3], 200).reshape(-1, 1),\n",
    "    )\n",
    "X_new = np.c_[x0.ravel(), x1.ravel()]\n",
    "y_predict = per_clf.predict(X_new)\n",
    "zz = y_predict.reshape(x0.shape)\n",
    "\n",
    "plt.figure(figsize=(10, 4))\n",
    "plt.plot(X[y==0, 0], X[y==0, 1], \"bs\", label=\"Not Iris-Setosa\")\n",
    "plt.plot(X[y==1, 0], X[y==1, 1], \"yo\", label=\"Iris-Setosa\")\n",
    "\n",
    "plt.plot([axes[0], axes[1]], [a * axes[0] + b, a * axes[1] + b], \"k-\", linewidth=3)\n",
    "from matplotlib.colors import ListedColormap\n",
    "custom_cmap = ListedColormap(['#9898ff', '#fafab0'])\n",
    "\n",
    "plt.contourf(x0, x1, zz, cmap=custom_cmap)\n",
    "plt.xlabel(\"Petal length\", fontsize=14)\n",
    "plt.ylabel(\"Petal width\", fontsize=14)\n",
    "plt.legend(loc=\"lower right\", fontsize=14)\n",
    "plt.axis(axes)\n",
    "\n",
    "save_fig(\"perceptron_iris_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Activation functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(z):\n",
    "    return 1 / (1 + np.exp(-z))\n",
    "\n",
    "def relu(z):\n",
    "    return np.maximum(0, z)\n",
    "\n",
    "def derivative(f, z, eps=0.000001):\n",
    "    return (f(z + eps) - f(z - eps))/(2 * eps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure activation_functions_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxAAAAEYCAYAAADMNRC5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3Xd4VFX6wPHvmfQKhJBACBDpIJ0gKKgRpK0KKvKTotgQlLWLdVHRXRGxrLIqCy4ICuIqYlcUV0IRpSmhCAZEOqTQkpCeOb8/ziQkYdLvzATyfp7nPjNz59x737mEufPe05TWGiGEEEIIIYSoDJunAxBCCCGEEEKcOySBEEIIIYQQQlSaJBBCCCGEEEKISpMEQgghhBBCCFFpkkAIIYQQQgghKk0SCCGEEEIIIUSlSQIhai2l1F6l1GQ3HGeqUmqbG45jU0rNVkodU0pppVScq49ZQTzzlVJfejIGIYQ4lyilblVKZbjpWFopdYM7jiVEVSmZB0JYQSnVA9gA/Ky17lvFbacCN2itO5Va3wg4rbXOtCjGGOBPoJfWemOx9cGAn9b6mBXHKef4VwNLgThgD3Bca53rymM6jhsHrAAaaa1Ti62vh/kOOOnqGIQQwh2UUvOBWxwv84ETwHZgCTBHa51Xw/0HACFa6+Sa7KfUPucD4Vrrq0utbwyc0FrnWHUsIawiNRDCKuOBt4BOSqkOVuxQa51iVfJQwXEyXJ08OLQGjmit12qtj7ojeSiP1vqUJA9CiPPQ90ATIAYYBHwBPAusVkoFVXenSikfrXWWlclDeRzXCUkeRK0kCYSoMccdmTHAHMxdnjuclIlSSi1yNN/JVEptVkpdoZS6FXgGuNBRXasd60o0YVJKva+U+rjUPm1KqQNKqYccr4copVYrpU4opY4rpb4tlcz86Xjc4DhOvGO7Ek2YHPt9yrHvHKXUVqXU8GLvxzi2H6GUWu74PL8ppQaWc47mA/8Emju23etYH6+UeqN02eJNixxl3lJKTVNKpSqlkpVSLyulbMXK+Dre3+eIeY9S6j5HrcsKR7EUx7Hnl3EcP6XUa0qpJKVUtlLqZ6VUv2Lvxzm2H6CUWuf43BsdtU+FZeoppd5zxJjtiOOBss6LEEK4QI7jx/chrfVmrfWrmJrfHsCjUPSd+aJS6qDju2yDUmpw4Q6Kfd/9RSm1XimVCwwu3oRJKdXWUaZz8YMrpSY4vqt9lFJeSqm5Sqk/lVJZSqldSqlHC7+/HTXwtwBXFbsGxjneK2rCpJRaq5R6pdRxQh37vL6Sn8lHKTVTKXXYcZ04oJSabumZF3WGJBDCCjcA+7TWW4H3gHFKKZ/CN5W547MSczfoWqAz8Jzj7f8CrwC/Y+4YNXGsK20h5gu2XrF1lzvKL3a8DgJeAy7CXCxOAV8opXwd71/keBzi2O76Mj7P/cAjwGOOWD8BliqlupUq9zwwE+iKab71gTLNocra53PAQcexe5VRrixjMdXxlwD3AA8ANxZ7fwEwDngI6IBJ4k4CB4ARjjIXOo59fxnHmOHY5+1Ad2ArsEwp1aRUuReAxzEX42PAIqWUcrz3D8w5uxpo59jXoSp+ViGEsJTWehuwjDPfh+9griFjgE6Y79AvlFJdS236IjAFaA+sK7XPRMx3/9hS24wFPnQ0l7JhvgP/D/Pd/DfgSeA2R9mXgQ85U2vSBFjr5CMsBEYVv3Hk+CzZwFeV/Ez3AdcBo4A2mO/7350cS4iKaa1lkaVGCxAPTHY8V8BeTJ+GwvfvBNIxbTydbT8V2OZk/d5i+/UGkoA7ir3/H+C7cuIKAgqAfo7XMYAGYss7PubL/mknn3Fhqf1MLPZ+U8e6fuXEMxnY62S/b5RaNx/4slSZn0qVWQ78x/G8jePYQ8o4bpzj/fCyjuM4V7nAuGLvewF/AP8otZ/Bxcr0dayLdrz+HJjn6b9JWWSRpW4upb8/S703HcgEWgF2oHmp9z8F3nI8L/y+G1GqzK1ARrHX9wH7ONOntLlj35eUE+N04PuKYnYc/wbH84aO7+gBxd7/HtOvg0p+ppnA/wpjlUWWmixSAyFqRCnVGugHvA+gtdbAIko2Y+oObNHFOvBWldY6H1MzMdZxXD/M3ZeFxWJppUxTpz+UUmmYhMOG+UKv7OcJBaKAH0u9tQboWGrdlmLPDzseIyp7rCraUur14WLH6o65cKyg+loBPhT73FrrAuAnqva5ZwE3KqUSHM2sLq9BTEIIYSWF+VHew/H8N6VURuECXIX5LixuI+X7AHPNuNTxejTwp9a6qBZBKXWXo7lniuM4D1KF6xKANv30lnHmGhgFXMGZa2BlPtN8oBuQqJR6Uyl1VakaDSEqzdvTAYhz3njMner9Z1qxoACUUs201gcsPNZC4CelVFOgN+CLGdWo0JeYJkITMbUI+cBvjnJWKD1kWdFoHlpr7fj8Vf0ytuM4X8X4OClXeuQQXY1jVVeZn7vYezYArfU3SqkWwFBgAPCVUuojrfVtCCGEZ3XEjIBnw3x39eLs79asUq9Pl7dDrXWyUmo55of9KsfjosL3lVI3YprWTsY0TUoD/oppSlRVC4G3lVKTMM2QDgCrHe9V+Jm01r84+sUNxnw/LwASlFIDtdb2asQj6jDJPEW1KaW8MZ2/nsDc1ShcumLuUhf+aPwV6KKUCi9jV7mYJKRcWuv1wG7MHZ6xwGda68LObA0xbVSnaa2/11rvAEIomSQXjnpU5rG01mmYu+qlh6Lth0lGrJaCafNaXOk2uBXZjPm/fEUZ71f4uTFNlXIp9rmVUl7AxVTxc2utU7XW72mtb8XURN3iqDESQgiPUEp1wvR/W4K5JimgsdZ6d6mlOn22FgIjlVI9MX3AFhZ7rx+wTmv9htb6F631bs6u5ajUNRDTRBRMH7OxwPuOWn8q+5m01ula6yVa67sxtRP9MSMEClElUgMhauIqIBx4W5caBlUp9QFwl1Lq75jmTY8DnymlHsfUDnQC0rXWKzB9HVo4RvPZ71hf1tB1izC1HjGU7AR9AkgF7lRKHcD0SXgJUwtRKBlzJ2awMqMgZWutTzk5xkvAc0qpXcAm4CZM9XQPJ2Vr6gfgNaXUMExntolAM8w5qRStdaJS6kPgP0qp+4FfgGggRmv9HqZ9rsZ0Qv8CyCpMvIrt47RSahbwolIqFTNi1YNAJGZ43kpRSj3nOP52zPfL9cCecv49hRDCan7KzKFgAxph7rY/ifk+f9nxfbcImK+UehjznRWGY44erfVS57st06fAbGAusEGbztWFEoFblVJDMTfARmE6Op8oVmYvMFQp1Q4zMMUp7WS+Cq11tjKjEU7B3Gi6udh7iRV9JmVGLDyCuemUh+lsnYapuReiSqQGQtTEHcCK0smDw0eYH/kDtdanMV+YBzHjcW/DjMldeOfkY+BrTOeuFEwNQ1kWYkb3OQV8V7jSUf16I9DFsf83gaeAnGJl8jEd3sZjahk+K+MYMzFJxAzHvq7DdKRLKCeu6ppXbPkR09n8k2rsZxwmUZsJ7MS0da0H4Lj79Axm1Kgk4A3nu+AxTD+TdzAXmC6YjtlHqhBHjuM4CZjPEwJcU7WPIoQQNXIl5ofyfsx1ZRhmsIzLHNcjMDXk72C+53dimsBehrnhUiXazFf0CeZH/cJSb8/GjLL0PmbEphjMyIPFvQ3swPS3SOHsGvDiFjqO86vWunTtcEWfKR0zwuB6TILRDRiq3TDfkjj/yEzUQgghhBBCiEqTGgghhBBCCCFEpUkCIYQQwmWUUvc4hrDMUY5Z0Msod4tSapNSKs0xk+4Mx0ANQgghahlJIIQQQrjSYcwM5fMqKBeImWE9HDNM8wDM0JdCCCFqGbm7I4QQwmUKR7RRSsViRgcrq9ysYi8POUaUKWtoYiGEEB50TiUQ4eHhOiYmxtNhcPr0aYKCgjwdRq0h56Ok2n4+dJ4m73ge3vW8sfm7vhKytp8Pd6st52PTpk2pWutGno6jHJdhhgN2Sik1AZgAEBAQ0LNZs2buiqtMdrsdm00q9gvJ+ShJzkdJcj5Kqi3nIzExsVLXhnMqgYiJiWHjxopmlXe9+Ph44uLiPB1GrSHnoyQ5HyXJ+SiptpwPpVSVh6t0F6XU7UAsZshlp7TWc4A5ALGxsVquDbWPnI+S5HyUJOejpNpyPip7bTinEgghRM1orVFKeToMIcqklLoWeAG4Umud6ul4hBBCnM3zdSVCCLfIO5nH+vbr2TNlDzL/i6iNlFJDMJNqXaO13urpeIQQQjgnNRBC1BGpH6eSlZhF2k9pUgsh3MYxFKs34AV4KaX8gXzHzPDFy/UHFgHXaa3Xuz9SIYQQlSU1EELUEUmLkgCIHBvp4UhEHTMFyAIeB25yPJ+ilGqulMpQSjV3lHsKqAd87VifoZT6xjMhCyGEKI/UQAhRB2QfzOZk/EmUn6LRiNo88I4432itpwJTy3g7uFg5GbJVCCHOEVIDIUQdkLw4GTQ0vLoh3vXkvoEQQgghqk8SCCHqAGm+JIQQQgirSAIhxHnu9PbTnE44jXd9bxr+paGnwxFCCCHEOU4SCCHOc4W1D41GNsLmJ//lhRBCCFEz8mtCiPOYtmtpviSEEEIIS0kCIcR57NSPp8jZn4NfMz/qXVrP0+EIIYQQ4jwgCYQQ57HC2oeI0REom0weJ4QQQoiakwRCiPOUPddOyocpgDRfEkIIIYR1ZEB4Ic5TykvRYVEHTsafJLhLcMUbCCGEEEJUgqU1EEqpe5RSG5VSOUqp+RWUfVApdVQplaaUmqeU8rMyFiHqOuWlaDi0Ia1ebOXpUIQQQghxHrG6CdNh4B/AvPIKKaUGA48DA4AWQEvgWYtjEUIIIYQQQljM0iZMWuulAEqpWCC6nKK3AHO11tsd5f8OLMIkFUKIGkpanETKRylEPxBN/cvqezqcWkFryM2FrCzIzob8fCgoMI/lLc7KFBSY/RUudnvlX+/c2YQdOypXvnT8Vr4WQgghqstTfSAuBD4r9joBiFRKNdRaH/NQTEJYImzdOhg40KO/2I4WTOcEsYR9+jfq277yWBwAl2sNqnojQGVpf1JoRDIRpNCIUzqUNIovIebRsT6DYLLxIxt/sgggG/9iS4DFn6y62nk6ACGEEKJGPJVABAOnir0ufB4ClEgglFITgAkAkZGRxMfHuyO+cmVkZNSKOGoLOR8l1UtK4ugVV/D7Y495LAadCqywkzj4PnaF3uexOMD8fQQHn92J++QpH44e8Scp2bEkBZCU7Mfx436cPOnDiZO+ZGVZ+xXl42PH18eOj68dby87Xl66zMVmK3u9zQZKmQTRZtMoBQo4XZBBts4iX+eRr/MoII988sjTuQR5BdI6uCUFBXlobztrjv9Igc6ngDwKdB52CkBpUHYuaXgx0QFNQMFvaTvYlrbNfABVMin1tflyfdS1Ra8/P/o5mQWZxUqcKd8xpCPd6ncB4Ej2EVZ8aumpFUIIUYd4KoHIAEKLvS58nl66oNZ6DjAHIDY2VsfFxbk8uIrEx8dTG+KoLeR8lLRz2TIaN21K4wEDPBvIjZ49fKHPPltDTkE/tm+HHTvOLMcqUdfo4wMREdCokVkaNIDQ0LOXevXMY1AQBASAv/+Zx8LFzw9sNhvldf3SWnM86zgADQMbAvDniT9ZkLCA5NPJpGSmmMfTKaRkpnAi6wS77t3FBQ0uAGDkRyNZ8tsSp/uOi4njy1tWEB8fT/c+3an/4rgS79uUjSCfIIJ8g3j0qn8zvH0fAD7avp4FCfH4efvh7+2Pn5cffl7meahfKM9e0aJoH4u2hJGdH1hU1tfLt2iJqR9DywYxAGTkhhMiw1YIIYSoJk8lENuBrsCHjtddgSRpviTOC3Y72Dw3xYrWGlXNJkM1lZ8Pv/wC69adWXbv7ue0bEgItGwJLVqYpXlzszRtapKFiAiTGFj1UbTWmHoCY8HmBfyW8hsH0w9yMM0sh9IOkVOQw/297+e1Ia8BcCTjCM+uLHuMh1M5ZypTY5vEkpmXST2/etT3r19iuaD+BWc+u18Iv036jSDfoKKkwc/Lz+m/28gLRzLywpGV+oxju4ytVLlgXxnWVwghRPVZmkAopbwd+/QCvJRS/kC+1jq/VNF3gflKqUWYkZumAPOtjEUIT1EeTCAyf89ky9AtRE2Movljzd1yzL174Ztv4Lvv4IcfIC2t5Pt+fgXExnrRpQt06HBmiYqyLjkolG/PZ/+p/fxx/A/+OPEHu4/v5o8Tf/DH8T84mHaQlEdS8LJ5ATBr4yzWHVp31j7q+dVDFUs0WjVoxZRLpxARFEGjoEZEBEWY54GNaBDQAF8v36Kyj/V7jMeouOmaTdno0KiDBZ9YCCGEcD+rayCmAM8Ue30T8KxSah7wG9BRa71fa71MKTUDWAEEAB+X2k6Ic5fdDl5eHjl00qIksv/MJvP3zIoL18CePbBkCXz0EWzcWPK9Nm2gb1/o3dssx46t4corL7f0+LkFuSQeS2R78nZahbUiNioWgCW/LWH0x6PL3O5A2gFi6scAcEf3OxjWbhjRodFFS9OQpgT5BpXYJjI4kr/3/7ul8QshhBDnMquHcZ0KTC3j7RJ15lrrV4FXrTy+ELWB0tojNRBaa5IWJQEQeVOk5fvPzoalS2HOHFi58sz6oCAYOhQGDzaDT7VoUXK7+Piaj0b14/4f+fngz2w6sonNRzeTeCyRAl0AwH0X3VeUQLQOa010aDStGrQyS5h5bB3WmlZhrajvf2ZI2zt73lnjuIQQQoi6yFN9IIQ4b3mqCVPaz2lk78nGN8qX+pdbN/fD0aPwz3/Cf/4Dx03/YgID4dpr4YYbYMgQ02HZCrkFuWw6vImfD/7Mfb3vK2pu9PB3D5dobqRQtGrQigsjLqRb425F62OjYjnw4AFrghFCCCGEU5JACGE1DyUQhbUPEaMjUF4171ywbx/MmAFz50JOjlnXvTtMnAijR5tRj2oqLSeNtQfWsmb/GtbsX8O6Q+vIzs8GYEjrIUX9BG7oeAPdGnejZ5Oe9GjSgw6NOhDoE1jzAIQQQghRZZJACGExpTV4u/e/lj3PTsp/UwCIHFuz5ksnT8Lf/w7/+hfk5Zl1114Ljz9u+jTURPERorYmbaXb7G7Ytb1EmY6NOtK3WV98vHyK1k2+ZHLNDiyEEEIIy0gCIYTVPFADcWL5CfJS8wjsEEhwt+oN0VlQYGobpkyBlBQzQtKYMfDEE9CpU/Xi0lqzP3M/M9fN5Ns/vsXPy4+lNy4FoH14e0L9QukQ3oF+zfvRr3k/Lml2CeGB4dU7mBBCCCHcQhIIISzmiT4QRZ2nx0ZWaw6IXbtg3Dj4+Wfzul8/eO016Nmz6rEU2AtYs38Nn+78lE9//5S9J/cWvRfgHUBOfg5+3n74ePmQPDm5RE2DOP8ope4BbgU6A4u11reWU/ZB4DEgEFgC3K21znFDmEIIIapAEgghrKa1W4dxzc/IJ/XTVAAixkRUaVu7Hd56Cx59FLKyzCRur74KI0dWf46Gt395m7u/urvodT2fevyl3V8Y3Gowg1oNws/7zBTIkjzUCYeBfwCDMcN2O6WUGgw8DvR3bPMJ8KxjnRBCiFpEEgghLObuGojUT1OxZ9oJvSSUgAsqPxxSaiqMHWsmgAO4+WaYORPqV3IAJ7u2s2b/Gt7f+j4tG7Tk0b6PAnB126t59adXubb9tVzb/lqyd2fT/4r+Vf1Y4jyhtV4KoJSKBaLLKXoLMFdrvd1R/u/AIiSBEOewffvMKHa//QYnTnTh7maH6bjzINETo4i+3/x3OLnmJIl3JVZpv1F3nr19vb71aDe7HQB5x/P49bJfq7RPZ9v7NPCh++ruRWUSBiaQc6TylYJlbd/1u65F63bdv4sT/ztRpVi7ftcVvyi/Etu3fq01YVeGAXB4zmEOzjxYpX06297Zea6KKv07nYb1Qesr3Kc7/p0qQxIIISzm7gQieVEyULW5H7ZsgeHDzSzS4eEwezZcf33ltv0t5Tfmb57P4m2LOZhmvqBj6sfwyCWPoJQiOjSaxHvPfMnG/xFf6bhEnXYh8Fmx1wlApFKqodb6WOnCSqkJwASAyMhI4uPj3RJkeTIyMmpFHLVFXT4fWsOiRc2ZPz+GgoLC60EYP2zcR0syWfHJIZp13W1Wrwe2V23/uzfsZnd8ye0zAzM5En/ErDtZ9X063b4eJf8NtwJJVdhpGdv/tOonMoIdfx+/Vj3Wn1b9BI0dLxzbb1m75cyv2mqcU2fbOzvPVVHVf6dMKp4E1i3/TpUgCYQQVnNjApGbnMvx5cdR3opGIxtVapuPPzb9HTIzoVcv+OQT03SpIvF74/nbD39j7YG1Reta1GvBmM5jGN1pdLX6XghRTDBwqtjrwuchwFkJhNZ6DjAHIDY2VsfFxbk6vgrFx8dTG+KoLerq+dAaHnnEDEphs5ma3tGjYcOG7Wz4rg23/dSQ9NW+vHHUl1GjIL9nPtl/ya7SMXwb+eIb6Quc2d4r2IuAGFMLbc+3k7m14h+jxTnbXnkpgjoEFZXJXJmJPc9e1i7OUtb2gW0DWbV2FXFxcWQvzCY/Lb9KsQa2DcTma66zhdv7N/PHu575WZvbMZfcB3KrtE9n2zs7z1VRlX+njRs2EtsrtsJ9uuPfqTIkgRDCYsqNfSC8G3jT6dNOZO7IxDfct8LyL71k+juAabI0Zw74+5ddvrDDM0BeQR5rD6wlxDeE0Z1Gc0u3W7g4+mJJHIRVMoDis4sUPk/3QCxCVNsrr5jFxwcWL4YRIyB7XzZB36dwz5TGvLqmIS+8YBKLqCi47DJvgjtVb/Q8AO+Qs7e3edtqtM+ytg9sV7P5d5xt79+8nItQJTjb3jfCF9+Iiq+JZXG2vbPzXBUV/julUuX9u+rfqTIkgRDCam6sgbD52Ai/OhyuLr+c1vDss2ZRyiQSDz3kvKN0vj2fz3Z+xsz1MwkPDOfj//sYgAEtB7DwuoUMbz+cYN/qf4kKUYbtQFfgQ8frrkCSs+ZLQtRWv/4KTz5pnn/wwZmmoae3nYbX4HDiIaZ91ZC8PHj5ZXMjJyGh8n3PhKgtJIEQwmKeGMa1PFrDY4+ZpMFmg/nzzUWrtBNZJ3j7l7d5c8Ob7D+1H4D6/vVJz0knxC8Em7IxtstY9wYvznlKKW/MtcYL8FJK+QP5WuvSbRbeBeYrpRZhRmGaAsx3Z6xC1EROjqlVyMuDv/61ZL8yXaABUDZz12baNFi5EjZsgHvvhffe80TEQlRf7fmVI8T5wk1NmPbP2M/2kdtJ25hWbigPPGCSB29vc0esdPKQcjqFR757hOavNeex7x9j/6n9tAlrw8whM9n3wD5C/EJc/EnEeW4KkIUZTekmx/MpSqnmSqkMpVRzAK31MmAGsALYD+wDnvFMyEJU3axZsGMHtGtnvnOLK0wgcFwafHxg0SIICICFC2HdOvfGKkRNSQIhhMXcVQNxZN4RUpakkH+87M5n06aZoVl9fWHpUjO/Q2mZeZm8tu41MnIzuLLllXw95mt23rOTe3vfS6hf6NkbCFEFWuupWmtVapmqtd6vtQ7WWu8vVvZVrXWk1jpUa32bTCInzhUnT8Lf/26ev/yySQyKK6qB8DrTbrRNG3ODB0zfNK3dEakQ1pAEQgiLuSuB6Lq8K61ebUX9/s4bzy5YAFOmmH4OixfDNdeY9b+n/s4T3z+BdlytWtRvwetDXmfjnRtZfvNyhrYZik3JV4MQQlTW9Olw/DhcfjlcdZWTAo4BcYonEGCalzZsCKtWwZdfuj5OIawivxKEsJqbEgj/Zv40e7AZNu+zj/XddzB+vHk+c6Zpi7v7+G5u+fQWOr7Vkek/TufTnZ8WlZ/UaxI9o3q6PGYhhDjfnDwJb75pns+Y4XxwiqImTKW+ruvVMzd6AJ5/XmohxLlDEgghLObqYVy1XRfVHjizdasZNjA/31SLX3PTPsZ/Pp72b7Tn3YR3sSkbE3pMIDaq4vGmhRBClG/OHMjIgAED4KKLnJdx1oSp0IQJEBZm+kGsXXvW20LUSpJACGE1F9dAnFh+gvVt13Po34fOei8tzSQPGRkwZgwEDvk7bd9oy9xf5wJwW7fbSLwnkdnXzKZZvWYui1EIIeqC3Fx4/XXz/OGHyylYYB6cJRCBgTBpknn+8svWxieEq0gCIYTFXN0HImlRElm7s8hLziuxXmvTbGnXLujSBf7zHwjw9SO3IJfRnUaz4687mDd8Hhc0uMBlsQkhRF3y4Ydw+DB07AhDhpRdTtudN2Eq9Ne/msEuPvsMdu+2Pk4hrCYJhBBWc2ETpoLMAlI/SQUgYkxEiff+9S/NRx+Bf1AuS5aYUUDu630fG+/cyPsj3qdNwzYuiUkIIeqq2bPN4wMPOO/7UKi8JkwAjRubWmOtzc0fIWo7SSCEsJgqKHBZDUTq56kUZBQQ2ieUwNZnpqp//5s/eOAhM5yr97UTCI8+AYC/t790jhZCCBfYuRPWrIGgIBg1qoLC5TRhKnTnneZx/nwzGZ0QtZkkEEJYTGntsgQieVEyABFjTe3DyeyT/PXTRxk7FnSBD/79ZvPKg5fI5G9CCOFi8+aZx1GjIKSCr9zCJkzlJRAXXwzt20NSEnz9tVVRCuEakkAIYTUX9YHITc3l+LLj4AXhI8NZsHkB7d5ox1vTm8KJVoRfcJC9X/4fE3pOwNvmbfnxhRBCGHl5Zq4dgDvuqLh8/bj6MPnMzR9nlDqzr7lzLQhSCBeSBEIIi7lqGNeUD1PQ+ZqwQWH4NPLhtXWvkby9Hay7H29vzfKl0UTWa2D5cYUQQpS0bBkkJ0OHDtCnT8XlgzoEwVVQv5/ziT8LjRsH3t7w1Vdm/0LUVpJACGE1F9VAHFl4BIDIsZF42bx49Yp/E/G/rwB44glFt26WH1IIIYQTixebx5tvLr/zdFVFRMDgweYy8tFH1u1XCKtJAiGExVwxjOt38d+R8VMGeX55NBzeEIAv/t2b5AMhdO58ZiZTIYQQrnX6tBluFSrRedoh/dd0+AxO/XyqwrKjR5vHwiRFiNpIEgghrGZhE6aU0ync/MnNvPv3dwFI6JJAlk8WGzbAa6+Zw7zzjhk/XAghhOt98QVkZpqmSxdUclqdE8tPwGuQ+nFqhWW278CMAAAgAElEQVSHDzfDcP/4I+zfX8NghXARSSCEsJgVw7hqrXkv4T06vNmBhQkLGbR1EAA3P3UzQT4h3HefyVMeegh6yiitQgjhNoU1A4U1BZUR3C0YroHQPqEVlw2Ga64xzz/4oBoBCuEGkkAIYbGaDuOaV5DH1YuvZtyn4ziWdYwxXmNontIcn0Y+NBrSiPffh59/NhMPPfWUhYELIYQoV3q66UCtFIwcWfntwgaFwUPQaESjSpUvTE6kH4SorWSsRyGsVsMEwsfLh5h6MTTwb8A/B/+Tm9rfxIkuJ8g7nkdmjo3HHjPlXnih4rHHhRBCWGfZMsjNhb59oUkT1x1n0CDTjGnjRjh0CJo2dd2xhKgOqYEQwmLKbq9yH4gTWSfYlryt6PWMgTPYNmkbt3S7BS9/L8KHh9PktiZMmwaHD8NFF5nh/oQQQrhPYefp4cOrtl32gWzYATmHcypVPjDQJBEAn39etWMJ4Q6SQAhhtSqOwrT8j+V0ntWZYYuHkZ6TDkCQbxBRIVElyv3xB7zyink+c6bLJrsWQgjhRF6emZ8Bqp5AHJ59GCbBkblHKr1N4TEKkxYhahP5CSKExSo7jGtmXib3fH0PgxYO4lD6ISKDIzmVU3KIv8R7Etk+ajund5zmiSdM1fm4cdC7t6uiF0II4czq1XDyJLRvD23bVnFju3lQXpWfNOLqq82l5IcfIC2tiscTwsUkgRDCapUYxnXdwXV0n92dNze8ibfNm+f7P8/q21YTHRpdVMaeaydpYRIp/01h5++Kjz4CPz94/nlXfwAhhBClVbf5EoAu0OZJFX51NWoEl1xiaj6WLav6MYVwJUkghLBYRTUQr6x9hb7z+pJ4LJELG13I+vHrefLSJ/G2lRzTwOZrI3ZzLG3fbsuz8wIBmDQJoqOd7VUIIYSraG1NAlGVGojix5JmTKK2kQRCCItVlEDE1I/Bru08fPHDbJywke5NupdZNiAmgAOdo/jiC9Op7vHHXRGxEEKI8mzZAvv2QWRkNZuQVqMJE5xJIL7+2tRECFFbSAIhhNVKNWGyazvrD60vej2i4wi2T9rOy4Next/b3+ku7Pl2tDZ3rArnerj/foiIcF3YQgghnCusAbjmmuoNYFGdJkwAbdpAhw6m78WqVVU/rhCuIgmEEBYrXgOx/9R+rnz3SvrO68svR34pKtOhUYdy93F07lHWt19P/FNJLF8OoaEwebJLwxbCJZRSYUqpT5RSp5VS+5RSY8oo56eU+rdSKkkpdVwp9YVSSka/F7VCYQIxbFj1tq9uEyaQZkyidrI0gajChWKqUipPKZVRbGlpZSxCeIzdjlaKBZsX0HlWZ1bsXUED/wYczzpe6V0kLUoiKzGLj/9rLjoPPwxhYa4KWAiXehPIBSKBscAspdSFTsrdD1wMdAGigBPAv9wVpBBlOXAAfvnFNCO98spq7qSaTZigZALhqJgWwuOsroGo7IUC4L9a6+Biyx6LYxHCI4555zJi21Pc+tmtpOWkMbzdcLZN2saVLSt35cnel82p1afA18a8XeGEhcEDD7g4aCFcQCkVBIwAntJaZ2it1wCfAzc7KX4B8K3WOklrnQ38Fyjr+iGE2xTO/VA4O3R1VLcJE5iJQxs3hv37YevW6h1fCKt5V1ykcopdKDpprTOANUqpwguFdP0UdcKa/Wu4fvBuUlJ+J8Q3hJlDZ3JL11tQqvJ3nZIWJwGwM6whmUe9eeRe04RJiHNQWyBfa51YbF0CcLmTsnOB15VSUcBJzE2ob8rasVJqAjABIDIykvj4eKtirraMjIxaEUdtcb6cj/ffvxBoRMuWvxMfX/mJ4Eo4aB527d7FrvhdVd68W7d2LFvWhLfe+oNRow5UL4Za5nz5+7DKuXY+LEsgqNqFAuAapdRx4AjwhtZ6lrNCcpGo/eR8nJGUnUS2TdPDpw2PdH+Oxicbs3LlyqrtZI55ePdoJH5+BXTv/jPx8efu8Bvy91FSHTsfwUDpKbBOASFOyu4CDgCHgAJgK3BPWTvWWs/B8b8lNjZWx8XFWRBuzcTHx1Mb4qgtzofzkZ8PCQnm+b33tiMmpl219rPz3Z0c5SjtOrajSVyTKm+flGTmgti1qxVxca2qFUNtcz78fVjpXDsfViYQVblQfIj54k8CegMfK6VOaq0Xly4oF4nar66fj1+P/ErXxl2xKVM3fcHTjxH75kJsvS6q8r4ytmSw8c+NZPt6syE3jLsn2Bg+vK/VIbtVXf/7KK2OnY8MoHT9WSiQ7qTsm4Af0BA4DTyKqYGQedeFx6xbZ2aBbtsWYmKqv5+aNGEC0/fCZjOzYWdkQHBw9WMRwgpW9oGo9IVCa/2b1vqw1rpAa70WeB24wcJYhHC5nPwcHl3+KD3n9ORf68709exw0hebt0+19pm00DRfWp4Xgfay8dBDloQqhKckAt5KqTbF1nUFtjsp2w2Yr7U+rrXOwXSgvkgpFe6GOIVw6rvvzOPgwTXbT4P+DeAaCGwXWK3tGzY0fSHy8mDFiprFIoQVrEwgqnKhKE0DVR+aQAgPSTiaQOzbsby09iWUUqTlnKl8UwUF1RooXNs1yYuTAfhORzBqVM3ueAnhaVrr08BS4DmlVJBSqi8wHHjPSfENwDilVD2llA8wCTistU51X8RClFSYQAwaVLP9NL6lMTwE9S6uV+19DBliHpctq1ksQljBsiZMWuvTSqnCC8V4zN2k4cAlpcsqpYYDqzAd5XoB9wFPWhWLEK5SYC/gpbUv8fSKp8mz59E6rDXvXfcefaL7nCmkdbUSiJOrTpJzMIck5cd2XY9Fj1oYuMXS0tJITk4mrxJTo9arV48dO3a4Iapzg7vOR1BQENHR0diqM+uVtSYB84Bk4Bhwt9Z6u1LqUuAbrXVhY4zJwExMXwhfYBtwnQfiFQKAEydg/Xrw8YHa0OpwyBCYOhW++cZcZqowNocQlrOyDwRU/kIxylHODzM2wYta6wUWxyKEpVJOpzD8g+H8dPAnAO6OvZuXBr5EkG9QiXLFJ5KriuRFpvZhuY5kyFBFly41j9kV0tLSSEpKomnTpgQEBFQ4wlR6ejohIc66QtVN7jgfdrudQ4cOkZqaSoSHpy/XWh8HrnWyfjWm71zh62OYkZeEqBX+9z+w2+Gyy2re5yAzMRN2Ql7XPHwaVK+Ja2ysmQ/ozz9h924zS7UQnmJpAlGFC8VoK48rhDs0CGhAgS4gKiSKucPmMqT1EOcFtQYvryrt255jJ3lJCgDfE8l/anHfh+TkZJo2bUpgYPXa8grXs9lsREZGsm/fPo8nEEKcq6xqvgSw57E98CmcjDhJo+sbVWsfXl4mlg8+MM2YJIEQnuTxum0harP9p/aTlGE6NnvbvPnwhg/Zdve2spMHqlcDcezrYxSczGcXwQR2CGLAgBqF7VJ5eXkEVHc2JeE2Pj4+5OfnezoMIc5JWsO335rnViQQAa0DoC14N6jZfVvpByFqC0kghHBCa828X+fR6a1OjP9iPFqbIfha1G9Bg4AG5W5bnQQibGgY77e/kPnEcM89tb9ta1UmxhOeIf9GQlRfYqKZ+Tk8HLp3r/n+Wr3UCmZDgyvKv35UpDCZWbECsrNrHpcQ1SUJhBClHEk/wrAPhnHH53eQnpuOj82H7PwqfFNXownTlh1evL2zEdtCwxk3rooBCyGEsFRh7cPAgdXq0uYyTZpAt26QlWXmhBDCU2rRfwshPEtrzeKti+k0qxNfJn5Jff/6LLxuIR//38cE+FS+yU5VayC01vzLMY3EbbfJBEFCCOFpVvZ/AMdEcnZr9iXNmERtIAmEEIBd2xnx4QjGLB3D8azjDGk9hG13b2Nsl7FVbwpSxQTil6HbiF7wG43I5q9/rWLgokpSUlKYNGkSMTEx+Pn5ERkZyYABA1i+fDkAMTExvPzyyx6OUgjhSTk5ZyZrsyqBSBicAAPg+PLjNd5X4aR2jq8tITzC6mFchTgn2ZSNNmFtCPUL5ZVBr3BH9zuq3YZcVWEeiNyUXNK+O0Y/rdhyZVsZVcPFRowYQWZmJnPnzqV169YkJyezcuVKjh075unQhBC1xNq1kJkJnTpBVJRFOy0wD8qr5n2TLr4YgoJg61Y4csQ0axLC3aQGQtRZiccSWbVvVdHrqXFT+W3Sb4zvMb5mHVDt9kr3gfAK8+WRJr2ZRgcmPiT5vCudPHmS1atXM336dAYMGECLFi3o1asXkydPZtSoUcTFxbFv3z4eeeQRlFIl/gbWrl3L5ZdfTmBgIE2bNuXuu+8mLe3M7ONxcXHcdddd3H///TRo0IAGDRrwyCOPYLdb1GZBCOE2VjdfAkcTJqxJIPz8zkxsJ7UQwlMkgRB1Tl5BHtPXTKfLrC6M/ng0p7JPARDgE0DT0KY13n9V+kB89x1sOhzA/gsiiqqlhWsEBwcTHBzM559/TraT4UuWLl1KdHQ0Tz/9NEeOHOHIkSMAbN26lUGDBjFs2DASEhJYunQpmzdv5vbbby+x/aJFi7Db7fz000/Mnj2bOXPm8Nprr7nlswkhrFOYQFj5naztJoGw6ldXYXJTGKsQ7ia3PEWdsvbAWiZ9NYmEpAQABrYciEZbeozKJhD2XDtvz1GAYvz42jXSx/nI29ub+fPnc+eddzJnzhy6d+9O3759GTlyJL179yYsLAwvLy9CQkJo3Lhx0XYvvfQSN954Iw8//HDRulmzZtG9e3eSk5OLJmpr0qQJM2fORClF+/btSUxM5NVXX+Whh2rxrIBCiBKSk+GXX8xd/ksvtXDHFjZhgjMJxPLlVe52J4Ql5E9O1Akpp1O4/bPb6TuvLwlJCcTUj+Hbm75l/rXzqe9f39qDVXIY1+2T9zL80w1cbDvGbbdZG4JHKOV0CQkNLfO9Gi9VNGLECA4fPswXX3zB0KFDWbt2LX369GHatGllbrNp0yYWLlxYVIMRHBxM3759Afjjjz+KyvXp06dEs6eLL76YQ4cOlWjqJISo3b7/3jxedhlYOV9mYQ2EVQlEu3bQrJlJeLZssWSXQlSJJBDivKe15qr3r+Kdze/g6+XLlEunsH3Sdga1srCBazGVqYHQds3h95JoQSY9LradH53gtHa6pKellflejZdq8Pf3Z+DAgTz99NOsXbuWO+64g6lTp5Kbm+u0vN1uZ/z48WzevLloSUhIYNeuXXTr1q0mZ0wIUcu4ovkSnOkDYdWvLqWkGZPwLGnCJM5bWuuizrBT46by+rrX+dfQf9G2YVvXHrgSCcTJNafwO5lDMn785XGLa0BElXTs2JH8/Hyys7Px9fWloKCgxPs9evRg+/bttG7dutz9rFu3ruhvDuDnn38mKiqK0NBQl8UuhLCO1q7pQA1Y3oQJTIxz55qYH33Ust0KUSlSAyHOO0czjnLn53fywLIHitb9pc1fWDZ2meuTByo3jOuvLyUBsD4kgsFDrbugiLIdO3aM/v37s3DhQrZs2cKff/7JRx99xIwZMxgwYAChoaHExMSwevVqDh06RGpqKgCPPfYY69ev56677uLXX39l9+7dfPnll0ycOLHE/g8fPswDDzzA77//zpIlS3jppZd48MEHPfFRhRDVsG3bmWFRO3Wydt9WN2ECGDDA1ESsXm2GnRXCnaQGQpw3svKy+OfP/+SFNS+QkZuBn5cfT1z6BI2DTYfYGg3NWhUVDONqz7WT+10K/kDk2MjKjvgqaig4OJg+ffrw+uuvs3v3bnJycmjatCljxoxhypQpADz33HNMnDiRVq1akZOTg9aaLl26sGrVKqZMmcLll19OQUEBLVu25Lrrriux/7Fjx1JQUEDv3r1RSnHHHXdIAiHEOaR47YPVlwurmzABNGwIsbGwYQOsWnVmhmoh3EESCHHOs2s7H2z7gMe/f5wDaQcAGNZuGDOunFGUPLhTRX0g9n54HP/cfPYQxI1PBLsxsrrNz8+PadOmldthuk+fPiQkJJy1PjY2lmXLlpW7f29vb9544w3eeOONGscqhHC/b781j5Y3XwKXNGECE+uGDSb5kQRCuJMkEOKclp2fzWXvXMaGwxsA6Na4G68MeoX+F/T3WEwVJRBbXk2iPnCgbQTNm7svLiGEEM5lZZm7+AADB1q/f1c0YQKTQDz/vHSkFu4nCYQ4p/l7+9OyQUsOph3k+f7PM67rOLxsHm4TVM4wrvlp+QRtPgZAx/si3RmVEEKIMqxeDTk50KMHNGpk/f6DOgeRpbKwBVjb9bRPHwgOhu3b4dAhaFrzuVCFqBTpRC3OKZuPbuaaxdfwvz3/K1o3c+hMdt+3m9u631YrkgeldZkNaLe+lYqPtrPNqx5X3+7v5uCEq8THx0vTJSHOYS5tvgR0WtIJ/g3+za393vf1hSuuMM+XL7d010KUSxIIcU5Yd3Ad1yy+hu6zu/Nl4pc8v/r5ovcigiII9An0YHTF2O3ocpov7ZljRl/K6B1p6SRFQgghqs9V8z+4g8wHITxBmjCJWm3l3pX8Y/U/+H6PmR40wDuAiT0n8sSlT3g4sjLY7WilcFb/kHUohwZ/niAPRZ/HXFBHLoQQosoOHzZDuAYFwcUXu+YY9nw72CkxV4xVChOI5csrNQ2REJaQBELUWgs2L+DWz24FINg3mHt63cODFz9IRFCEZwMrT0FBmd/e66anYAN+DWjIg1f7uDcuIYQQThXeuY+LAz8/1xzj52Y/w1HIPZiLX1NrD9KmDbRoAfv2webNph+HEK4meaqoNdJz0tlwaEPR62HthhEdGs3Uy6ey74F9vHDlC7U7eYBymzAtTm/CVDpScEMzuUMkhBC1hFuaLynH4oJuekpJMybhfvIzRnjcrmO7mPzdZKL/Gc2wD4aRk58DQIOABvx5/588E/cMYQFhHo6ykux2px2os7Lgg0+8WEkE1zxRzwOBCeE5SqkwpdQnSqnTSql9Sqkx5ZTtoZRapZTKUEolKaXud2esom6x2890PnZVB2qASw5fAj+AX2PXVHFIAiHcTZowCY/ILcjls52fMXvTbP7355kRlbpGdiXpdBLN65kJErxt59ifqN2OdjKE6xefa9LSFD17QocOHohLCM96E8gFIoFuwFdKqQSt9fbihZRS4cAy4EFgCeALRLs5VlGH/PorpKZC8+bQtq2no6m+/v1N69k1a+D0adOfQwhXkhoI4Xb7Tu6j+T+b839L/o///fk/ArwDuK3bbWy4cwOrbltVlDyckwoK0KVqILTWMGEjT7CDW0fkeSgwURlxcXHcc889ng4DqFwsnTp1YurUqe4JqJqUUkHACOAprXWG1noN8Dlws5PiDwHfaq0Xaa1ztNbpWusd7oxX1C3Fmy9Z3LfZrcLCoFcvyMuDlSs9HY2oC86x27viXJRyOoVV+1YxouMIAJrXa059//qEB4YzsedEbu56M/X963s4Sos4GQLjYPxpItJO04M8+t0i/+U8KSUlhWeeeYavv/6aI0eOUL9+fTp16sTjjz/OwIEDWbp0KT4+taODe22KpYbaAvla68Ri6xKAy52U7QNsVUqtBVoD64C/aq33ly6olJoATACIjIwkPj7e6rirLCMjo1bEUVucC+fjww+7Ag1o2nQ78fEprjvQX6Egu4D4N+PBRVMAtW0bw7p1Mcybd5DAwN2uOYiFzoW/D3c6186H/JoRLnEq+xRfJn7Jf7f/l292f0OBvYC9D+yleb3mKKVYeetKIoIiLB/OzuMcw7gW99n2YKZzEcNis7kh6jz7vOeYESNGkJmZydy5c2ndujXJycmsXLmSY8fM7OBhYbWnr01tiqWGgoG0UutOASFOykYDPYCBwFZgBrAY6Fu6oNZ6DjAHIDY2VsfFxVkXcTXFx8dTG+KoLWr7+cjIMDM422xw330X0qCB64616s9VkAWXXnYpXoGumfDU2xveew9++y2auLja3/Kvtv99uNu5dj6kCZOwTGZeJgs2L+CaxdcQ8XIEN31yE18kfoHWmqFthpKek15UNjI48vxLHsDpMK7vvQeHCOTSh86bH4TnpJMnT7J69WqmT5/OgAEDaNGiBb169WLy5MmMGjUKOLvZUFJSEsOGDSMgIIAWLVrwzjvvnNVsSCnFrFmzGD58OIGBgbRt25YVK1Zw8OBBBg8eTFBQEN26deOXX34pEc/SpUvp3Lkzfn5+NGvWjOeff940d3MoHUtycjLDhw8vimXevHkuOlOWywBCS60LBdKdlM0CPtFab9BaZwPPApcopWTkAWG5+HjT5Oeii3Bp8gCgCxz/t134q6t3bwgJgR074MAB1x1HCJAEQtSA1pqjGUeLXucV5DHhywl8mfgleQV5XN7icmYOmcmhhw7x1ZivuDDiQg9G6yalhnHdmVDA+vXmS334cA/GJQgODiY4OJjPP/+c7OzsSm1zyy23sG/fPn744Qc+++wzFi5cyL59+84q949//INRo0aRkJBAbGwso0aN4o477mDSpEn8+uuvREVFceuttxaV37RpEyNHjuT6669n69atTJ8+nRdeeIE33nijzFhuvfVWdu/ezffff8+nn37Ku+++y969e6t6GjwhEfBWSrUptq4rsN1J2S2ALvZaOykjhCW+/dY8unL0pUKFCYTyct2NMx8f05kazowsJYSrSBMmUSWnsk+xev9qvvvjO77a9RXHs46z5KIlANTzr8djfR+jaUhTrm1/LZHBkR6O1gNKDeO6eezvzOU0Oy9tS2Dg+X0TVT1b9oVx9tWzmdBzAgBzNs1h4pcTyyyrnznzm7HnnJ78cuSXCstVhre3N/Pnz+fOO+9kzpw5dO/enb59+zJy5Eh69+59Vvnff/+db7/9lp9++ok+ffoAMH/+fGJiYs4qO27cOEaPHg3Ak08+yeLFixk8eDDDHVnjo48+yhVXXEFqaip+fn68+uqrXH755Tz77LMAtG3bll27dvHiiy9y7733nrX/xMREvvnmG9asWUPfvqY1z4IFC2jZsmWVzoEnaK1PK6WWAs8ppcZjRmEaDlzipPg7wMdKqZmYBOMpYI3W+pTbAhZ1xrJl5tGl8z8UspsHVyYQYJKhzz4zncNvv92lhxJ1nNRAiArtPbmXx79/nIvevoiwGWFcs/ga/rX+X+w5sQcv5cXhrMNFZZ+74jkmxk6sm8kDlBjGNS8tnwa/pdKS0wwZ4+vhwASYPhCHDx/miy++YOjQoaxdu5Y+ffowbdq0s8ru3LkTm81GbGxs0bpmzZoRFRV1VtkuXboUPY+MNH/7nTt3PmtdcnIyADt27ChKBAr169ePQ4cOkZZWuruAKW+z2bjooouK1rVo0cJpLLXUJCAASMb0abhba71dKXWpUiqjsJDW+gfgSeArR9nWQJlzRghRXbt3m6VBA9P0x5W01kV1acrm+gQCTA1EQYFLDyXqOKmBECUcSjvEukPr8LZ5M6zdMMDMEP3ijy8CZl6GPtF96B/Tn7+0+QsXNb2I1atWezLk2qXYMK5rX0rFT9tJ9A1l/OgADwfmemXVCKSnpxMScqa/7ISeE4pqIyqyacImS2Irzt/fn4EDBzJw4ECefvppxo8fz9SpU5k8eXK191l8tKTCvj3O1tnt9gr3VV7foHO135DW+jhwrZP1qzGdrIuvmwXMclNooo765hvzOGgQOJm6x1Lu6P9QqFUruOAC+PNPM8dFsfsfQlhKEog6LC0njYSjCaw7tI6fD/7MukPrOJh2EIA+0X2KEogLIy7kqcue4pJml9CveT+CfYPL223dVmwY10Pzk4kC8i6PLN2vWtQiHTt2JD8//6x+Ee3bt8dut7Np06aiJk4HDx7k8OHDznZTJR06dODHH38ssW7NmjVER0eXSLZKx7J+/XouucS0/Nm/f78lsQhRFxUmEEOHuuFghfcN3HAdUMokRbNnm2ZMkkAIV5EEog6wazt7TuxhS9IW+kT3ISrENHt46oenmLl+ZomyoX6hXNT0Ii5rflnROpuy8dwVz7k15nOWYxjXtH25RB48Tj6Ky55u5OmoBHDs2DFGjhzJ7bffTpcuXQgJCWHjxo3MmDGDAQMGEBpacqCgdu3aMXjwYO666y5mzZqFv78/jzzyCIGBgTWuCXj44Yfp1asXU6dOZcyYMWzYsIFXXnnFaVOqwliGDBnCxIkTmTNnDgEBATz00EMEBJz/NVtCWC0rC1asMM/d0f/BnTUQUDKBePJJ9xxT1D2SQJxn8u35fLT9I34/9ju/H/udnak7STyWSGZeJgALrl3AuK7jAOjVtBc9mvSgV1QvejftTZ/oPrQLb4dNye3yanMM47rq2WSCge2hYVzZT/o/1AbBwcH06dOH119/nd27d5OTk0PTpk0ZM2YMU6ZMcbpNYafruLg4IiIieO6559izZw/+/jWbCapHjx589NFHPPPMM0ybNo3IyEgef/zxcmeeLoylf//+hIeH88wzzxT1qRBCVN6qVZCdDd27Q+PGrj+euxOI/v1NRfiPP0J6uhkFUAirSQJxDsktyGX/qf3sPbm3xGJTNt697l3A1Bbc8fkdZOVnldg2KiSKLpFdaBjQsGjdTV1u4qYuN7n1M5z3HMO4pn+SRDAQODzC0xEJBz8/P6ZNm1bmXX7grFlAGzduzBdffFH0OjU1lQkTJtC6deuidcXnbgAIDw8/a1379u2L1qWnm+kPrr/+eq6//vpKxxIZGcnnn39eYt348ePL3F4I4Zxbmy+BW5swAdSvDxdfbBKI77+H665zz3FF3SIJRC1g13ZSM1M5nH6YI+lHzGPGEa5tfy2dIjoB8OKaF3nif0+gnQyLHuIbgtYapRQ2ZeOu2Lvw8/KjfXh72oW3o13DdjQIcPEsOcKw2zmS05YmJ9PJxIvBU8M9HZGogR9++IH09HQ6d+5McnIyf/vb3wgPD2fIkCGeDk0IUU3uTiDcXQMBcNVVJoH46itJIIRrSALhAscyj3E04yinck5xLPMYx7KOkZqZSmpmKmEBYTza91EAcvJziP5nNMezjmPXZx/aC2IAACAASURBVI/O0iS4SVECER4YjlKK5qHNiakfc9ZS3KuDX3X5ZxRlsNtZf2o0UcCeqHD+0tLFw3sIl8rLy2PKlCns2bOHwMBA+vTpw6pVqwgKCvJ0aEKIatizBxITzV16x/QurmcHvByLm1x1len/8NVXJcb2EMIyliYQSqkwYC4wCEgFntBav++knAKmA4X17/8BHtel6/3dICkjiVM5pzide5rMvMwSS3RoNJe2uLSo3EtrXyItJ43dB3Yz49AMTuWcIi0njVPZp/jkxk/oGdUTgKdXPM1bG99yeryOjToWJRB+3n6czj2NXdsJCwgjKiSKJsFNih47R54ZR/6mLjcxrus4fLx8nO5X1A55J/IJymoJaBrfVEfnwjiPDB48mMFumWVKCOEOhbUPAweCt5tuofo09CEuP+6sZomu1LkzREfDwYNmONeePd12aFFHWP3f500gF4jEzDb6lVIqQWu9vVS5CZgxwbtipldZDvwJ/Lu8nR9KP8Q9X99DbkEuOQU55Bbkmuf5OdzS9RZGXjgSgG93f8vk5ZOL3itdPuWRFEL9zIgrNy65kZX7Vjo93qhOo4oSiPTcdF756ZUzb6aULHs863jR82b1mtE+vD2hfqGEB4bTMKAh4YHhhAeG07xe8xLb7XtgH/X961eYGPh5+5X7vvA8rTW/PJlNPTQ7bSHc9DdpNiaEELWJ2/s/eIhSphZi9mxTCyEJhLCaZQmEUioIGAF00lpnAGuUUp8DNwOPlyr+/+3dd3hUVfrA8e+Z9B5CDT1UBaQJItWA2Auo2EURBIVFFwvINmHFdVf9qassFgRRUEFWAUGxrRKlCgEBaUF6CYEQQkJInzm/P07ahEkyCZPMTPJ+nuc+M3PvOXfe3Lm5M+ee9hDwqtb6WEHeV4GxVFCACDwQSOzdsQ63hQeEs85/HQB++X5My5pGcngyE8ZNKEozb9Y8QrNCyRyVSXhzU4B4cM6DTPptEkopFMru0d/Hn3WPm31qNN/mfotFWVj70lp6D+xNeEA4licsWH+x0nFgR2hr3ufe1fcyaOYgyrKOdRes67SwE5FXRQJw+J+HOT7zOC2ntqT5E80BSPkqhYSxCeUdngs4yh91YxSXzLkEgOyj2Wzps6VS+3SYPwLYXZxmY+eN5KfmO73PgOYBXL6x+OpWmL/3jt74RZmC1c57dpL2c1qlYnWU39FxrozyPqf8tHySd4MFH3YM60RouHdO+iWEELVRdnbx8K11oRtTyQLEc8+5OxpR27iyBqIDkK+13lti3TbgKgdpOxdsK5mus6OdKqXGYWosaKfa0SCjjE6pGZBLLgAWLDSgAZHBkSzssxBf5YufxY/wV8NRGYrd8bvZs28PAG1y2kB62X9U4T4B/DHDcV4eeDnBx4PJJx+OA0mwa/Mudll2mYQ7gBNl79ORrRu3Fk11X5h/3/Z97IvbZ9bFV36fjvIn7U0iKS7JrEuq/D4d5bdZbfZVs8co95iWlmvLdZh/7eq1pnACsL/ysTrK7+g4V0Z5n5PW8LilG0H4MKb/XuLiUiu3cy8RERFRNJKQM6xWa6XS13Y1eTyys7NrtNmEEJ7sxx8hM9MM3xodXXPvm5OYw/YbtkMQsKHm3nfIEAgIgI0b4eRJaCytaoULubIAEcqFPxvTAEcjEIcWbCuZLlQppUr3g9BazwZmA1ze7XLd9+u+TgekfBT+jYvH4M9JyAEN/k38URZzdzjvmzxsORd2YC7P+j3riY2NtcvvG+WLT6DpIZXfMx/rK9ZK7dNRfp8wH3zDzEdk7WMlf6zzd/UBh/ktgZaiu/K2fBt5g/IqtU9H+ddvKD4eADl7cnAwWFSZLvic9rjmc3KUvzo/p7Vr4VAKRPueZNKkbvjU0v7Tu3fvdjhbclnOnTtXqfS1XU0ej8DAQHr06FEj7yWEpyscBXnYsJp9X1uOjfPbz0MNzDlRUkgIDB4M33xjmm6NGlWz7y9qN1cWIDKA8FLrwgFHt9pKpw0HMirqRK38FAFNq94XICD6wrx+9avQKblEHYuj/L7hvviGV/3QOsrvE+SDT1DVf5E6ym/xtVzU8SzKH2W/3tFxrgyXfU4V5Hf157RggXm8M/IrfHxGV3m/QgghXMtmg8IpXW69tWbfO6BpAL229iJ+S3zNvjFw882mAPHVV1KAEK7lyoG99gK+Sqn2JdZ1A0p3oKZgXTcn0gnhFXJyYPFi8/yuel+5NxghhBB2tmyBxEQzMlH37jX73pYAC6HdQiGmZt8XTD8IgO++g9zc8tMKURkuK0Borc8DS4DnlVIhSqn+wDBggYPk84GnlFLNlFJNgaeBD1wVixA17auvIDUVurU9R6eQA+4OR7jRPffcw4gRI9wdhhCihMLmS7feakYoqitat4ZOnSA9HdascXc0ojZx9dQiEzDdhE4BC4HxWuudSqmBSqmMEuneBVYAv2G6on5VsE4Ir1TYfGnk0KS69e3kJZRS5S6jpG5fiFqtZAGipmUfy2bP6D0wr+bfG4prIQqbcAnhCi6dB0JrfQYzv0Pp9asxHacLX2tgSsEihFdLSTE1EBYL3BebiI6XKT89zYkTxcNtffnll4wdO9ZuXVBQkDvCEkLUgMOHYds2CA2FEuN91Jj8lHyS5iVBm5p/bzCdxl95BZYuhddek3tcwjXkl44QF2nxYsjLg6FDIToyy5QkhEdp0qRJ0RIZGXnBuogIM97vU089Rfv27QkKCiImJoa//OUv5JZoODx16lR69erF/PnziYmJITw8nBEjRpCaeuGQva+88grR0dFERUUxduxYcnJyauaPFULYKbzzfv31ZljTmqZtBePDuOmroW9faNLEFKS2bnVPDKL2kV86QlykouZLIwGbDS0FCK8VERHB/Pnz2b17N2+++Sbz5s3jlVdesUuTkJDAihUrWLFiBStXrmT9+vVMnz7dLs3333/PoUOHWLVqFR999BGLFi3irbfeqsG/RAhRyF3DtxbS1oIChJvu/FssMLygbciSJe6JQdQ+8ktHiIuwbx+sXw/BwQUXaJutTtYPK1X2Eh4eVu72i1lcbdq0afTt25fWrVtzyy23MGXKFBYuXHhBug8++IAuXbowYMAARo8ezQ8//GC3vUGDBsycOZNLLrmEG2+8keHDh1+QRghR/dLSIC4OfHzgxhvdFEThdENunBvottvM49Kl7otB1C4u7QMhRF3z0Ufm8fbbTftarFapgfBiCxcuZObMmRw4cICMjAzy8/Px9/e3S9OmTRtCQkKKXjdt2pRTp07ZpenSpQuWEudB06ZNSUhIqN7ghRAX+OYb08T0qqsgKqri9NXB3U2YwPT9iIyEnTshIQE6dnRfLKJ2kF86QlSR1sUFiJEjC1bW0SZMWpe9pKefK3f7xSyuFBcXx8iRI7n11lv58ssv+fXXX3nuuefs+kAA+PnZT0qolMJms1U6jRCi+n3+uXl0V/MlcH8TJgB/fzOpHEgthHCNuvdLRwgXWbcO9u+H6Gi4+uqClXW0CVNtsHbtWtq2bVvUUbp9+/YcOnTI3WEJIaro/Hn48kvz3K1Ts3hAEyYwNeUgBQjhGlKAEKKKPvzQPI4cadrXAnW2BqI26NChAwcPHmTx4sXs37+fN998k88Lb1+Ki6KUilJKLVVKnVdKHVZK3VdBen+l1G6l1LGailHUPl99BVlZZhSiFi3cF4cnNGECuO46CAqCjRvh6FH3xiK8n/zSEaIKsrLg00/N84ceKrHBapVhXL3UiBEjePzxx5kwYQLdu3dnzZo1TJs2zd1h1RazgFygMXA/8LZSqnM56ScDyTURmKi9Fi82j3fd5d44PKEJE5jBPq6/3jxftsy9sQjvJ790hKiCL76A9HTo1Qs6dSqxwWZDSxMmjzZixAi0gw4USileffVVTp8+zblz51i8eDFPPPEE2dnZRWn+9a9/ER8fb5fvscce4/Tp00WvFy1axGeffWaXxlG+ukIpFQLcAfxNa52htV4DLAdGlpE+BngA+GfNRSlqm4wMWLnSPHdr8yXwmCZMIM2YhOvIKExCVEFh8yW72gcwfSCkBkKIkjoA+VrrvSXWbQOuKiP9TODPQFZ5O1VKjQPGATRu3Ji4uLiLj/QiZWRkeEQcnsKdx+PHHxuSldWZzp3T2LfvV/btc0sYRsHkbfm2fLefH5GRvvj69uOnnxRLlqwnKiq34kzVRP5f7Hnb8ZAChBCVlJgI330Hfn5w772lNsowrkKUFgqkl1qXBoSVTqiUug3w0VovVUrFlrdTrfVsYDZAr169dGxsuclrRFxcHJ4Qh6dw5/GYOdM8jh0b4fbP5HTGaXawA18/X7fHAmY+jOXL4ejRfkU1Eu4g/y/2vO14yC8dISrp449NRcPNN0P9+qU2SidqIUrLAMJLrQsHzpVcUdDU6WXgiRqKS9RS5855UPMlIHJQJJf/ejk87u5IjPsKhjD45BP3xiG8m/zSEaIStC6n+RLIMK5CXGgv4KuUal9iXTdgZ6l07YHWwGqlVBKwBIhWSiUppVrXQJyilvjyS8jOhv79oVkzd0cDvuG+hHUPAw+IBeCWW8zEpxs34t6mXcKrSQFCiErYssXM5NmgAdxwg4MEUgMhhB2t9XlMYeB5pVSIUqo/MAxYUCrpDqAF0L1geQQ4WfBcBp0UTvvvf82ju0df8lTBwXDbbea51EKIqpJfOkJUQmHtw333mZk9LyDDuArhyAQgCDgFLATGa613KqUGKqUyALTW+VrrpMIFOAPYCl5by961EMVSU838D0rBHXe4Oxoj/Zd09ozeY8Ye8xD3328eP/nE1KwLUVnSiVoIJ+XmFt+tcdh8CWQYVyEc0FqfAYY7WL8a08naUZ44oHn1RiZqm0WLzLX6mms8o/kSQNb+LJLmJcEQd0dS7OqroVEjSEiAX3+Fnj3dHZHwNnKrVAgnrVwJKSnQpQv06FFGIhnGVQgh3OaDD8zjqFHujMJeeJ9wOs7pCDe5O5Jivr5w993m+ccfuzcW4Z3kl44QTirZebrMSgYZxlUIIdxi1y7TMTg8HIZfUN/lPkFtg4geEw0edpe/cDSmRYtM61shKkN+6QjhhORk067WYiluO+qQdKIWQgi3KLzJc/fdpqOwKF+fPtCmjZnbyIvmLxMeQn7pCOGE+fMhL8+MvBQdXU5CGcbVo40aNQqlFEopfH19admyJePHjyc1NdXpfcTFxaGU4vTp02W+x80331xmvpSUlCrHL4RwLD8fFhSM61VmHzU3Ob/zPMdnHS+akdpTKAUPPGCez53r3liE95EChBAV0Bree888HzeugsTShMnjDR06lBMnTnDo0CHmzJnDihUrmDBhgrvDEkJchO+/hxMnoF076NfP3dHYS1uTxu8Tf4cf3B3JhcaMMQWJzz+HMu6JCOGQ/NIRogKrV5uRKpo2hRtvrCCxdKL2eAEBATRp0oTmzZtz7bXXcvfdd/Pdd98VbU9LS2PcuHE0atSIsLAwrrrqKuLj490YsRCiIoXNl0aN8rxKYG0tGCfVA78aWrY0Neu5uaamXQhneeDpLIRnmT3bPD78sBm5olwyjKtXOXDgAN988w1+fn4AaK256aabOH78OF9++SW//vorgwYNYsiQIZw4ccLN0QohHElNhWXLTMFh5Eh3R3MhTy5AQHHN+uzZMieEcJ7MAyFEOc6cgc8+M8/HjHEiQx2ugYhTcZVKH9ozlF6be12QP1bHFq2LvzyejC0ZDvOXTFcZ33zzDaGhoVitVrKzswF47bXXAFi1ahVbt24lOTmZoKAgAGbMmMGKFStYsGABU6ZMqdJ7CiGqz4IFkJNj5jZo2dLd0ThgK3j00K+Gm24yffsSEkyN+6BB7o5IeAMPPZ2F8AwffWS+mK69FmJinMggfSA83qBBg9i6dSsbN27k8ccf58Ybb+SJJ54AYPPmzWRmZtKwYUNCQ0OLlh07drB//343Ry6EKM1mg//8xzwfP969sZTF02sgfH2Lb5AV1rgLURGpgRCiDFoXX0zHjnUyUx0exrWsGoFz584RFhZWpfwlayhcJTg4mHbt2gHw5ptvMnjwYGbMmMH06dOx2Ww0btyY1atXX5AvPDzcqf2Hh4c7LGycPXsWi8VCaKjDiZeFEFXw3Xfw++/QogUMG+buaBzz9AIEmALEP/5hatzfeAPq13d3RMLTefDpLIR7bdgAO3dCw4Zw661OZpJhXL3OtGnTeOmll0hMTKRnz56cPHkSi8VCu3bt7JZGjRo5tb+OHTuya9cusrKy7NZv2bKFVq1aERAQUB1/hhB10syZ5nHCBCf6qLmLhzdhAmjdGq67ztS4Fw6HK0R5PPh0FsK93nnHPI4aBf7+TmaSJkxeJzY2lk6dOvHCCy8wdOhQ+vfvz7Bhw/j66685ePAg69evZ9q0aRfUSuzYsYOtW7faLTabjfvvvx9fX18efPBBNm/ezL59+5g3bx7//ve/mTx5spv+SiFqn3374OuvISAAHnnE3dGUzRtqIKC4M/Vbb5l7YUKUx8NPZyHc49QpWLTIVCY89lglMtbhTtTe7Omnn2bu3LkcOXKElStXMmTIEMaOHUvHjh256667SEhIoGnTpnZ5Bg8eTI8ePeyWzMxMIiMjWb16NVarlVtvvZXu3bvzxhtv8Nprr/FYpU4mIUR5Zs0yTU3vuw8aNHB3NGXzlgLELbdAq1amSdjy5e6ORng6T63wE8Kt3nvPjIt9883Qpk0lMsowrh7tgw8+cLj+vvvu47777it6/cYbb/DGG284TBsbG4uuYKzDDh06sGTJkirHKYQoX0YGvP++ef744+6NpUKFd/M9/KvB1xeefBImTYL/+z8YPtzdEQlP5uHlYSFqXl4evP22eV7pLyapgRBCiGq3YAGkp0P//tCjh7ujKV9RDYSPe+NwxpgxUK8erF0L69e7OxrhyeSXjhClLFsGx49Dx44wdGglM0sfCCGEqFZWK/z73+a5x9c+ANrmHU2YAEJDi4fDfeUV98YiPJsXnM5C1KzCUT0mTqxCZYI0YRJCiGq1dCns3WtGDrrjDndHU7GGtzWk45yOcKW7I3HOxIlm4JBly8xxFsIR6QMhRAnbtpmZOMPC4KGHqrADacIkhBDVRmv45z/N88mT3T90q9YaVXDT6EzWGZ5b9RxJGUmczjxNSlYKadlp5Nvyybfl80TME8QSC8CiHYt4d/O7NA1rSrOwZjQPb077qPZ0atiJFhEtsCj3fY9ER8PIkTB3Lrz+enGTXiFKkgKEECUUzmg6apQpRFSa1Yr283NlSB6p5Jem8EwVdfQWwht9/z1s2QKNGsHDD9fse9u0jV3Ju1h9eDWbEjfxa9KvtI5szdK7lwIQ4BPArE2zysyfbc0uen4w9SBxh+IcposKiiJ5cnJRIeLU+VM0DG5Yo9fcp582BYgPPoDp06Fx4xp7a+ElpAAhRIFTp+Cjj8zzP/yhijux2cDHC3rKXQQ/Pz+ysrIIDg52dyiiHHl5efi6+/asEC6ktfkxC/DUUxAUVDPv+9Xer5jz6xx+PvwzZ7LO2G07df5U0fMQ/xDeuvEtooKiaBDcgPrB9YkMjCR/Qz55u/M4ln6sKO2D3R6kd7PeHE8/TuK5RI6kHSEhJYFdybtoFNKoqPCgtebSWZcS5h/G0DZDGdpmKNe0uYb6wdU7VfSll5oJVJcvN30h/u//qvXthBeSbxchCsycCdnZZujWjh2ruJM60AeiUaNGHD9+nGbNmhEUFCQ1ER7IZrNx8uRJIiIi3B2KEC7z7bdmZKAGDS7iJo8TdpzaQb3AejQLbwZAQkoCy/YsA6BZWDMGtRpE3+Z96Rndk66Nu9rlHd97/AX7+33l75yceRK/PxTXTjcLb1a0/9Iy8zKLnidlJAFwOO0wc3+dy9xf5+KjfIhtHcvtl97OXZ3vokFw9UyCMX26KUDMmmVqJKKjq+VthJeSAoQQmDHFZxXUPD/77EXsqA70gQgPDwcgMTGRvLy8CtNnZ2cTGBhY3WF5jZo6HiEhITTw5Nm1hKgEreG558zzZ581owW5UsLpBD7d+Smf7vyUXcm7+OvAvzJjyAwAbr/09qIagNaRrSt90yRiQAQ6X5PYNtGp9MF+xbW70WHRJE9OZmvSVv534H98t/87fjr8Ez8c/IEfDv5A9ybdiwoQNm1zad+JHj3gtttMp/V//QvKmBpH1FEuK0AopaKAucC1wGngT1rrT8pIOx34C5BTYnVXrfUBV8UjRGW89x6kpkK/fjBgwEXsqI4M4xoeHl5UkKhIXFwcPTx9oPYaJMdDiMpbuhQ2bTJt8SdMcM0+03PSWfjbQub8Oof4xPii9VFBUfhain8etY5szdjLx1b5fRrd1YhGdzUiMc65AkRpFmWhZ3RPekb3ZEr/KaRmpbJi7wp+OPgDVzYvHtpp+KLh+Fh8eLj7w9zQ7gb8fC6+P9706ebYv/OOmWSudeuL3qWoJVxZAzELyAUaA92Br5RS27TWO8tI/6nW+gEXvr8QVZKbC6+9Zp5fVO0D1IkmTEJUlrM3mJRSk4GHgFYF6d7SWsto9HVcXh5MnWqeT5sGrup+NXHlRBZsXwBAeEA4t11yG3d3vpuhbYa65Md3dakXVI8Huz3Ig90eLFqXmpXKt/u/Jdeay7I9y2gY3JCRXUcyvvd42kW1q/J7de0K998PH38Mf/mLeRQCXDQPhFIqBLgD+JvWOkNrvQZYDox0xf6FqE4LF8KxY9Cpk+n/cFHqQBMmIaqg5A2m+4G3lVKdHaRTwINAPeB6YKJS6p4ai1J4pNmz4fffoUMHeOSRqu0jJz+H9399n3VH1xWte7j7w8S2juWj2z4i6ekkPhj+ATe0d82d+5Ky9meR/ks6nHXpbu3UC6rHoT8e4uWhL3Npg0tJzkzmtQ2v0WFmB2765Cb2plR9QocXXjDzQnzyCWze7MKghVdzVQ1EByBfa13yDN0GXFVOnluUUmeAE8B/tNYORxpWSo0DxgE0btyYuLg410R8ETIyMjwiDk/hzcfDZoNp03oDIdxyy25+/vnkRe3v0sREsj3kPPUU3nx+VIe6djxK3GDqorXOANYopQpvME0tmVZr/XKJlwlKqS+A/sCimopXeJYzZ4pHXnrpJajsKNkpmSm8E/8OMzfO5OT5k1zX9jq+eeAbAAbHDGZwzGDXBuzA4RcOk/RBEkwGhlff+0SHRTO5/2Se6fcMmxI38Xb82yz8bSHf7/+e8IDiJqdWmxUfi/OjBbZuDU88YUZievJJ+OknkIp24aoCRCiQXmpdGlDWSPqLgdnASaAP8LlS6qzWemHphFrr2QVp6dWrl46NjXVRyFUXFxeHJ8ThKbz5eHzxBRw+DM2bw/PPX4q//6UXt8N33iElONhrj0d18ObzozrUweNRlRtMKNNTdSDwbhnb5eaSh3PF8XjjjfacPt2M7t1TiYjYhrO7O551nM+OfcY3Sd+QbTPzL7QNacvlfpezatWqmh09rqDrQ3Zedo2eHw9FPMTwPsPZkbaDPfF72MMerNrKmPgxdIvsxp3N7qR5cHOn9nXVVb7MmXMFq1f787e/7WLo0FMVZ6qA/L/Y87bj4VQBQikVR9kX+7XA40DpHpXhwDlHGbTWu0q8XKeUegMYAVxQgBCiupjaB/P8mWdMFa0rdip9IISwU9kbTIWmY5rZznO0UW4ueb6LPR5bt5phRH18YMGCenTp4ty+Fu1YxMjPR6Ixkyle3+56nu77NFfHXO2WYad3zdnFKU4RGBTolvNjGMOKnq85sobDPx/mcOZhViSu4LZLb2Nyv8l2nbHL8uqrMGYMzJvXiWef7VS1yVZLkP8Xe952PJxqrK21jtVaqzKWAcBewFcp1b5Etm5AWR2oL3gLTNtXIWrM55/Dtm3QrBk8+qiLdmq1Sh8IIexlUIkbTABKqYmYvhA3aa1zykonai+r1VyXbTaYOBG6dCknrc3K/jP7i14PiRlCeEA4D3d/mN/G/8bX93/N0DZD3TdnjbXg0QO+Gga0HMDOCTsZ02MMfj5+LNm9hL5z+zJw3kCWJyzHpm1l5h01Cq64AhIT4a9/rbmYhWdyyemstT4PLAGeV0qFKKX6A8OABY7SK6WGKaXqKeMK4AngC1fEIoQzrNbiMcX/9jdw2bD8NludGMZViEqo1A0mpdRoTN+Iq7XWxxylEbXfzJmwcaO5wfP8847TnM89z382/ocO/+nAkPlDyLflA9AopBHHnzrO+8Pep0ujckoeNUTbTE2IJxQgADo17MScW+dw6I+H+NOAPxEZGMmaI2t44usnsNqsZeazWMxwrj4+5vPZsKEGgxYex5Wn8wQgCDiFaYo0vnAIV6XUQKVURom09wD7MHeg5gMvaa0/dGEsQpTr449hzx6IiYGHH3bhjqUJkxB2KnODSSl1P/AicI3MC1R3HThghgwFePttKD3lzIlzJ/jLD3+hxestePzrxzmQegAf5cOhs4eK0oT4h9RcwBXQVs8qQBSKDovmxatf5MikI7x+3evMGDyjaASqU+dP8eLqFzmTdcYuT48epsmv1qY5U3a2OyIXnsBl80Borc9QxvgCWuvVmHawha/vddX7ClFZeXnFo3pMn+6ivg+FZBhXIRyZALyPucGUQsENJqXUQOBrrXXh98MLQH1gU4nmJh9prR+r6YCFe1it8OCDkJkJd98Nt9xSvO1s9lme+vYpPv7tY3KtuQBc2fxKnu77NLddclulRhaqUR7UhMmRsIAwJl05yW7drI2zeP7n5/nH6n8wpscYnrzySWLqxQCm7+CSJbBrl6nBf0VmaqmTPPR0FqL6zJsHBw/CJZeYCXJcymqVGgghStFan9FaD9dah2itWxZOIqe1Xl2i8IDWOkZr7ae1Di2xSOGhDnn5ZVi7FqKjYdYs+21h/mHEHYojz5rHbZfcxtrRa1k/Zj0jOo3w3MIDnteEyRlDYoZwbdtryczLZObGmbSb2Y67P7ubA67raAAAIABJREFUTcc3ERQECxaYpkyvvorTI2OJ2sWLTmchLt65c8W1D3//u7kAupTNVg07FUKI2m/9+uK+ae/Nyefr4x/RZ04fTpw7AYCPxYf3h73P3sf3suTuJfRr0c+N0TrPU5swleeq1lfx7QPfsu2xbTzY7UEsysLinYu5Ys4VPPv9s/TpY5qZaQ0PPADJye6OWNQ0Lzqdhbh4//wnnDhhRpIYMaIa3kD6QAghRKWdPg133QX5+TDgrk08lhDDyKUj2Xh8I+9tea8oXWzrWNpFtXNjpFXg4U2YytO1cVc+HP4hB/94kCn9phAeEM61ba8FzEhMva/M5fhxU4iwlt3/WtRCXng6C1E1+/eb6laAN9+spq4KMoyrEEJUSn4+DL8zg2PHwNJiA2s69uNY+jE6NezE3FvnMqX/FHeHeFG8sQlTac3Dm/PSNS9x7MljDIkZAphZwes/+AdUyGm++w6enprp5ihFTfLi01mIynn6acjNNR30+vSppjeRYVyFEKJSnnkG1saFQvApbCPu5LoOV/PN/d/w2/jfGN1jNIG+rhpn2z28sQlTWcICworm08jJz+G071b07feCyueN/wtm8B8XsDt5t5ujFDXBZaMwCeHJvv8evvgCQkPhX/+qxjeSJkxCCFGurLwsPv7tYy5rdBlbVvThjTfA18/Gjc99wD9Hf0unhp3cHaJLNbi1AcHtg0lslOjuUFwqwDeAjY9sZNXQVTzh+zY75z1O3H/uoVPqDVw9FN684c1a91mKYlKAELVeXh5MKhih7q9/NaN7VBsZxlUIIRzac3oP78a/y4fbPiQ1O5VeaS+y+d+mOnj2uxYefti7myqVpfkTzQFIjKtdBQgApRRDYoaw4/0hjA4+zbxZDWDx58RFDCLytsiidFab1aNHyhKVJwUIUeu9/LIZr7pt2+KCRLWRYVyFEKJIrjWXpbuX8s7md4g7FFe0vkPq42x7ewpaw4wZLp7QU7jFnDcbcO4kfPZZBJH/3cD5J4IgDGzaRrd3utEzuieP9HyEgS0HouR70utJAULUajt2mOFaAd59FwICqvkNZRhXIYQo8tr61/jTD38CIMQvhPsuu49u55/hmYc7kJcLEycWzzpdW2X8loEt0wZZ7o6kelksMH8+nDwJq1cHMWgQ/PADnI/YzK7kXexM3smC7QvoUL8Do7uPpn1ue3eHLC6CtLUQtVZ+vrmrlZcHjz4KV19dA28qfSCEEHVUalYq78S/w/xt84vWjew6kp7RPXnrxrdIfDqRqzNn89RDHcjONtflN96A2n7JTBiTwJYrt8BBd0dS/YKC4OuvYfBgSEqC2FgISOnNvif28ecBf6ZpWFP2puxl6g9TuXP9nQxbNIzk8zKJhDeSAoSotV59FeLjoUUL04ypRsgwrkKIOiTPmseKhBXc+d87afJqE8Z/NZ4ZP89AazPyULPwZmwet5nxvcczZ1Y499xjRsN7/HF46626cbkM6RxCWO8wCHZ3JDUjJAS+/BKuvdZMMDd4MBzf0YZ/XP0PDk86zIp7VzD8kuEopdhyYgtRQVFFeXcl7yo6d4RnkyZMolbatat4RtM5cyA8vIbeWIZxFULUATtP7WTWpll8tuszkjPNHWSF4po21/BgtwexaVtRWqsVJk+G1183r19+2QzdWttrHgpdMu8SAOLi4twbSA0KDjYjH955pylMXH01zJwJjz7qy80dbubmDjez5LslNLqkUVHn6lPnT9H17a40D2/OPV3u4d4u99K1cVfpL+GhpAAhap2sLLj/fnOXa8wYcxekxkgTJiFELZRnzSM1O5VGIY0AOJJ2hLfj3wbg0gaX8lC3h7i/6/00D29uly8x0cxSvGqVmXjsgw/gvvtqOnrhDoGBsHQpTJliCo+PPQZbt5pma/7+EOUfxYCWA4rS7zuzjyahTTicdpiX1r7ES2tf4tIGl3JnpzsZdskwejTpIYUJDyK3SkWt88c/motUu3bFM0/XGBnGVQhRS6TnpLNk9xJGfzGaJq824bEvHyvadnWbq3lu0HNsGbeFnRN28uyAZy8oPGzcGEX37qbw0LgxfPdd3Sw82PJsZjK5Otgyx9cXXnvNFBwDAuCdd2DAANjtYK65fi36ceTJI/w06iceu/wx6gfVZ/fp3Tz/8/Nc8d4VpGanFqWVZk7uJzUQolaZPx/ee89cqP77X4iIqOEAZBhXIYQXO5B6gCW7l7Dy95WsPrKafFt+0bYjaUewaRsWZcHfx5+/D/67w32kp5s5d2bO7ArA0KHw0UemEFEXxXeNJ3NPJnzg7kjc56GH4NJLYcQI2LQJevSA0aObM3Cg/cCFFmVhUKtBDGo1iDdveJMfDv7Asj3LSMtJK+orYdM2Or/Vme5NunN92+u5pu01NA1r6qa/rO6SAoSoNXbsMFWkALNmQffubghChnEVQniRkxkn0WiahDYBYHnCciZ/PxkwP+YGtBzADe1uYPglwyucVVhrc+Nm0iQ4cQIsFs2MGYqpU+t2xay2Ftwtr+P3lq64An77DZ58EubNg7ffbse2bfDmm3D55Rem9/Px4/p213N9u+vt1m9L2sae03vYc3oPi3YsAqBTw05c0+YarmlzDYNjBhPsV0d6rLuRFCBErZCaau5sZGWZOx2jR7spEOkDIYTwYCczTvLT4Z+IOxRH3KE4dp/ezdT+U/nn0H8CcHOHm/k16VdubHcj17a9lnpB9Zzab3w8/PnP8P335vWVV8KYMZt55JFe1fWneI2iAoTcWyIiAt5/H26/HUaNymHdugB69YKRI+Ef/zCjJlakR3QP9j2+jxV7V/D9ge/56dBP7Erexa7kXbzxyxvEj43n8qamRPJ7yu80CmlERGBNN0eo/aQAIbxedjYMGwYJCXDZZWZoQLf9hpdhXIUQHmjaqml8uvNTElIS7NYH+wWTlV88w1m7qHZ8OPxDp/e7dStMmwbLl5vX9erBv/4FjzwCP/+c4ZLYvV7hgFRyb6nIzTfDBx9s4qefBvDmm7Bggam9euwxU0PRsmX5+dtGtWXSlZOYdOUkcq25bDi2ge/3f88vx3+he5Pi5gejl49m7ZG1XNb4Mga0GED/lv3p16IfrSJaSYfsiyQFCOHVbDZz52L1amjWDL76ygwf586AZBhXIURNy7flk3A6gW0ntxGfGM8vx3/hk9s/oVVkKwASzyWSkJJAsF8w/Vv0J7Z1LLGtY+nVtBf+Pv6Vei+rFVauhP/8x3SMBnPdnTjRDNfaoIGr/zrvVlQDIV8NdkJD83nlFRg/HqZONQWIf//bnFf33gtPPw3dulW8H38f/6J+EyVprfGz+OFr8WX7ye1sP7mdt+LfAqB+UH2mx05n4hUTAfP/46N8pFBRCVKAEF5La3jqKfjsMzPPw9dfO1f9Wa2kCZMQooaczjzNlO+nsO3kNnae2kmONcdu+4ZjG4oKEE/2fZJxl4+jW5NulS4wFDpwABYtMgNVHDpk1gUFmbvGzz5bdztJV0SaMJWvTRtYvBh+/RVeeQU+/dTUSCxYAL16mSbJ994LkZGV269Sih8f+pGsvCziE+NZe3Qta46sYcOxDaRkpRDiF1KUduFvC5n07SS6N+lO54adzdLIPDrbjK+ukQKE8Epaw/PPm/Gk/fxg2TLTfMntpAmTEMIFsvKy+P3M7yScTmBvyl4SUhJISEmgcUhjlt9r2guF+ocyf9t8rNoKQExkDN2adKN74+70ad6Hvs37Fu2vog7QZdm/H1asMAWHX34pXt+mDUyYAA8/DFFRZecXSBMmJ/XoAZ98YvpCvP66GVUxPt4sTz5p5nS67Ta45ZbK1XIF+QUxsNVABrYaCJiaiaPpRwkPKJ5hdlfyLs5kneHHgz/y48Ef7fK3i2rH3ol7i2on4g7F0SS0CTGRMQT4Blz8H+6lpAAhvI7W8Kc/wUsvmd/q8+fD4MHujqqA1EAIIZyQnZ/NkbQjHDp7iMNnD3Ndu+toGWEafk9bNY0ZP89AO5g4oHFI8W3+QN9APhz+Ia0iW9G1cVe7H0RVlZIC69bB//5nanV//714W3Cw6W/2wANw3XUy4JyzpAlT5cTEmJGZXnrJ3Bx8/3344QdTkF2xwnzv9+ljZrceMgT69jWT1jlLKVX0v1boxatfZELvCWw/uZ2dyTvNcmonu5J3EREQUVR4sGkbN3x8A9n52SjMftpGtaVdvXa0jWrLzR1urnJh3dtIAUJ4FZvNDBE4c6aZoOajj+Duu90dVQkyjKsQdVpOfg4nz5/kxLkTAPRp3geAcznnuPfze0nKSOJY+jFOnj9pl+/TEZ8W/aiJDIzEx+JDm3pt6Fi/o1kamMcO9TvY5bu/6/1VjjU9HbZvh23bTGfodetg1y77NBER5s7v7bebO78hIY73JUp5+WXo3RsGD0bbymjCtGqVmRRhypQaD88bBAWZpkv33muGBV6+3Mxs/eOPsH69WV54wcxq3bWrOdy9epmlUyfzG8FZSilaRLSgRUQLbupwU9F6m7ZxJutM0euM3AwGthzIvjP7OJx2uGgprLWIDo0uKkC8tektZvw8gxbhLWge3pwW4S1oGtaURiGNaBLahBva3+CaA+UmUoAQXiM317S1nTfPXDAWLzZ3wzyK1EAI4fW01mTmZZKWk0Z6TjrpOemczT5LSmYKKVkp3NX5LhqFNALglbWv8OnOT0nJSuHUuVNk/pRZtJ8rml3BL4+Ydj/BfsF8ve9rbNq0Z/G1+NIivAWtIlvRKqKV3URYj/Z6lIlXTMTPx++i/5b8fDhyxDRFKlz27TMFhwMHLkwfEGDu7g4cCNdfb4ZjrcwPMVGgd2+46y7zRWUt+BxLfjWsWlW8XVQoOhoefdQs6enw88+mIPHjj6YAXNjUqVBgILRvDx06QMeOZunQAVq1Mn11nG1pbFEWGgQXt5cKDwjnu5Fm5IBcay6Hzx5mf+p+9p3Zx/4z++kR3aMo7ZG0IyRlJJGUkcSmxE12+20c0pikZ5KKXnd5qwup51Jptb8VjUIa0TC4IfWC6lEvsB5DYoYU3YhIy07j5PmT1AusR2RgpEuuEVUllwXhFU6ehDvugLVrzYVh2TJThe5xpA+EEBdQSkUBc4FrgdPAn7TWnzhIp4B/AY8UrJoDTNVaX9iWp4Q8Wx4HUg+QmZdpt2TlZTHskmH4WsxX3YdbP2Rvyl4y8zI5n3e+qHCQnpPOkJghPD/4eQB2nNpB13e6lvl+3Zt0LypAJJ5LZPOJzUXbfJQPjUMbEx0aTZeGXYrXW3xYce8K6gfVp1l4M6JDo/GxOK6tLG8SLKvV/IA6e7Z4SU0118gTJ+yXpCSz3mp1vC9/f+jc2Uy62a2bmeirZ09TiBAXafBgUzi46y507mJA0WLxQrBkmC+xe+812z2m/a33CA83w8DefLN5nZZmOmAXFiLi401B+bffzFKar68ZtbFFC2je3BQoGjY0/SoaNLB/Hh5u/h8c3Rf09/Gnff32tK/f3mGcLwx5gT/0/gPH0o9xNP0oR9OOkpSRxKnMUwT7Fv+Pa63Zd2YfOdYcEo8lXrCfl4a+VFSA+Hb/t9z9WXGzi1D/UML8wwj1DyXUP5SfRv1EWEAYAK+ue5UjaUcICyjeHuofSpBvEG2j2tKrqZmjJSc/h4NnDxLkG0SQX5BTnwFIAUJ4gS1bYPhwOHrU/NMvXWpu7ngkGcZVCEdmAblAY6A78JVSapvWemepdOOA4UA3QAPfAweBd8rb+fak7bR9vQOgQKuCRwtoxbGnEokIiMRmg/c3fM7Ph362216YJyK/IycuMX2sMs/WIyCjHWH+4YT6h5lHv3AiA+oRGVCPcyeakJBl0l4b9hQ9rxxDqE8k+3Ye5oru/bBaFXl55u7/smXm0by+kZP5sK1gW3a2mfwyM9M8Fi6lX2dkFBcW0tMrf/CbNYO2be2XLl3MXVk/993ArP0GD4aFC9HXZANBtPp4AXw+v3gcXCk8uEREBMTGmqVQejrs3WuWhASz/P67+R2RnAyHD5vFGb6+EBbmeAkJMQWMwiUwsORrXwICWhAY2IKAgL60DoD2fuDrb1o6/+9/5tFigWX9ktmwdQ3NO0RzNieFtNwzZOSncS73LC3yr+LgQZM2LTmUVpYrScs9S1rOWTIyrGSoPOAMqBTOpweQ72sKPIs2f0184kZQGtB2j/dddh/vD5+LUrAr+Xd6vtvdbFe2Co5GiePi/EckRM2y2eDtt+GZZ8wXbd++8PnnpirTY0kTJiHsKKVCgDuALlrrDGCNUmo5MBKYWir5Q8CrWutjBXlfBcZSQQGi/YkO/GfGbIfbdr64rej5X3iav/A084hhEaa/wQCS+Ru7WEsDmj5q0jWmPsuZU847HucwxwHTrL0xsBbF8/QvyJ/FfDZyhkAepE9Rrs9YRxh55f0pdk6WkX90aD98ovyIjIQ/HNtOm7OpKArukioueE5ywbLB7KdfYj/86puSw/abtpP6v1Qu+/Iyoq4xwykd/NtBjrx8xOk4Acf5RwGxZnvy0mR23bOrrOwOxTwfQ8tnW9rlbzC8AZ0/7QxA1qEsNnbcWKl9Osof2CqQPnuLj/O66HXknXH+cyorf7/EfvhlZxOmDmDTfijyITvP/MrMzq5U3KJywsOL+0OUlp0Nx46ZwsSxY6ZAkZwMp08XPxYu6emm8J+aapbqoYAwwHGfiLl2r24sWByLfrnkq/+Vme6TgsXoAuSXiqdiUoAQHunoUTP28/8Kzv8xY2DWLC+oWpcmTEKU1gHI11rvLbFuG3CVg7SdC7aVTNfZ0U6VUuMwNRZ0oAP+DkYsKksImdTjDApNOOn4owkmm8YkodA0IqdS+wNowBmuZD1+5FGffPyxEM55hrEMX/LxI48QGuFfickAmnOC1QwghPPUI5WjvImVcA5mNMAvIx2OwDZeJpWyq2Qd/hUNGgDpBdtfRtMbrr0OiC9Y9wiaSnbOdpA/ZvYcmF14l30Qmr9Xapd66p9h6sd2+fXiz2FxYdOwxmgWVW6fDvLbfj8I6sqiNDaWoYlwep9l5i84zj1KZ8jONj3S67hYN71vINCuYHFGDv6cI8zhkkkwOQSQTSA5BFywlF6fiz9WfBwu+fg6tc2GBY2qpsX53y9SgBAexWo1naSfeca0a6xfH955B0aMcHdkTpIaCCFKC6Xw12qxNMwtN0dp00qlC1VKqdL9ILTWs4HZAL0u76UHrbOfhbY8g3xgjq/5otTWKHR+DIMs8KyfpXDf6NyW5e3iwn1a4LK1PxMbG1uQ34Q7LKD4C9mW43zzgEKWgOFFz1sW5Ff+Z4saZV+WayujlFA2R/mV30awmHWt8220LqPfRJn7dJD/5zXQ6uqPAGhg1QzKr2SgPoPAt1R+yyDwew6AQK0ZlFvJfZaVP+CeoiT9qvA5Ocqv/M/CV1/BnXfa1zgEBppplwsb8ddRcXFxxJZsd+ShAgqW6p5g3VOOh7M/YaQAITzG6tXwxz+azlBgbtDMng1Nmrg3rkqRYVyFKC0DKD1BQThwzom04UBGRZ2oUWAJqFrNn/JRKB/7b0ylFCqg6jcCyspf1RjLy2/xv8h9Oshv8bVc1K+DovwlLoWOjnNleO3nFBho7owFBqJzclABAUWvhfBm0tZCuN1vv5kbNIMGmcJD8+ZmNsovvvCywgNIDYQQF9oL+CqlSg5V0g0o3YGagnXdnEgnhOdbtcqMtrRyJfz3vxx6+GFT87BypVm/apW7IxSiyqQGQrjNL7/Aiy+ayWHATBozZYpZgssexdCzSR8IIexorc8rpZYAzyulHsGMwjQM6Ocg+XzgKaXUSkzDnKeBmTUWrBCuUnKeh4LRlg6HhhJT2ESlYIhXGcpVeCspQIgalZlprpezZ5tZJMHU5I4dC5Mnm3GZvZoM4yqEIxOA94FTQAowXmu9Uyk1EPhaax1akO5doA1QOHr7nIJ1QniXTZvKLxwUzhOxaZMUIIRXkgKEqHb5+fDTT+ZauWhR8Tjm4eEwYQJMmmQmcvF6hc20pQmTEHa01mcw8zuUXr8a03G68LUGphQsQnivKU6cwoMHS+FBeC0pQIhqkZZmanBXrjQTKSUnF2+78koYN87U3oaEuC9Gl5PmS0IIIYSoA6QAIVzizBnYsAHWrTMFh19+Mb+nC7VvbzpK33MPXHaZ++KsVjabFCCEEEIIUetJAUJUis1mpn/fvt2MnrR9O2zYcAVHj9qn8/WFAQPgmmtg+HBTaKj1LXtkCFchhBBC1AEuKUAopSZiJq2/DFiotR5VQfongWeBYOAzTIe6HFfEIi6O1WqaGyUlwfHjcPAgHDhQ/Lh/P5w/XzpXMAEB0Ls39OsH/ftDbKzp41CnSA2EEEIIIeoAV9VAJAIvANcBQeUlVEpdB0wFhhTkWwr8vWCduEhaQ04OZGWZiS8zMuDsWdMnIS2t+HnJx5MnTYEhKckUHmwVTMLZpImpUeja1Tzm5MQzalQv/P1r5m/0WNIHQgghhBB1gEsKEFrrJQBKqV5A8wqSPwTM1VrvLMgzA/gYJwoQqalmDpbCwW60Ll7Ke+3qtHv3NmP79srtV2vzw9xqNaMSFS6VfZ2bawoGhQWErCz759nZFR3FijVsaAoJTZpATAy0aVO8xMRAVJR9+ri4DCk8gDRhEkIIIUSd4I4+EJ2BL0q83gY0VkrV11qnlJfxwAEzco/7ta84iRv5q1wCVQ5BlhxCLFlE+pwjwpJhHn0yiPRJt3vd2DeFJr6naeKXQkPfVPxUvtnRyYJlQ/nv1ysjA0JDy09UF1itSElKCCGEELWdOwoQoUBaideFz8MwEwzZUUqNA8YBBPlfSt8uB0FpVNF2UCVeF24r7LCrlC6xL8fbSr5WlL9NKU1evhU/X58L8xXmKRFX4WsUWJTGx2LDx6fg0aLx8Sl4tBRuM68tFo1vibSWgu2+PjYC/K0E+uXj72clwN9KQMHzQL98/PxM/or5AVEFS0uswPGCpbIys7IIDiq35VqdkR8aSkZGBnFxce4OxWPI8bAnx0MIIYS3q7AAoZSKA64qY/NarfWASr5nBlCye23h83OOEmutZwOzAXr16qV/iI+p5Nu5XlxcHLGF09EL4uLi6C3Ho4icH/bkeNiT4yGEEMLbVViA0FrHuvg9dwLdgMUFr7sBJytqviSEEEIIIYRwP5cMGaOU8lVKBQI+gI9SKlApVVbhZD4wRinVSSkVCfwV+MAVcQghhBBCCCGql6vGnPwrkIUZSemBgud/BVBKtVRKZSilWgJorb8BXgZWAUeAw8A0F8UhhBBCCCGEqEauGsZ1OjC9jG1HMB2nS657DXjNFe8thBBCCCGEqDky65UQQgghhBDCaVKAEEIIIYQQQjhNChBCCCGEEEIIp0kBQgghhBBCCOE0KUAIIYQQQgghnCYFCCGEEEIIIYTTpAAhhBBCCCGEcJoUIIQQQgghhBBOkwKEEEIIIYQQwmlSgBBCCCGEEEI4TQoQQgghqoVSKkoptVQpdV4pdVgpdV85aScrpXYopc4ppQ4qpSbXZKxCCCGc5+vuAIQQQtRas4BcoDHQHfhKKbVNa73TQVoFPAhsB9oC3ymljmqtF9VYtEIIIZwiNRBCCCFcTikVAtwB/E1rnaG1XgMsB0Y6Sq+1fllrvUVrna+1TgC+APrXXMRCCCGc5VU1EJs3bz6tlDrs7jiABsBpdwfhQeR42JPjYU+Ohz1POR6tqnn/HYB8rfXeEuu2AVdVlFEppYCBwLvlpBkHjCt4maGUSriIWF3FUz5bTyHHw54cD3tyPOx5yvFw6rvBqwoQWuuG7o4BQCkVr7Xu5e44PIUcD3tyPOzJ8bBXh45HKJBeal0aEOZE3umYGvJ5ZSXQWs8GZlc1uOpQhz5bp8jxsCfHw54cD3vedjykCZMQQohKU0rFKaV0GcsaIAMIL5UtHDhXwX4nYvpC3KS1zqme6IUQQlwMr6qBEEII4Rm01rHlbS/oA+GrlGqvtf69YHU3wFEH6sI8o4GpwCCt9TFXxSqEEMK1pAaiajyq2twDyPGwJ8fDnhwPe3XieGitzwNLgOeVUiFKqf7AMGCBo/RKqfuBF4FrtNYHai5Sl6oTn20lyPGwJ8fDnhwPe151PJTW2t0xCCGEqIWUUlHA+8A1QAowVWv9ScG2gcDXWuvQgtcHgeZAyWZLH2mtH6vZqIUQQlREChBCCCGEEEIIp0kTJiGEEEIIIYTTpAAhhBBCCCGEcJoUIFxAKdVeKZWtlPrI3bG4i1IqQCk1Vyl1WCl1Tim1VSl1g7vjqklKqSil1FKl1PmC43Cfu2NyFzkfyibXi7pBPme5DhSS74Zick6UzduuGVKAcI1ZwCZ3B+FmvsBRzCyzEcBfgcVKqdZujKmmzQJygcbA/cDbSqnO7g3JbeR8KJtcL+oG+ZzlOlBIvhuKyTlRNq+6ZkgB4iIppe4BzgI/uDsWd9Jan9daT9daH9Ja27TWXwIHgcvdHVtNKBjz/g7gb1rrDK31GmA5MNK9kblHXT8fyiLXi7pBPmdDrgPy3VCanBOOeeM1QwoQF0EpFQ48Dzzl7lg8jVKqMdCBciaNqmU6APla670l1m0D6updJjt18Hy4gFwv6gb5nMtWR68D8t1Qjjp6Ttjx1muGFCAuzgxgrsyYak8p5Qd8DHyotd7j7nhqSCiQXmpdGhDmhlg8Sh09HxyR60XdIJ+zA3X4OiDfDWWow+dEaV55zZACRBmUUnFKKV3GskYp1R0YCrzu7lhrQkXHo0Q6C2am2VxgotsCrnkZQHipdeHAOTfE4jHq8Plgp65dL2or+V6wJ98LTpHvBgfq+DlRxJuvGb7uDsBTaa1jy9uulJoEtAaOKKXA3GXwUUp10lr3rPYAa1hFxwNAmQMxF9NR7EatdV51x+VB9gK+Sqn2WuvfC9Z1o25Xy9bl86G0WOrQ9aK2ku8Fe/K94BT5bihFzgk7sXjpNUNmoq4ipVQw9ncVnsGcBON3lD2JAAAA9UlEQVS11sluCcrNlFLvAN2BoVrrDHfHU9OUUosADTyCOQ4rgX5a6zr5RVHXz4eS5HpRN8jnfCG5Dsh3Q2lyThTz5muG1EBUkdY6E8gsfK2UygCyPf0Dry5KqVbAo0AOkFRQkgZ4VGv9sdsCq1kTgPeBU0AK5gJQV78g5HwoQa4XdYN8zvbkOlBEvhsKyDlhz5uvGVIDIYQQQgghhHCadKIWQgghhBBCOE0KEEIIIYQQQginSQFCCCGEEEII4TQpQAghhBBCCCGcJgUIIYQQQgghhNOkACGEEEIIIYRwmhQghBBCCCGEEE6TAoQQQgghhBDCaf8P25CxgbsuqcUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 792x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "z = np.linspace(-5, 5, 200)\n",
    "\n",
    "plt.figure(figsize=(11,4))\n",
    "\n",
    "plt.subplot(121)\n",
    "plt.plot(z, np.sign(z), \"r-\", linewidth=1, label=\"Step\")\n",
    "plt.plot(z, sigmoid(z), \"g--\", linewidth=2, label=\"Sigmoid\")\n",
    "plt.plot(z, np.tanh(z), \"b-\", linewidth=2, label=\"Tanh\")\n",
    "plt.plot(z, relu(z), \"m-.\", linewidth=2, label=\"ReLU\")\n",
    "plt.grid(True)\n",
    "plt.legend(loc=\"center right\", fontsize=14)\n",
    "plt.title(\"Activation functions\", fontsize=14)\n",
    "plt.axis([-5, 5, -1.2, 1.2])\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.plot(z, derivative(np.sign, z), \"r-\", linewidth=1, label=\"Step\")\n",
    "plt.plot(0, 0, \"ro\", markersize=5)\n",
    "plt.plot(0, 0, \"rx\", markersize=10)\n",
    "plt.plot(z, derivative(sigmoid, z), \"g--\", linewidth=2, label=\"Sigmoid\")\n",
    "plt.plot(z, derivative(np.tanh, z), \"b-\", linewidth=2, label=\"Tanh\")\n",
    "plt.plot(z, derivative(relu, z), \"m-.\", linewidth=2, label=\"ReLU\")\n",
    "plt.grid(True)\n",
    "#plt.legend(loc=\"center right\", fontsize=14)\n",
    "plt.title(\"Derivatives\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.2, 1.2])\n",
    "\n",
    "save_fig(\"activation_functions_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def heaviside(z):\n",
    "    return (z >= 0).astype(z.dtype)\n",
    "\n",
    "def mlp_xor(x1, x2, activation=heaviside):\n",
    "    return activation(-activation(x1 + x2 - 1.5) + activation(x1 + x2 - 0.5) - 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmkAAAEMCAYAAACfoCGmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XmcXHWZ7/HPk3S6sxFISEwMkATDEsRrkEUZEUUdxu0yhuXeURBc0LgML1fM5XpFkXEUM+I4IKKMLGMUrldZ3MaFGfUOyHhFuKhwDWggYZsEJE2SztKd7n7uH+cUVirV3VWnfmet7/v1qlfSVafOeaq6+lff+tV5zjF3R0RERESKZVLeBYiIiIjI3hTSRERERApIIU1ERESkgBTSRERERApIIU1ERESkgBTSRERERApIIS0DZrbezM7PYDsXmdm9GWxnkpl92cyeMjM3s5PS3uYE9VxnZt/LYbtL4sd/bNbbblJLy8+BmZ0U1z13nGXOMDMdn0fGpbEt9XpyGdsmUqS6WqnFzL5nZtdlVFJQpuOk7cnMjgbuBH7h7ie0ed+LgDPc/XkN188Dtrv7jkA1LgEeAo5z91/VXT8T6HP3p0JsZ5zt/2fgJuAk4EFgs7sPpbnNeLsnAT8F5rn7H+uu35fotfx02jU01LOEJr+HPLTzHJhZLzAH2ORjDABmdgbwTXe3sJVKXjS2tbR9jW0tKFJdrdQSh7g/uvtbMisskJ68CyigtwNfBM4xsyPc/XedrtDdn+y8rJa2MwAMZLCpQ4D/cPc7MtjWhNx9S9415K2d5yB+09mYYjlSTBrbJqaxrQVFqqtItaTC3XWJL8A04GngPwFXA59tssxC4OvAU8AO4B7g5cBbAG+4vCW+z3rg/Pj/1wM3NqxzEvAI8MH451cDtwH9wGbgR8ARdcs3budn8fUXAfc2rPfCeN2DwG+B19fdviS+/+nArfHj+X/AyeM8R9c1bHt9fP3PgC80WfZ7dT//jOhN4lPAH4EngM8Ck+qW6Y1v3xDX/CDw3rpa6y/XjbGdPuDzwCZgF/AL4CV1t58U3/+VwP+JH/evgKPbfL209PwBzwW+D2yLH/MNwIK6248Dfhw/J1uB24E/q7u9lddM43Pw0vhxDwBbgF8Cz2t4/HPrlj8nfs53AN8D/hrwhm2eAtwVP6cPAX8L9Ob9d6tLS69VjW0a29p9zXysrtaNwFfHefwzgK8SjTebgP9ONI5cV7fM+nid1xGNhY8AfwXsB/zP+L6/B/6ioY6Xxo9lV7zuv6du3GlSy/T4ulotH2mspUyX3Aso0gU4G/h1/P+T4j+0KQ0vxN8DPwdOBJYCpxENZNPiP8q1wIL4Mq3uxVkbyF4bv9j2rVvvy4Fh4Nnxz6fHl0OB5wP/C/hD7YVJ9KbuwKvi7cyJr7+IPQeyDxC96Z8JHAZcDIwAR8W3L4nXs5boDfhQ4J+IBumZYzxH+wKfiP/AFhBNz0PrA9mWuI7DgP8aP+431i1zA/Bo/PifEz835wCT4+faiULPgtpz2GQ7/wD8B/A64AjgH+M/2GfX/W6dKLi8HFhG9GbxO+JdAOLlHLhonNfLhM8f8GyiQfszcS3PB75LNOhMipd5BdFr74i4li8QvYnt38Zr5pnngGiGvJ/o9bg0XueZxG+GNIQ04EXAKPA/4t/LO+PH4HXbexXRa+mt8TpfDtxPkzd7XYp3QWObxrb2xrbT4+f3dcAi4FjgvHEe/5eIAt3JwJFEoWsLe4e0zcB74t/HpUSvl3+On4dDiD5APAFMje9zALA9Xv8RwH8mCoyXjlPLF4HHiF5DzwO+GT+W68Z6vEW+5F5AkS7xH1ptwLH4RXVG3e3vIPoEMHeM+19E3UBSd/36uvX2EKX7c+tu/wrw43HqmkE0AL0k/nlJ/Ed27Hjbj1+oH2vyGL/WsJ531t1+QHzdS8ap53ziT5kN621lIPv3hmVuBb4S///QeNuvHmO7J9EwA9S4nfi5GgLOqbt9MrAO+GTDel5Vt8wJ8XUH1l23lrqBqUk9Ez5/RIP2vzbcb3a8zAvHWK8RDcRvavU10/AczInX/7JWnkeiGZBbG5b5CnuGtH8DLmxYZgXRG4Q1244uxbmgsW2vv80x6tHYFt3+QaIPYVPGuL2+rplxXW9o+L32s3dIu6Hu55lxXZfVXbfH759otv737Dkj+Rai2b3pY9QyCJzVsJ2nKWlIU3dnzMwOAV5C9IaFR7/drwPn1i32AuA3XrdjZ7vcfRj4BnBWvN0+ok8tX6urZamZXW9m68xsK9HAN4noE02rj2cW0dcXP2+46XaiT2v1flP3/8fjf5/V6rba9JuGnx+v29YLiGZ0ftrB+pcCU6h73O4+Avw7bT5ud1/m7l9oYZvjrecY4KVmNlC7EH1Sr9WKmT0r7ih7wMy2EL1ZPov4993Ka6aeu28mGrh+ZGbfN7MPmtl4r50jiJ6feo0/HwP8j4bHcT3RYLxgnHVLzjS2PUNj25/uN9HY9k1gKvCQmV1tZv8l/n2OV9cv69a/HWjWjfubumUGiL6O/W3d7Zsaaj2CqNFltG6Z24m+Oj5kjFp6qRu/4u38tsmypaDGgT95O9GnkofNnmloMwAzO8jdHxnrjgl8Dfh3MzuA6KumXqKOoprvEU2Lv5PoE+Mw0f4UvYG27w0/737mBnePH3+7AX6U+PmqM6XJcrsbfvYE20pqzMddd1uSWsZ7/iYR7Y/W7DAFtQHpn4D5RF/hrCf6JPiv7Pn7nug1swd3f6uZfZ5oH6C/BP7WzFa4+4/afXB1j+MTRIN3o0x2HpfENLahsa2dWtz9ETM7nGjftj8n+mry42b2ojiAJdXsOUpaa+NjriTNpAFm1gO8mWhnx6PqLsuJkv9b40X/L/D8cY4vNUQ0GI7L3X9JtB/GG4k+dX47TvuY2f5E+xF8yt3/xaMOrH3YM1DXWsLH3Ja7byX6BNXYav8SokExtCeJ9r+qt7zNddxD9Jp8+Ri3T/i4iab+h6h73GY2Gfgz0nncE7mbaB+NDe7+h4bLtniZlwCXu/v33f0+opm0PZ7L8V4zY3H3X7v7Z9z9JKKvY948xqK/A45vuK7x57uBZU0ewx/iGRQpII1tQXTl2Obuu+Ix6QNE+woeyd7Pea2u3fEytbqmE+0P1qnfAcebWX1WeQnR87BunFqeGb/MbEagWnKhkBZ5HTAX+Ed3v7f+QrQD5Fst+gh2PdFOjd82sxPN7Dlm9pdmVvvDWw8sNrOjzWzuONPDEH3d8PZ42/VfW/UT7Wj+DjM7xMxeRrTTZP0b4RPATuBVZjY/Pk5MM38HnG9mbzSzw8zsYqKdgj/b6hPThp8Ar4mfj8PN7HPAQe2swN0fINqR+CtmdrqZHRw/z2fHi2wg+vT0OjObFx87qXEd24Ergc+Y2WvN7Ij45/lEO5S2zMzWmtl57dyniSuIdkj+hpm9KH7N/LmZXWVm+8TLPAC8ycyea2bHEb3mmh2baazXTGPdB5vZJWb2YjNbHL8+n8/YA/llwJ+b2X83s0PN7B3AqQ3LXAycaWYXm9nzzGyZRQe8Xd3i8yD50NjWua4b28zsLWb2djP7T2Z2MFGY3020f1hjXQPANXFdrzSz5xLtiziJzme7vkj01fYXzewIM3sdcAnRPoJ7HZsvruXquJaTzezIuLYJP2AUlUJa5Fzgp978QInfJNqZ8eT4j+RlRNP13yX6zv0T/OmFeCNRp8q/En36euM42/wacDhRB8yPa1fG373/FdGb6r1Eb/IXEn0FVltmmKh1++1Enyi/PcY2LiMazFbH6zoVON3dfz1OXUldU3f5OdFs0M0J1nMO0RvGZUQ7t15HFHJw98eAjxPtTLqJqAuymf9GtG/MtUSfYJ9PtMPuf7RZy+FEb3CJuXvtE/8o8EPgPqLf6SB/+p2+jWjn1ruI3jivIXpTbNT0NdPEDqIOs28SBcB/Inrj/MwYNf6C6G/g3USzK6cR7ahdv8yPiN50X06078kvgQuAh8epQ/Knsa1z3Ti2PU302rmN6Pk9HTjN3R8aY/nz42W/Q7Tf3W+IDv2xq8269hA/L68h2qfvHqLfwQ1Eh9UYy/lxDTfH/95L1PhUSjrjgIiIiAQTz7RuAP7O3S/Nu54yU+OAiIiIJGZmLyDqxPwl0X6G/y3+9xt51lUFwb7uNLPzzOxXZjZo45zI1MzebGZ3mdlWM3vUzFbHO7eKiORGY5hIRz5I1IDyE6L95F7q7o/mW1L5hdwn7XHgk0TfGY9nOvB+ou/DX0TU4tvs8AQiIlnSGCaSgLv/X3c/1t33cffZ7v5yd78r77qqINinP3e/CcDMjgUOHGe5K+t+fMzMvs7YbckiIpnQGCYiRVOEKfqXEnW8NWVmK4GVAFOn9h1z4EFpHSy6fe49mBXrEFGhahrxSYwMT8ZGRideeByTeyYxMtzZOkJSPeMrWj0AGx5Z/0d3n5d3HeMYcwxrHL8OKtD4BdmNYcMtfmkzySczaiMpVzOxEY/q7XFj2PJprvMmf4Y9NonhZjcE2WDj8XpbM2WSsXu09eco7aezZ5IxnMEYZi02XXY6fuUa0szsbUQnbn37WMu4+1XAVQCHHrbIv/UvxelGfXztB1i4rFiHiQpX0whr+l/Erdccz8Ibx+q6nthp7z+Om1bfGaCeMFTP+IpWD8AGLt+Qdw1jmWgMqx+/DjtskX/3X4v1oW792vNZsuySTLb1ra1HT7jM0kdWsO6gWzKoZmK3blzGmTuO4vrp9+RWw/pH93xv/8DMxVw6kO6fQ9/D7Z384b2HHMBlf3is7e3ssyGd9/JzTzyAq297jH3XDU68cId61068y90GvtjRLyy346SZ2Qrg08BrOjlfnKTn7Nl3cPLbfsH9qxbnXYpI4WgMa88Zs+7Ou4S2nLxgLbOmdHSYr44tOTD7M64NLmp2HO3wti1ONnPXqi1L+9iydLxjLnduaNmYe0UEk0tIM7NXA/8InOLupT3xaTc4e/Yd3Lzi89y/ajE7n3dA3uWIFILGsGTKFtQgCmt5WnLgk5mHtaoENaD0QS3kITh6zGwq0ekXJpvZ1GZt6Wb2CqKjn58en+dNSuDmFZ9n819vV1CTytIYlg0FtWQU1JIrc1ALOZP2UaJzrl0AvCn+/0fNbJGZDZjZoni5C4lOhfHP8fUDZvaDgHVIStYsv5bNf72dx08/OO9SRNKgMSwjCmrJ9PVmu0+jglrr0gpqwUKau1/k7tZwucjdH3b3me7+cLzcy929J76udnlNqDokXWuWX8vJb/uFgppUjsawbJ0x6+7ShbUiBDXNqCWXRVALHdZ0gnVpW62hQEFNRDqloNa+PIJaFmFt22LLpKEgbSGDmkKaJKLOTxEJpYxBLe+wps7P5MrU+amQJomp81NEQilbUIP8Z9XU+dmZMgQ1hTTpmDo/RSSE2ZN35F1C2/IOaqD91DqRxdefnVBIkyDU+SkiIWhGLRkFteSKHNQU0iQYdX6KSAgKaskoqCVX1KCmkCZBqfNTRELQITqSyTqojfaOqvMzRQppEpw6P0UklDIGtbzDmjo/k8ui87MdCmmSilrn5+CCXjUUiEhHyhbUIP9ZNXV+dqYoQU0hTVK1dL8n1PkpIh1TUEtGQS25IgQ1hTRJnTo/RSQEBbVkFNSSyzuoKaRJJtT5KSIhKKglo6CWXJ5BTSFNMqPOTxEJQZ2fyeicn8nlFdQU0iRT6vwUkVDKGNTyDmvq/Ewuj87Pnky3ViGjwyP09vwDo8MjTOqZnHc5pXL27Ds4e8UdnMr7WfTPw0y797G8S6q8n5x1NUPTJz7lzg/7gXdMvL7eHdN5xdfP7bwwyc3I8AhTJl/GyPAIk0s8hp0x626+tfXovMtoy8kL1nLrxmW5bb8W1NY/Oi+zbQ4uGqLv4d7E9183+nFG2Dbhcu9ZDxw08fp6RvZh+eMXJaply9I+9l03mOi+7dJMWkI7+7cyyR5kZ//WvEspLZ3zMzutBLQ81yfZG+gfwOxBBvoH8i6lY2WbUYPu/fozqVYCWjuGJ3e2vqxm1BTSEhgdHmFoYDtmztDAdkaHR/IuqbTU+SmSvZHhEXZs24GZs2PbDkYqMIYpqCVTpqBWNFkENYW0BHb2bwWPf3A0m9YhdX6KZGugf2CPMawKs2mgoJaUglpyaQc1hbQ21WbR6mk2rXPq/BTJRm0WrV5VZtNAnZ9JVbXzMwtpBjWFtDbtMYtWo9m0INT5KZK+PWbRaio0m1ZTxqCWd1ircudn2tLq/FRIa0OzWbQazaaFUTvn5/2rFquhQCSwZrNoNVWaTaspW1CD/GfVqnzOzyyEDmpBQ5qZnWdmvzKzQTO7boJlP2BmG81sq5ldY2b5nyRrAk1n0Wo0mxaUOj8la1Ufv2CMWbSaCs6mgYJaUgpqyYUMaqFn0h4HPglcM95CZvYq4ALglcBi4DnAJwLXEtR4s2g1mk0LS52fkrHKjl8w/ixaTRVn00BBLSkFteRCBbWgIc3db3L3W4CnJlj0zcDV7n6fu/cDfwO8JWQtoY07i1aj2bTg1PkpWany+AUTzKLVVHQ2DRTUklJQSy5EUMvrjANHAt+u+/nXwHwz29/d9xggzWwlsBJg3ry5PL72Y9lV+YwtTJ1yMdbCGScGtw6x5al3AbNSr6qZ3bvm8/jaVblsu5kQ9bwSOOrVM9h67EuY0t/ZUZ5nL5jBaauO62gdIWVVzw/7w68zq+fxh+/LZDPtSDx+rV97YXZV7mELvT1/09IYtn3Lbvqfeg95jGGDuxawfu0Fqa3/2Pjf/pHpLS3fN7QfSx9ZkVo9rVgKbN09FYA5o9M5c8dR2RcxBwaHmseF+ZP7+NDMwM1ez43+mTT0p3mk8x4IuwmAt77ywHg7E316Se6uqzu7f14hbSawpe7n2v/3oeFTrLtfBVwFcOhhi3zhstWZFFhv+5P9DG3b3dKyZrvZd/9PM2Pe7JSrau7xtavI4zkaS6h6FgJr+l/MTf/7hRy+ekPi9Zy26jhuWn1nx/WEklk9LZzqqV1Feh4zlmj8OuywRb5k2SWZFNhoy5Nb2LG19TFs9v6fZt95+6Zc1d7Wr72ALJ6jJdDSqaSWPrKCdQfdkno9LXvwDVw//Z58tj29+WmkPjRzMZcOJB+TJ9LJqaQmctkf/nRKwn02pBfUOpFXd+cAe35Mq/0/7HkfAmhlX7RG2jctHer8lIIozfgFre2L1qiq+6bVK+PXn7Om7Mp1+1Xu/Ez75OxJ5RXS7gOW1/28HNjU+FVBEbS0L1oj7ZuWKnV+Ss5KM35Bi/uiNarwvmn1yhjUtJ9aeooY1EIfgqPHzKYCk4HJZjbVzJp9pfpV4Fwze66Z7Qd8FLguZC0hJJlFq9FsWrrU+SmhVW38gmSzaDXdMJsGCmpJ5XHg2ywULaiFnkn7KLCTqD39TfH/P2pmi8xswMwWAbj7D4HVwE+Bh4ENwMcD19KxRLNoNZpNS506PyWwSo1fkHAWraZLZtNAQS0pBbX0BW0ccPeLgIvGuHlmw7KfAz4XcvshdTKLVjM0sJ1ps2cxqWdyoKqk0dmz74C3wa0cz8IbH8q7HCmxKo1f0NksWs2ObTuYOXsmk7tgDKsFtVYaCori5AVruXXjslxr6OsdznX7aakFtbwbCnRaqDF0NItWo9m0TOicnyJ762gWraaLZtNqyjar1q3n/MxK3rNqCmlNhJhFq9G+adlQ56fIn4SYRavpln3T6pUtqEH+X3/m0fmZlTyDmkJaE0Fm0Wo0m5YpdX6KBJpFq+nC2TRQUEtKQS0shbQmRgbDtvuGXp+MT52fe+vd0doR1vNan4Q1tCvsmBN6fWUxe3KY2cgsVTWoTZo0c+KF2jCZfdq+Tx5BLa8zDhTarAPnt7Rc0Y7uL3+yZvm1rFn0YjUUxF7x9XNbWq5oZ2SQZOYdtPeR4ZvJ6gj/ZXbGrLtL1UwAxWgoWHLgk03PUJDUQYs+OuEyfQ/38t5DDtjjTAKhbVtsmTYTaCZNKqvWUKAZNRHpxBmz7i7d159VnVEbz+CiIUZ7R1PfzrbFltmsmkKaVJo6P0UklDIGtbzDWh77qFXpDAUKaVJ59Z2fPm1K3uWISImVLahB/rNqOudncgpp0jVuXvF5hp81qs5PEemIgloyCmrtU0iTrnLwtKfU+SkiHVNQS0ZBrT0KadJ1dM5PEQlBQS0ZBbXWKaRJV1Lnp4iEoM7PZPIIalmEtdCdnwpp0rXU+SkioZQxqOUd1tT5OTGFNOlqOueniIRStqAG+c+qqfNzfAppIuicnyIShoJaMlUNap1SSBOJ6ZyfkqZhDbddQ0EtGQW1vWnUEKmjzk9JU9nOASnJKaglo6C2J4U0kQbq/JQ0Kah1D3V+JlPVzs8kFNJEmlDnp6RJQa27lDGo5R3Wqtz52Q6FNJExqPNT0qSg1l3KFtQg/1m1Knd+tkohTWQC6vyUtCiodRcFtWS6OagFDWlmNsfMbjaz7Wa2wczOHGO5PjP7kpltMrPNZvZdM9M7oBSWOj+rL6/x61tbj1ZY6yIKasl0a1ALPZN2BTAEzAfOAq40syObLPc+4M+A5wMLgX7g8sC1iASlzs/Ky3X8UlDrHgpqyXRjUAsW0sxsBnA6cKG7D7j77cB3gLObLH4w8CN33+Tuu4BvAM0GQ5FCUednNRVl/FJQ6x7q/Eym2zo/zd3DrMjsBcDP3X163XXnAy9z91Malj0W+AfgvwBPA18BnnD39zdZ70pgJcC8eXOP+aevfSxIvSHs3jWfKVM35V3GHopWU1XreWpkBk9vm0Hfxs7+eGcvmEH/xu0d1xNK0eoBWPm+c+5y92PT3EYW49fceXOPufyrn26pntmTdyR8JO0Z3LWAvqkbM9lWK7q1nv6R6RMvBPQN7cdg79MpV9OarbunAjBndDqbJ2Xzeq03ONTT9Pr5k/vYNDKYyjYnDbU/r3XemX/V0fjV/FEmMxPY2nDdFmCfJsv+HngEeAwYAX4LnNdspe5+FXAVwKGHLfKFy1aHqrdjj69dRZHqgeLVVNV6Fsb/nnrL+1n0z8NMu/exROs5bdVx3LT6zo7rCaVo9WQo9fHrOYct8XUH3dJyQVnMsqxfewFLll2S+nZa1a31LKG1WdSlj6ygnddQ2m7duIwzdxzF9dPvyX7jca5d/+i8Pa7+0MzFXDqwIbXN9j3cm9q6mwm5T9oAMKvhulnAtibLXgH0AfsDM4CbgB8ErEUkE+r8rIzCjV/66rO7lO2rT+jerz+zFDKkPQD0mNmhddctB+5rsuxRwHXuvtndB4l2un2hmc0NWI9IJtT5WQmFHL/U+dldyhjUZk3ZlXcJlQ5qwUKau28n+kR5sZnNMLMTgNcDa5osfidwjpnta2ZTgPcAj7v7H0PVI5IldX6WW9HHLwW17lHGoKYZtfSEPgTHe4BpwBPADcC73f0+MzvRzAbqljsf2EW0b8eTwGuBUwPXIpIpdX6WXqHHLwW17qHOz2Sq2PkZsnEAd98MrGhy/W1EO+bWfn6K6DhEIpVy9uw74G1w09IXcvjq9HZelfDKMH59a+vRpXvzluTOmHV3qcJ5LajdunFZbjX09Q5nvs3BRUOpNRTotFAigemcn5KmMr1pS+fKGMrznlWr0jk/FdJEUqLOT0mLglp3UVBLpgpBTSFNJEXq/JS0qPOzuyioJVP2oKaQJpIydX5KmhTUukdWZ6IISUGtMwppIhlQ56ekSUGte6jzM5mydn4qpIlkpBbU7l+1OO9SpIIU1LpLGYNa3mEt66AWgkKaSIbU+SlpUlDrLmULapD/rFoenZ+dUEgTyUGt89OnTcm7FKkYBbXuoqCWTFmCmkKaSE7WLL+W4WeNaj81CU6dn91FQS2ZMgQ1hTSRHB087Sk1FEhqFNS6h4JaMkUPaqUKaYMj+mpIqkedn5ImBbXuoc7PZIoc1EoV0ibtHuXUW97Pmv4X512KSFDq/JQ0Kah1lzIGtbzDWlGDWqlCGsDhqzdw6zXHK6hJ5ajzU9KkoNZdyhbUIP9ZtSJ2fpYupAEsvPEhbr3meM7+9VvzLkUkOJ3zU9KioNZdFNSSKVJQK2VIgyiozblihoKaVJLO+SlpUednd1FQS6YoQa20IQ1g2r2PMeeKGZx6y/vzLkUkOJ3zU9KkoNY9FNSSKUJQK3VIgyioHb56g4KaVJI6PyVN/SPT8y5BMqLOz2TyDmqlD2k1taCmhgKpGnV+Spo0o9ZdyhjU8g5reQa1yoQ0UOenVJc6PyVNCmrdpWxBDfKfVcur87NSIQ3U+SnVps5PSYuCWndRUEsm66BWuZAG6vyUalPnp6RFnZ/dRUEtmSyDWtCQZmZzzOxmM9tuZhvM7Mxxlj3azP7NzAbMbJOZvS9kLer8lCpT52d4RRq/8qag1j0U1JLJKqiFnkm7AhgC5gNnAVea2ZGNC5nZXOCHwJeB/YFDgB8HrkWdn1Jp6vwMrlDjV94U1LqHOj+TySKoBQtpZjYDOB240N0H3P124DvA2U0W/yDwI3f/ursPuvs2d/9dqFoaqfNTqkqdn2FkMX6NePn2LlFQ6y5lDGp5h7W0g1rIUeMwYNjdH6i77tfAXp9EgeOBzWZ2h5k9YWbfNbNFAWvZizo/parU+RlEJuPXrRuXBSg1Wwpq3aVsQQ3yn1VLs/PT3D3MisxOBL7p7gvqrnsHcJa7n9Sw7APAs4CTgd8Cq4Fj3P2EJutdCawEmDt37jGfuvBzHdW5e3YfzBrm4GlPdbQegN275jNl6qaO1xNS0WpSPeMLXc9DO/en54lJ2M7die4/e8EM+jduD1ZPCCvfd85d7n5smtvIZPyaN/eYv73675+5bdaUXeEfSJv6hvZjsPfplpadPXlHytXA4K4F9E3dmPp2WtXN9bR6oON2XkNp27p7KnNGp7N5Uvqv1bEMDvXs8fN7z3hjR+NXz8SLtGwAmNVw3SxgW5NldwI3u/udAGb2CeCPZravu2+pX9DdrwKuAliy6GC/afWdHRe683kHsPmvt7Nm+bUdrefxtatYuGx1x/WEVLSaVM/4QtezEDj712/F/mU2C298qO37n7bqOEL8jZVQ6uPX4kOf49dPv2ePFeU9A7D0kRWsO+iWtu6T5kzL+rUXsGTZJamtv13dXM+XhkxWAAAb4ElEQVQSWptFTfIaStWDb6Dx7yxT02H9o/OCrS7k150PAD1mdmjddcuB+5os+xugfgovzHRei9T5KVWmzs9Echm/9PWnFFkZv/oswgx1yK8+g4U0d98O3ARcbGYzzOwE4PXAmiaLXwucamZHmdkU4ELg9sZPoWlS56dUmTo/25Pn+KWgJkWmzs9kQgW10O1G7wGmAU8ANwDvdvf7zOxEMxuoLeTuPwE+Anw/XvYQYMxjEqVJnZ9SVer8bFtu45eCmhRdGYNa3mEtRFALGtLcfbO7r3D3Ge6+yN2vj6+/zd1nNix7pbsf4O6z3f0Ud38kZC3tUOenVJU6P1uX9/iloCZFV7agBsWYVetE+Q7ckxKd81OqTOf8LIdbNy4rXVhTUOsuCmrZUkiro3N+SpXVzvmpoFZ8ZQxqCmvdQ0EtOwppDdT5KVW2Zvm1HPuZu9VQUAJlC2qgWbVuoqCWDYW0JtT5KVWmzs/yUFCTIjtj1t2ZHOQ4pLIFNYW0cajzU6pKnZ/loaAmRVe2WbUidH62SiFtAur8lKpS52d5KKhJ0ZUtqEE5ZtUU0lqgzk+pMnV+loM6P6XoFNTCU0hrkTo/pcpqnZ+7Z/flXYpMoIxBTWGteyiohaWQ1gZ1fkqVrVl+LbPmDqihoATKFtRAs2rdREEtHIW0NqnzU6ps/8nb1flZEgpqUmQ652cYCmkJHb56A+uefpYaCqRy1PlZHgpqUnRlDGpFCmsKaR3o2zikzk+pJHV+loeCmhRd2YIaFGdWTSGtQ+r8lCpT52c5qPNTik5BLRmFtADU+SlVpnN+lkcZg5rCWvdQUGufQlog6vyUKtM5P8ujbEENNKvWTRTU2qOQFpA6P6XKdM7P8lBQkyJT52frFNJSoHN+SlWp87M8yhjU+kem512CZKiMQS3rsKaQlhKd81OqSp2f5VHGoKYZte5StqAG2c6qKaSlSJ2fUmXq/CwHdX5K0SmojU0hLWXq/JQqU+dneZQxqCmsdQ8FteYU0jKgzk+pMnV+tsZH866gfEENNKvWTRTU9hY0pJnZHDO72cy2m9kGMztzguV7zex3ZvZoyDqKSJ2fUmVV6PzMYvxa/+i8zgvtkIKaFJk6P/cUeibtCmAImA+cBVxpZkeOs/yHgScD11Bo6vyUqqpA52cm45eCWjIKat2ljEEtjbAWLKSZ2QzgdOBCdx9w99uB7wBnj7H8wcCbgE+HqqEs1PkpVVXWzs+sx6/1j87LPawpqEnRlS2oQfhZtZAzaYcBw+7+QN11vwbG+iR6OfARYGfAGkpDnZ9SZSXs/Mxl/Mo7qG3dPbV0YU1Brbt0e1Azdw+zIrMTgW+6+4K6694BnOXuJzUseyqw0t1fY2YnAV9z9wPHWO9KYCXA3Llzj/nUhZ8LUm8IsxfMoH/j9o7W4dOmMPysUQ6e9lSQmnbvms+UqZuCrCsE1TO+qtfz0M796XliErZzd+J1rHzfOXe5+7HBimoik/Fr3txjPnbl5U2339c7HOJhtG3O6HQ2T9oBwKwpu3KpoV7f0H4M9j7d8vKzJ+9IsRoY3LWAvqkbU91GO4pWD2RXU6sHOm73NZSmrbun8s6/PLuj8asnYD0DwKyG62YB2+qviL9WWA28tpWVuvtVwFUASxYd7DetvrPzSgM5bdVxhKhn5/MO4OHX9nDzis93vK7H165i4bLVHa8nFNUzvqrXsxBY0/9ibr3meBbe+FCw9aYg9fFr0dLn+KUDG8ZcdsmB2e+ee+aOo7h++j3P/Jz3yaSXPrKCdQfd0tZ90pxpWb/2ApYsuyS19beraPVAdjUtobVZ1CSvoSIL+XXnA0CPmR1ad91y4L6G5Q4ler5vM7ONwE3As81so5ktCVhPaajzU6qsJJ2fuY9feX/1CdpPTYqtjJ2fnQoW0tx9O9GAdbGZzTCzE4DXA2saFr0XOAg4Kr68HdgU//+RUPWUkTo/paqK3vlZlPFLQS0ZBbXu0k1BLfQhON4DTAOeAG4A3u3u95nZiWY2AODuw+6+sXYBNgOj8c8jgespHXV+SlWVoPOzEOOXOj+TUVDrLt0S1IKGNHff7O4r3H2Guy9y9+vj629z95lj3OdnY+10263U+SlVVtTOz6KNX0UIamULawpq3aUbgppOC1VQOuenVJnO+dmavIMalG9WTef87C5VD2oKaQWmc35Klemcn61RUEtGQa17VDmoKaQVnDo/pcpK0vmZOwW1ZBTUukdVOz8V0kpCnZ9SVUXv/CwKBbVkFNS6S9oHOM6aQlqJqPNTqqoEnZ+FoM7PZBTUukuVZtQU0kpGnZ9SZUXt/CyaIgS1soU1BbXuUpWgppBWQur8lCpT52dr8g5qUL5ZNXV+dpcqBDWFtJJS56dUmTo/W6OgloyCWvcoe1BTSCsxdX5KldUaCmR8CmrJKKh1jzJ3fiqkVYA6P6Wqzp59R94llIKCWjIKat2ljEFNIa0iap2fT43MyLsUEcmBOj+TUVDrLmULagppFbLwxofY+seZaigQ6WJFCGplC2sKat2lTEFNIa1ipvQPqvNTpMvlHdSgfLNq6vzsLmUJagppFaTOT5ECcst0cwpqySiodY8yBDWFtIpS56dI8fQ93Jvp9hTUklFQ6x5F7/xUSKs4dX6KFIuCWjkoqHWXogY1hbQuoHN+ihRLHkEt77BWxqDWPzI97xIkQ0UMagppXULn/BQplqyDGuQ/q6bOTym6ogU1hbQuonN+ihRL38O9+vqzBNT52V2KFNQU0rqMOj9FikdBrRwU1LpHUYKaQloXUuenSPEoqJWDglr3KELnp0JaF1Pnp0ixZB3UBod6Mt1eMwpqUnR5BrWgIc3M5pjZzWa23cw2mNmZYyz3YTO718y2mdlDZvbhkHVI69T5KRIpyvilzs9yUFDrLnkFtdAzaVcAQ8B84CzgSjM7sslyBpwDzAZeDZxnZm8IXIu0SJ2fIkCBxq9u7fzcuntqrjW0S0Gtu+QR1IKFNDObAZwOXOjuA+5+O/Ad4OzGZd19tbvf7e7D7n4/8G3ghFC1SPvU+SndrIjjlzo/y0Gdn90l66Bm7h5mRWYvAH7u7tPrrjsfeJm7nzLO/Qy4G/iyu3+pye0rgZUAc+fOPeZTF34uSL0hzF4wg/6N2/MuYw+d1uTTpjC0r7F0vyeC1LN713ymTN0UZF0hqJ7xFa0egNe+6r13ufuxaW4jm/Fr3jEXXfaFRPWN9o4mut9E5k/uY9PI4F7X9/UOp7K9icwZnc7mSTsAmDVlVy411Osb2o/B3qdbXn725B0pVgODuxbQN3VjqttoV9FqyqqeVg90/MbXvKOj8SvkXqMzga0N120B9pngfhcRzehd2+xGd78KuApgyaKD/abVd3ZWZUCnrTqOItUD4Wq6f9Vibl7x+Y7X8/jaVSxctrrj9YSiesZXtHoylPr4teg5S/2yPzyWuMDBRUOJ7zuWD81czKUDG5retuTAJ4NvbyJn7jiK66ff88zPJy9Ym3kN9ZY+soJ1B93S1n3SnGlZv/YCliy7JLX1J1G0mrKqZ0n8b9qzqCH3SRsAZjVcNwvYNtYdzOw8on07Xufue3+ck9yo81O6TOHHL331WQ766rO7pP31Z8iQ9gDQY2aH1l23HLiv2cJm9jbgAuCV7v5owDokEHV+Shcpxfilzs9yUFDrLmkGtWAhzd23AzcBF5vZDDM7AXg9sKZxWTM7C/gUcLK7PxiqBglPnZ/SDco0fnVr52fZwpqCWndJK6iFPgTHe4BpwBPADcC73f0+MzvRzAbqlvsksD9wp5kNxJe9drqVYlDnp3SJ0oxf6vwsB3V+dpc0glrQkObum919hbvPcPdF7n59fP1t7j6zbrmD3X2Ku8+su7wrZC0Sls75KVVXxvFLQa0cFNS6R+igptNCSct0zk+R4lFQKwcFte4R8pyfCmnSNnV+ihSLglo5KKh1lxBBTSFNElHnp0ixqPOzHBTUpB0KaZKYOj9FikWdn+WgoCatUkiTjqjzU6Q15rDPhjCn4RuPOj/LQUFNWqGQJh1T56dI67IIaqD91MpAh+iQiSikSRDq/BRpnYJaesoW1ECzajI2hTQJSp2fIq1RUEuPgppUhUKaBKfOT5HWVDmo5R3WFNSkChTSJBW1zs+Hdu6fdykihVbVoAb5z6qp81PKTiFNUrPwxofoeWKSOj9FJrDPBlfnZ4oU1KSsFNIkVbZztzo/RVpU1Vk1BbX2qfNTQCFNMqDOT5HWKailp2xBDTSr1u0U0iQz6vwUaU1WQW3SULZvAQpqySioda+evAsoop+cdTVD03dMuNwP+4F3TLy+3h3TecXXz+28sAo4fPUGbl13PLwNzp59R97lSJ3R4RF6e/6B0eERJvVMzrucrrfPBmfbYkt033WjH2eEbRMud94Dra1v0qSZHLToo4lqaVQLaksOfDLI+pK4deMyTl6wNrftJ/GtrUdzbN5FFNzI8AhTJl/GyPAIkysyhmkmrYlWAlqe6ys7nfOzmHb2b2WSPcjO/q15lyKxpDNqrQS0doyODgRdH+Q/q1bGzs/+kel5l1BoA/0DmD3IQH/412teFNIkFzrnZ7GMDo8wNLAdM2doYDujwyN5lySxrDo/85B3UIPyff2prz6bGxkeYce2HZg5O7btYKQiY5hCmuRG5/wsjp39W6GWAxzNphWQglp6yhjUFNb2NNA/sMcYVpXZNIU0yZU6P/NXm0Wrp9m0YlJQS0/ZghpoVq2mNotWryqzaQppUgjq/MzPHrNoNZpNKywFtfQoqJXTHrNoNRWZTVNIk8LQOT+z12wWrUazacVV1aA2ONSTe1hTUCuXZrNoNVWYTQsa0sxsjpndbGbbzWyDmZ05xnJmZp8xs6fiy2fMLFmvuVSKOj+z1XQWrabLZtPKNn5VNahB/rNqZez87Nag1nQWraYCs2mhZ9KuAIaA+cBZwJVmdmST5VYCK4DlwPOBU4B3Bq5FSkqdn9kYbxatpstm00o3fmXZ+dmNZyjYuntq3iW0pduC2nizaDVln00LFtLMbAZwOnChuw+4++3Ad4Czmyz+ZuBSd3/U3R8DLgXeEqoWKT91fqZv3Fm0mi6ZTSv7+KWglp4yzqh1S1gbdxatpuSzaeYe5o/bzF4A/Nzdp9dddz7wMnc/pWHZLcBfuPv/iX8+Fvipu+/TZL0riT65Mnfu3GM+deHngtQ7npX95wRf51Wzvxp8nc3MXjCD/o3jz45kKUQ9gwt6WbrfE0Hq2b1rPlOmbgqyrhDyq2cLU6dcjNnuCZd0n8Ku3R8HZqVfVhOvfdV773L3VA+2ntX4dfFnv5Dio4DR3uhb1/Me+Kvg6/7CYd+o285o8PUDzJ/cx6aRwb2u7+sdTmV7E5kzOp3Nk6KZmllTduVSQ72+of0Y7H265eVnT07/QOqDuxbQN3Vj6tvZ2xZ6e/6m5TFsaPhj5DGGvfov3tfR+BXytFAzgcaP3FuAvQaueNktDcvNNDPzhtTo7lcBVwEsWXSw37T6znAVj6WFUz21K5O6gdNWHZfZtloRqp77Vy3mtJf9suNTST2+dhULl63uuJ5Q8qpn+5P9DG2beHADMNvNvvt/mhnzZqdcVa5SH78WL36OX33bY+EqHkPSU0lN5LI/7Fn74KKh4Nv40MzFXDqwoelteZxG6swdR3H99Hue+TnvU0ktfWQF6w66pa37nDHr7pSqiaxfewFLll2S6jaa2fLkFnZsbX0Mm73/p9l33r4pVxVeyH3SBtg7ps6CpucnaVx2FjDQOMCJ1KjzM5xW9kVr1AX7pmUyfu27bu9ZotCq/NVn3l9/lu2rT6jmfmqt7IvWqKz7poUMaQ8APWZ2aN11y4H7mix7X3zbRMuJPEOdn2G0tC9ao+rvm5bZ+LXvusFMwloWsg5qkP9+aur8zF9L+6I1Kum+acFCmrtvB24CLjazGWZ2AvB6YE2Txb8KfNDMDjCzhcCHgOtC1SLVpc7PziSZRaup8mxaHuNXlYKaGgqKrypBLcksWk0ZZ9NCH4LjPcA04AngBuDd7n6fmZ1oZvUR9svAd4HfAvcC34+vE5mQOj+TSzSLVlP92bTMx6+qBDVQ52cZVKHzM9EsWk0JZ9OChjR33+zuK9x9hrsvcvfr4+tvc/eZdcu5u69y9znxZZX2R5N26Jyf7etkFq2m4rNpuYxfCmrJKaglU9ag1sksWk3ZZtN0WigpNZ3zs3UdzaLVVH82LRcKaskpqCVTxqDW0SxaTclm0xTSpPTU+TmxELNoNVWeTcuTglpy6vxMpkxBLcQsWk2ZZtMU0qQS1Pk5viCzaDWaTUuNOj87U4SgVrawVpagFmQWraZEs2kKaU307pg+8UI5rk+aU+fn2EYGwx54NPT6ZE+dBrWekWbH4E1uctNj+k5MnZ/lUIagNrQr7JgTen1pCXnGgcp4xdfPbWm5oh3dX2qdnwdw6mvfz80rPp93OYUx68D5LS1XtDMydLN91w2yZWlfovsuf/yilpY798QD+PzDjyfaRjv6Hu5N5QwFY1n/6LxczlBQ79aNy3I/Q0E7akEt7TMUJDXvoNbCd15nQEiLZtKkctT5KVWhMxQkpxm1ZMowq9ZNFNKkstT5KVWgoJacgloyCmrFoZAmlabOT6kCBbXk1PmZjIJaMSikSeWp81OqIIvOz6oGNch/Vk2dn5KEQpp0hVrn50M798+7FJGOZBHUsghr6vwsBwW1fCmkSdeYdu9j9DwxSQ0FUnr6+jM5BbX2VeGcn2WlkCZdxXbuVuenVIKCWnIKaskoqGVPIU26kjo/pQoU1JJTUEtGQS1bCmnStdT5KVVQpaA2aSjbtyR1fiajoJYdhTTpaur8lCpQ52dnihDUyhbWFNSyoZAmXU/n/JQsmKcfctT5mVzeQQ3KN6umoJY+hTQRauf8nKGGAklV79pH6V37aKrbqNLXnwpqxdc/Ml1hLUUKaSIxnfNTsqKg1joFtXJQUEuHQppIA3V+ShYU1FqnoFYOCmrhKaSJNKHOT8mCglrr8ghqg0M9mW6zkYKaKKSJjEGdn5KFLIKaOj+Ty3tWTZ2f3S1ISDOzOWZ2s5ltN7MNZnbmOMt+2MzuNbNtZvaQmX04RA0iaVDnZ3fIewxLO6iBOj87kXdQg/LNqimohRFqJu0KYAiYD5wFXGlmR46xrAHnALOBVwPnmdkbAtUhEpw6P7tC7mOYOj/b041BbevuqXmX0Bad87NzHYc0M5sBnA5c6O4D7n478B3g7GbLu/tqd7/b3Yfd/X7g28AJndYhkiZ1flZX0cYwBbXWdWNQK9uMGmhWrRPmHR5g0cxeAPzc3afXXXc+8DJ3P2WC+xpwN/Bld//SGMusBFbGPz4PuLejgsOaC/wx7yIaFK0m1TM+1TOxw919n7RWnuYYVvDxC4r3+1Y94ytaPVC8mopWT0fjV4jWlZnA1obrtgCtFHUR0WzetWMt4O5XAVcBmNmv3P3YZGWGV7R6oHg1qZ7xqZ6JmdmvUt5EamNYkccvKF5Nqmd8RasHildTEevp5P4Tft1pZj8zMx/jcjswAMxquNssYNsE6z2PaL+O17l7+vPvItKVNIaJSFlNOJPm7ieNd3u8P0ePmR3q7r+Pr14O3DfOfd4GXAC81N3Tb2sSka6lMUxEyqrjxgF33w7cBFxsZjPM7ATg9cCaZsub2VnAp4CT3f3BNjd3VUfFhle0eqB4Name8ameiaVaU4ZjWNc9twmonvEVrR4oXk2VqqfjxgGIjjEEXAOcDDwFXODu18e3nQj8wN1nxj8/BBwI1H898DV3f1fHhYiIJKAxTESKKEhIExEREZGwdFooERERkQJSSBMREREpoEKHtLzPp9dODRb5jJk9FV8+Ex/oMqg26snkHKnt/I7i5XvN7HdmlkpHXJuvmaPN7N/MbMDMNpnZ+/Kqx8z6zOxLcR2bzey7ZnZACvWcZ2a/MrNBM7tugmU/YGYbzWyrmV1jZn2h62mnJjN7s5ndFdfzqJmtNrMQx3pMhcavjurJ7BzPGsPC1NOtY1ja41ehQxoFOJ9eGzWsBFYQte4/HzgFeGeA7SetJ6tzpLbzOwL4MPBkCnW0VY+ZzQV+CHwZ2B84BPhxXvUA7wP+jOi1sxDoBy5PoZ7HgU8S7SQ/JjN7FdEhJl4JLAaeA3wihXpargmYDryf6IjiL4prOz+lmkLQ+JW8nizP8awxLEA9dO8Ylu745e6FvAAziF4Yh9Vdtwa4pMX7XwZcnlUNwB3AyrqfzwV+UZTnJMTz0Wk9wMHA74DXAI/m+ZohOoTCmtA1dFDPlcDqup9fB9yfYm2fBK4b5/brgU/V/fxKYGPKz9e4NTVZ/oPAd9OsKYvf/Rj31/gV+PkIUZPGMI1hSetpsnxL41eRZ9IOA4bd/YG6634NjPcJB3jmfHonMs7BKFOo4cj4tomWy6qeZwR8Pjqt53LgI8DOwHUkqed4YLOZ3WFmT8RT84tyrOdq4AQzW2hm04k+sf4gcD3taPZ6nm9m++dUTzMvJfxrOhSNX53V84wUx68kNWkM0xgWSkvjV5FDWqrnBE2hhpnxbfXLzQy8X0fS5+Qiwjwfiesxs1OBye5+c+AaEtVDdJyrNxNN0S8CHgJuyLGe3wOPAI/F9zkCuDhwPe1o9nqG1v7+UmfREf+PBT6bdy1j0PjVWT31LiKd8autmjSGTViPxrAWtTN+5RbSrBzn02unhsZlZwEDHs9rBtL2cxL4+UhUj0Wn3VkNvDfw9hPVE9sJ3Ozud7r7LqJ9FV5sZvvmVM8VQB/RviUziI6An+en0GavZ5jg7y8LZrYC+DTwGnf/Y041aPxKtx4gk/OjagwLV4/GsBa0O37lFtLc/SR3tzEuLwEeID6fXt3dWj2f3is9zPn02qnhvvi2lmrNoJ40no+k9RwKLAFuM7ONRH+8z467bpbkUA/Ab4D6N6A0jurcTj1HEe3PsDl+M7oceGG8c3Aemr2eN7n7UznVA4CZvRr4R+AUd/9tXnVo/Eq9nizGr3Zq0hg2cT0awyaQaPxKaye6QDvi/U+i6dsZwAlE05VHjrHsWcBG4Ig8agDeRbRD6QFEnS33Ae/K6zlJ6/lIUg/QAyyou5xG1BGzgOjrgzyen1cQdR8dBUwB/h64Lcff17XAjcC+cT0fAR5LoZ4eYCrRJ7k18f97miz36vj181xgP+AntLjTe4o1vYLolE0vTfM1nfXvPl5W41cGz0fSmjSGaQwLUE+i8SvVF3+ABz8HuAXYDjwMnFl324lE0/G1nx8CdhNNcdYuX0qrhibbN6Lp8M3xZTXxabeyeE6yej6S1tNwn5NIoTOq3XqAdxPtP9EPfBc4KMff1/7A14EngKeB24EXplDPRUSfuOsvFxHt0zIALKpb9oPAJqL9S64F+lL6nbVUE/BTYLjhNf2DNGpK83c/xu9f41cO41c7NTXc5yQ0hmkMa6MeEo5fOneniIiISAEVubtTREREpGsppImIiIgUkEKaiIiISAEppImIiIgUkEKaiIiISAEppImIiIgUkEKaiIiISAEppImIiIgU0P8Hut8DXs9ag3AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x1s = np.linspace(-0.2, 1.2, 100)\n",
    "x2s = np.linspace(-0.2, 1.2, 100)\n",
    "x1, x2 = np.meshgrid(x1s, x2s)\n",
    "\n",
    "z1 = mlp_xor(x1, x2, activation=heaviside)\n",
    "z2 = mlp_xor(x1, x2, activation=sigmoid)\n",
    "\n",
    "plt.figure(figsize=(10,4))\n",
    "\n",
    "plt.subplot(121)\n",
    "plt.contourf(x1, x2, z1)\n",
    "plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
    "plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
    "plt.title(\"Activation function: heaviside\", fontsize=14)\n",
    "plt.grid(True)\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.contourf(x1, x2, z2)\n",
    "plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
    "plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
    "plt.title(\"Activation function: sigmoid\", fontsize=14)\n",
    "plt.grid(True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FNN for MNIST"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using the Estimator API (formerly `tf.contrib.learn`)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Warning**: `tf.examples.tutorials.mnist` is deprecated. We will use `tf.keras.datasets.mnist` instead. Moreover, the `tf.contrib.learn` API was promoted to `tf.estimators` and `tf.feature_columns`, and it has changed considerably. In particular, there is no `infer_real_valued_columns_from_input()` function or `SKCompat` class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()\n",
    "X_train = X_train.astype(np.float32).reshape(-1, 28*28) / 255.0\n",
    "X_test = X_test.astype(np.float32).reshape(-1, 28*28) / 255.0\n",
    "y_train = y_train.astype(np.int32)\n",
    "y_test = y_test.astype(np.int32)\n",
    "X_valid, X_train = X_train[:5000], X_train[5000:]\n",
    "y_valid, y_train = y_train[:5000], y_train[5000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpuflzeb_h\n",
      "INFO:tensorflow:Using config: {'_evaluation_master': '', '_session_config': None, '_model_dir': '/tmp/tmpuflzeb_h', '_task_type': 'worker', '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f4fcb4e15c0>, '_save_summary_steps': 100, '_is_chief': True, '_save_checkpoints_steps': None, '_log_step_count_steps': 100, '_master': '', '_service': None, '_keep_checkpoint_every_n_hours': 10000, '_task_id': 0, '_tf_random_seed': None, '_num_ps_replicas': 0, '_global_id_in_cluster': 0, '_train_distribute': None, '_num_worker_replicas': 1, '_save_checkpoints_secs': 600, '_keep_checkpoint_max': 5}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /tmp/tmpuflzeb_h/model.ckpt.\n",
      "INFO:tensorflow:loss = 122.883514, step = 0\n",
      "INFO:tensorflow:global_step/sec: 480.267\n",
      "INFO:tensorflow:loss = 9.599711, step = 100 (0.209 sec)\n",
      "INFO:tensorflow:global_step/sec: 599.191\n",
      "INFO:tensorflow:loss = 19.580772, step = 200 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 640.184\n",
      "INFO:tensorflow:loss = 2.1866307, step = 300 (0.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 716.395\n",
      "INFO:tensorflow:loss = 11.493204, step = 400 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 713.653\n",
      "INFO:tensorflow:loss = 4.0078278, step = 500 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 722.021\n",
      "INFO:tensorflow:loss = 10.612131, step = 600 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 669.446\n",
      "INFO:tensorflow:loss = 6.692636, step = 700 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 720.49\n",
      "INFO:tensorflow:loss = 4.2058306, step = 800 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 766.548\n",
      "INFO:tensorflow:loss = 9.13055, step = 900 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 773.506\n",
      "INFO:tensorflow:loss = 4.1445055, step = 1000 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 755.713\n",
      "INFO:tensorflow:loss = 8.442559, step = 1100 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 762.721\n",
      "INFO:tensorflow:loss = 1.4401194, step = 1200 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 659.992\n",
      "INFO:tensorflow:loss = 13.526959, step = 1300 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 701.683\n",
      "INFO:tensorflow:loss = 5.039109, step = 1400 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 751.167\n",
      "INFO:tensorflow:loss = 1.8074234, step = 1500 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 700.915\n",
      "INFO:tensorflow:loss = 6.4867635, step = 1600 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 733.041\n",
      "INFO:tensorflow:loss = 0.5804969, step = 1700 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 764.306\n",
      "INFO:tensorflow:loss = 1.5091155, step = 1800 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 664.096\n",
      "INFO:tensorflow:loss = 3.6764488, step = 1900 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 687.762\n",
      "INFO:tensorflow:loss = 1.4820085, step = 2000 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 667.013\n",
      "INFO:tensorflow:loss = 1.2964534, step = 2100 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 704.008\n",
      "INFO:tensorflow:loss = 1.0711427, step = 2200 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 664.444\n",
      "INFO:tensorflow:loss = 2.6691673, step = 2300 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 691.387\n",
      "INFO:tensorflow:loss = 0.9668397, step = 2400 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 749.867\n",
      "INFO:tensorflow:loss = 1.4468323, step = 2500 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 725.923\n",
      "INFO:tensorflow:loss = 1.8073778, step = 2600 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 732.432\n",
      "INFO:tensorflow:loss = 3.8904514, step = 2700 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 710.886\n",
      "INFO:tensorflow:loss = 2.3015192, step = 2800 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 713.338\n",
      "INFO:tensorflow:loss = 4.671579, step = 2900 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 708.305\n",
      "INFO:tensorflow:loss = 1.3551085, step = 3000 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 647.185\n",
      "INFO:tensorflow:loss = 1.9145899, step = 3100 (0.155 sec)\n",
      "INFO:tensorflow:global_step/sec: 673.601\n",
      "INFO:tensorflow:loss = 1.0741266, step = 3200 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 721.531\n",
      "INFO:tensorflow:loss = 1.0713774, step = 3300 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 713.421\n",
      "INFO:tensorflow:loss = 1.3074391, step = 3400 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 633.857\n",
      "INFO:tensorflow:loss = 2.0073137, step = 3500 (0.158 sec)\n",
      "INFO:tensorflow:global_step/sec: 672.287\n",
      "INFO:tensorflow:loss = 13.952677, step = 3600 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 639.094\n",
      "INFO:tensorflow:loss = 1.6767453, step = 3700 (0.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 731.891\n",
      "INFO:tensorflow:loss = 0.27798674, step = 3800 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 728.154\n",
      "INFO:tensorflow:loss = 3.5524733, step = 3900 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 714.217\n",
      "INFO:tensorflow:loss = 0.6761815, step = 4000 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 744.721\n",
      "INFO:tensorflow:loss = 0.79083383, step = 4100 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 685.419\n",
      "INFO:tensorflow:loss = 1.3305103, step = 4200 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 632.625\n",
      "INFO:tensorflow:loss = 0.14447726, step = 4300 (0.158 sec)\n",
      "INFO:tensorflow:global_step/sec: 679.755\n",
      "INFO:tensorflow:loss = 1.8386902, step = 4400 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 751.428\n",
      "INFO:tensorflow:loss = 0.94889283, step = 4500 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 750.376\n",
      "INFO:tensorflow:loss = 0.28424773, step = 4600 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 735.964\n",
      "INFO:tensorflow:loss = 3.266353, step = 4700 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 742.054\n",
      "INFO:tensorflow:loss = 3.171119, step = 4800 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 779.657\n",
      "INFO:tensorflow:loss = 1.12006, step = 4900 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 730.952\n",
      "INFO:tensorflow:loss = 0.5669488, step = 5000 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 715.897\n",
      "INFO:tensorflow:loss = 0.3067366, step = 5100 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 631.438\n",
      "INFO:tensorflow:loss = 0.5437011, step = 5200 (0.159 sec)\n",
      "INFO:tensorflow:global_step/sec: 654.717\n",
      "INFO:tensorflow:loss = 0.25085437, step = 5300 (0.153 sec)\n",
      "INFO:tensorflow:global_step/sec: 658.829\n",
      "INFO:tensorflow:loss = 0.30891788, step = 5400 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 672.908\n",
      "INFO:tensorflow:loss = 0.8258436, step = 5500 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 574.211\n",
      "INFO:tensorflow:loss = 0.19280735, step = 5600 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 653.783\n",
      "INFO:tensorflow:loss = 0.17635345, step = 5700 (0.155 sec)\n",
      "INFO:tensorflow:global_step/sec: 635.343\n",
      "INFO:tensorflow:loss = 2.1531484, step = 5800 (0.155 sec)\n",
      "INFO:tensorflow:global_step/sec: 628.505\n",
      "INFO:tensorflow:loss = 1.3385924, step = 5900 (0.159 sec)\n",
      "INFO:tensorflow:global_step/sec: 743.577\n",
      "INFO:tensorflow:loss = 1.7493693, step = 6000 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 759.487\n",
      "INFO:tensorflow:loss = 1.3990214, step = 6100 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 715.001\n",
      "INFO:tensorflow:loss = 0.08636901, step = 6200 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 682.812\n",
      "INFO:tensorflow:loss = 1.2878852, step = 6300 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 668.255\n",
      "INFO:tensorflow:loss = 2.8647041, step = 6400 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 702.609\n",
      "INFO:tensorflow:loss = 0.38349468, step = 6500 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.91\n",
      "INFO:tensorflow:loss = 0.71950877, step = 6600 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 719.603\n",
      "INFO:tensorflow:loss = 0.8812942, step = 6700 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 697.269\n",
      "INFO:tensorflow:loss = 0.5575855, step = 6800 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 733.281\n",
      "INFO:tensorflow:loss = 0.97547567, step = 6900 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 731.343\n",
      "INFO:tensorflow:loss = 0.10276175, step = 7000 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 739.462\n",
      "INFO:tensorflow:loss = 1.1644993, step = 7100 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 674.078\n",
      "INFO:tensorflow:loss = 0.87335706, step = 7200 (0.149 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 754.457\n",
      "INFO:tensorflow:loss = 0.02294194, step = 7300 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 665.539\n",
      "INFO:tensorflow:loss = 0.03988394, step = 7400 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 664.489\n",
      "INFO:tensorflow:loss = 0.35611078, step = 7500 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 642.162\n",
      "INFO:tensorflow:loss = 0.37163323, step = 7600 (0.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 710.797\n",
      "INFO:tensorflow:loss = 0.2848217, step = 7700 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 761.344\n",
      "INFO:tensorflow:loss = 0.7845428, step = 7800 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 765.77\n",
      "INFO:tensorflow:loss = 0.10574758, step = 7900 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 723.478\n",
      "INFO:tensorflow:loss = 0.69526076, step = 8000 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 750.877\n",
      "INFO:tensorflow:loss = 0.58130324, step = 8100 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 730.965\n",
      "INFO:tensorflow:loss = 0.31362545, step = 8200 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.223\n",
      "INFO:tensorflow:loss = 0.8284373, step = 8300 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 660.215\n",
      "INFO:tensorflow:loss = 0.82293755, step = 8400 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 650.197\n",
      "INFO:tensorflow:loss = 0.08349186, step = 8500 (0.154 sec)\n",
      "INFO:tensorflow:global_step/sec: 686.024\n",
      "INFO:tensorflow:loss = 0.48411164, step = 8600 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 775.917\n",
      "INFO:tensorflow:loss = 0.35096797, step = 8700 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 773.152\n",
      "INFO:tensorflow:loss = 0.22725578, step = 8800 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 691.659\n",
      "INFO:tensorflow:loss = 0.0485073, step = 8900 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 773.385\n",
      "INFO:tensorflow:loss = 0.07985984, step = 9000 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 741.822\n",
      "INFO:tensorflow:loss = 0.0542172, step = 9100 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 749.122\n",
      "INFO:tensorflow:loss = 0.50927365, step = 9200 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 681.55\n",
      "INFO:tensorflow:loss = 0.9489171, step = 9300 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 680.382\n",
      "INFO:tensorflow:loss = 0.060666546, step = 9400 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 764.193\n",
      "INFO:tensorflow:loss = 0.28517923, step = 9500 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 702.206\n",
      "INFO:tensorflow:loss = 0.20915146, step = 9600 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 678.51\n",
      "INFO:tensorflow:loss = 0.579557, step = 9700 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 593.939\n",
      "INFO:tensorflow:loss = 0.93460476, step = 9800 (0.168 sec)\n",
      "INFO:tensorflow:global_step/sec: 665.897\n",
      "INFO:tensorflow:loss = 0.047924682, step = 9900 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 711.483\n",
      "INFO:tensorflow:loss = 0.4922812, step = 10000 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 712.135\n",
      "INFO:tensorflow:loss = 0.047895633, step = 10100 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 645.55\n",
      "INFO:tensorflow:loss = 0.5398847, step = 10200 (0.155 sec)\n",
      "INFO:tensorflow:global_step/sec: 638.582\n",
      "INFO:tensorflow:loss = 0.15529627, step = 10300 (0.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 751.801\n",
      "INFO:tensorflow:loss = 0.9468999, step = 10400 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 699.759\n",
      "INFO:tensorflow:loss = 0.117633805, step = 10500 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 681.536\n",
      "INFO:tensorflow:loss = 2.7077622, step = 10600 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 639.102\n",
      "INFO:tensorflow:loss = 0.4701725, step = 10700 (0.158 sec)\n",
      "INFO:tensorflow:global_step/sec: 657.787\n",
      "INFO:tensorflow:loss = 0.036744572, step = 10800 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 610.198\n",
      "INFO:tensorflow:loss = 0.5130552, step = 10900 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 641.731\n",
      "INFO:tensorflow:loss = 0.10233489, step = 11000 (0.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 676.149\n",
      "INFO:tensorflow:loss = 0.27252752, step = 11100 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 685.492\n",
      "INFO:tensorflow:loss = 0.03905679, step = 11200 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 739.538\n",
      "INFO:tensorflow:loss = 0.4037779, step = 11300 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 761.957\n",
      "INFO:tensorflow:loss = 0.07931179, step = 11400 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 690.435\n",
      "INFO:tensorflow:loss = 0.25471577, step = 11500 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 731.688\n",
      "INFO:tensorflow:loss = 0.095315784, step = 11600 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 737.292\n",
      "INFO:tensorflow:loss = 0.26888248, step = 11700 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.634\n",
      "INFO:tensorflow:loss = 0.0791325, step = 11800 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.395\n",
      "INFO:tensorflow:loss = 0.22025564, step = 11900 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 674.538\n",
      "INFO:tensorflow:loss = 0.23279706, step = 12000 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 694.35\n",
      "INFO:tensorflow:loss = 0.12199997, step = 12100 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 666.45\n",
      "INFO:tensorflow:loss = 0.28451073, step = 12200 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 714.277\n",
      "INFO:tensorflow:loss = 0.118836306, step = 12300 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 697.121\n",
      "INFO:tensorflow:loss = 0.040629867, step = 12400 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 695.514\n",
      "INFO:tensorflow:loss = 0.12426446, step = 12500 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 748.65\n",
      "INFO:tensorflow:loss = 0.123113535, step = 12600 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 705.816\n",
      "INFO:tensorflow:loss = 0.08067431, step = 12700 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 677.035\n",
      "INFO:tensorflow:loss = 0.12166683, step = 12800 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 715.319\n",
      "INFO:tensorflow:loss = 0.44292378, step = 12900 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 663.768\n",
      "INFO:tensorflow:loss = 0.115894824, step = 13000 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 730.272\n",
      "INFO:tensorflow:loss = 0.48085117, step = 13100 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 715.079\n",
      "INFO:tensorflow:loss = 0.04950906, step = 13200 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 728.079\n",
      "INFO:tensorflow:loss = 0.22041032, step = 13300 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 743.21\n",
      "INFO:tensorflow:loss = 0.96188104, step = 13400 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 736.865\n",
      "INFO:tensorflow:loss = 0.48876682, step = 13500 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 753.917\n",
      "INFO:tensorflow:loss = 0.016977577, step = 13600 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 723.362\n",
      "INFO:tensorflow:loss = 0.21165565, step = 13700 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.43\n",
      "INFO:tensorflow:loss = 0.07759167, step = 13800 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 720.41\n",
      "INFO:tensorflow:loss = 0.30665547, step = 13900 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 628.631\n",
      "INFO:tensorflow:loss = 0.061501402, step = 14000 (0.159 sec)\n",
      "INFO:tensorflow:global_step/sec: 652.142\n",
      "INFO:tensorflow:loss = 0.22736073, step = 14100 (0.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 615.068\n",
      "INFO:tensorflow:loss = 0.07389079, step = 14200 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 688.411\n",
      "INFO:tensorflow:loss = 0.29698288, step = 14300 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 711.249\n",
      "INFO:tensorflow:loss = 0.14422297, step = 14400 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 719.837\n",
      "INFO:tensorflow:loss = 0.014385125, step = 14500 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 728.481\n",
      "INFO:tensorflow:loss = 0.072385326, step = 14600 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 633.549\n",
      "INFO:tensorflow:loss = 0.2490507, step = 14700 (0.158 sec)\n",
      "INFO:tensorflow:global_step/sec: 655.833\n",
      "INFO:tensorflow:loss = 0.13730074, step = 14800 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 669.018\n",
      "INFO:tensorflow:loss = 0.094497725, step = 14900 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 648.228\n",
      "INFO:tensorflow:loss = 0.02053002, step = 15000 (0.154 sec)\n",
      "INFO:tensorflow:global_step/sec: 705.368\n",
      "INFO:tensorflow:loss = 0.07519817, step = 15100 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 716.654\n",
      "INFO:tensorflow:loss = 5.0296702, step = 15200 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 694.54\n",
      "INFO:tensorflow:loss = 0.0641305, step = 15300 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 706.583\n",
      "INFO:tensorflow:loss = 0.046975322, step = 15400 (0.142 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 760.143\n",
      "INFO:tensorflow:loss = 0.19096217, step = 15500 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 729.355\n",
      "INFO:tensorflow:loss = 0.07447162, step = 15600 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 721.879\n",
      "INFO:tensorflow:loss = 0.008595724, step = 15700 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 725.15\n",
      "INFO:tensorflow:loss = 0.02350374, step = 15800 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 750.054\n",
      "INFO:tensorflow:loss = 0.23393407, step = 15900 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 686.044\n",
      "INFO:tensorflow:loss = 0.052663296, step = 16000 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 571.582\n",
      "INFO:tensorflow:loss = 0.07737052, step = 16100 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 621.862\n",
      "INFO:tensorflow:loss = 0.059392054, step = 16200 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 680.005\n",
      "INFO:tensorflow:loss = 0.112042084, step = 16300 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 733.16\n",
      "INFO:tensorflow:loss = 0.019262556, step = 16400 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 747.072\n",
      "INFO:tensorflow:loss = 0.11752561, step = 16500 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 734.132\n",
      "INFO:tensorflow:loss = 0.23796175, step = 16600 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 737.207\n",
      "INFO:tensorflow:loss = 0.2813179, step = 16700 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 735.658\n",
      "INFO:tensorflow:loss = 0.08527176, step = 16800 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 760.729\n",
      "INFO:tensorflow:loss = 0.0708789, step = 16900 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 780.434\n",
      "INFO:tensorflow:loss = 0.10494808, step = 17000 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 802.2\n",
      "INFO:tensorflow:loss = 0.053045884, step = 17100 (0.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 758.93\n",
      "INFO:tensorflow:loss = 0.019496916, step = 17200 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 777.573\n",
      "INFO:tensorflow:loss = 0.2686581, step = 17300 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 798.333\n",
      "INFO:tensorflow:loss = 0.0589393, step = 17400 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 779.037\n",
      "INFO:tensorflow:loss = 0.04246951, step = 17500 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 781.962\n",
      "INFO:tensorflow:loss = 0.1414603, step = 17600 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 637.081\n",
      "INFO:tensorflow:loss = 0.03246697, step = 17700 (0.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 617.478\n",
      "INFO:tensorflow:loss = 0.1726821, step = 17800 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 670.541\n",
      "INFO:tensorflow:loss = 0.16460724, step = 17900 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 775.135\n",
      "INFO:tensorflow:loss = 0.2628479, step = 18000 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 732.683\n",
      "INFO:tensorflow:loss = 0.092953764, step = 18100 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 766.433\n",
      "INFO:tensorflow:loss = 0.04855925, step = 18200 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 724.314\n",
      "INFO:tensorflow:loss = 0.1251901, step = 18300 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 617.447\n",
      "INFO:tensorflow:loss = 0.07948378, step = 18400 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 605.474\n",
      "INFO:tensorflow:loss = 0.0709505, step = 18500 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 681.306\n",
      "INFO:tensorflow:loss = 0.047453385, step = 18600 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 763.621\n",
      "INFO:tensorflow:loss = 0.25552556, step = 18700 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 782.048\n",
      "INFO:tensorflow:loss = 0.09249738, step = 18800 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 592.506\n",
      "INFO:tensorflow:loss = 0.013389738, step = 18900 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 673.123\n",
      "INFO:tensorflow:loss = 0.13284124, step = 19000 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 734.839\n",
      "INFO:tensorflow:loss = 0.1085002, step = 19100 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 679.734\n",
      "INFO:tensorflow:loss = 0.2795027, step = 19200 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 630.326\n",
      "INFO:tensorflow:loss = 0.04235497, step = 19300 (0.159 sec)\n",
      "INFO:tensorflow:global_step/sec: 744.772\n",
      "INFO:tensorflow:loss = 0.184058, step = 19400 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 720.66\n",
      "INFO:tensorflow:loss = 0.10944146, step = 19500 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 682.055\n",
      "INFO:tensorflow:loss = 0.61868834, step = 19600 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 669.81\n",
      "INFO:tensorflow:loss = 0.013134284, step = 19700 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 728.217\n",
      "INFO:tensorflow:loss = 0.07665045, step = 19800 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 765.268\n",
      "INFO:tensorflow:loss = 0.13330126, step = 19900 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.119\n",
      "INFO:tensorflow:loss = 0.1017472, step = 20000 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 766.664\n",
      "INFO:tensorflow:loss = 0.32613552, step = 20100 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 755.936\n",
      "INFO:tensorflow:loss = 0.08524483, step = 20200 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 750.554\n",
      "INFO:tensorflow:loss = 0.10158778, step = 20300 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 776.959\n",
      "INFO:tensorflow:loss = 0.029059863, step = 20400 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 788.694\n",
      "INFO:tensorflow:loss = 0.012308217, step = 20500 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 782.133\n",
      "INFO:tensorflow:loss = 0.059527285, step = 20600 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 751.991\n",
      "INFO:tensorflow:loss = 0.008613001, step = 20700 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 600.703\n",
      "INFO:tensorflow:loss = 0.006468304, step = 20800 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 555.84\n",
      "INFO:tensorflow:loss = 0.041916292, step = 20900 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 636.831\n",
      "INFO:tensorflow:loss = 0.004384146, step = 21000 (0.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 683.558\n",
      "INFO:tensorflow:loss = 0.024349477, step = 21100 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 640.381\n",
      "INFO:tensorflow:loss = 0.04808281, step = 21200 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 679.647\n",
      "INFO:tensorflow:loss = 0.12497167, step = 21300 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 746.897\n",
      "INFO:tensorflow:loss = 0.037052717, step = 21400 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 755.997\n",
      "INFO:tensorflow:loss = 0.006926307, step = 21500 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 773.32\n",
      "INFO:tensorflow:loss = 0.074007586, step = 21600 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 718.055\n",
      "INFO:tensorflow:loss = 0.10291884, step = 21700 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 750.981\n",
      "INFO:tensorflow:loss = 0.012438581, step = 21800 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 765.381\n",
      "INFO:tensorflow:loss = 0.0038113636, step = 21900 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 748.057\n",
      "INFO:tensorflow:loss = 0.03924905, step = 22000 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 758.258\n",
      "INFO:tensorflow:loss = 0.0941266, step = 22100 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 748.376\n",
      "INFO:tensorflow:loss = 0.0047066705, step = 22200 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.068\n",
      "INFO:tensorflow:loss = 0.049610026, step = 22300 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 764.656\n",
      "INFO:tensorflow:loss = 0.026602667, step = 22400 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 719.336\n",
      "INFO:tensorflow:loss = 0.037756026, step = 22500 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 708.671\n",
      "INFO:tensorflow:loss = 0.009034704, step = 22600 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 696.592\n",
      "INFO:tensorflow:loss = 0.121186815, step = 22700 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 719.805\n",
      "INFO:tensorflow:loss = 0.018444436, step = 22800 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 750.769\n",
      "INFO:tensorflow:loss = 0.15817365, step = 22900 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 735.057\n",
      "INFO:tensorflow:loss = 0.07102643, step = 23000 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 723.982\n",
      "INFO:tensorflow:loss = 0.053571053, step = 23100 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 728.232\n",
      "INFO:tensorflow:loss = 0.0041952957, step = 23200 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 746.255\n",
      "INFO:tensorflow:loss = 0.07324758, step = 23300 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 698.442\n",
      "INFO:tensorflow:loss = 0.013589903, step = 23400 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 672.646\n",
      "INFO:tensorflow:loss = 0.10314279, step = 23500 (0.149 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 720.016\n",
      "INFO:tensorflow:loss = 0.05611958, step = 23600 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 670.816\n",
      "INFO:tensorflow:loss = 0.040629435, step = 23700 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 679.358\n",
      "INFO:tensorflow:loss = 0.039468687, step = 23800 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 591.633\n",
      "INFO:tensorflow:loss = 0.07725768, step = 23900 (0.169 sec)\n",
      "INFO:tensorflow:global_step/sec: 626.386\n",
      "INFO:tensorflow:loss = 0.07133426, step = 24000 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 622.443\n",
      "INFO:tensorflow:loss = 0.01620267, step = 24100 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 757.569\n",
      "INFO:tensorflow:loss = 0.033674255, step = 24200 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 759.175\n",
      "INFO:tensorflow:loss = 0.02904886, step = 24300 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 643.196\n",
      "INFO:tensorflow:loss = 0.021125063, step = 24400 (0.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 764.936\n",
      "INFO:tensorflow:loss = 0.010064006, step = 24500 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 623.595\n",
      "INFO:tensorflow:loss = 0.22093584, step = 24600 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 642.635\n",
      "INFO:tensorflow:loss = 0.04337728, step = 24700 (0.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 642.833\n",
      "INFO:tensorflow:loss = 0.017090034, step = 24800 (0.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 648.077\n",
      "INFO:tensorflow:loss = 0.058798455, step = 24900 (0.154 sec)\n",
      "INFO:tensorflow:global_step/sec: 611.414\n",
      "INFO:tensorflow:loss = 0.026018398, step = 25000 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 637.37\n",
      "INFO:tensorflow:loss = 0.05266387, step = 25100 (0.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 662.643\n",
      "INFO:tensorflow:loss = 0.02904391, step = 25200 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 614.739\n",
      "INFO:tensorflow:loss = 0.03309081, step = 25300 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 598.224\n",
      "INFO:tensorflow:loss = 0.114326894, step = 25400 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 663.694\n",
      "INFO:tensorflow:loss = 0.038087483, step = 25500 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 636.034\n",
      "INFO:tensorflow:loss = 0.042078175, step = 25600 (0.158 sec)\n",
      "INFO:tensorflow:global_step/sec: 587.527\n",
      "INFO:tensorflow:loss = 0.021071143, step = 25700 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 664.395\n",
      "INFO:tensorflow:loss = 0.039268643, step = 25800 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 705.829\n",
      "INFO:tensorflow:loss = 0.00580811, step = 25900 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 636.116\n",
      "INFO:tensorflow:loss = 0.06418324, step = 26000 (0.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 619.763\n",
      "INFO:tensorflow:loss = 0.030529939, step = 26100 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 592.966\n",
      "INFO:tensorflow:loss = 0.019166447, step = 26200 (0.169 sec)\n",
      "INFO:tensorflow:global_step/sec: 589.27\n",
      "INFO:tensorflow:loss = 0.0116247, step = 26300 (0.169 sec)\n",
      "INFO:tensorflow:global_step/sec: 695.168\n",
      "INFO:tensorflow:loss = 0.030719474, step = 26400 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 732.142\n",
      "INFO:tensorflow:loss = 0.11685369, step = 26500 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 760.473\n",
      "INFO:tensorflow:loss = 0.01657794, step = 26600 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 671.327\n",
      "INFO:tensorflow:loss = 0.02757436, step = 26700 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 668.14\n",
      "INFO:tensorflow:loss = 0.013070018, step = 26800 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 771.784\n",
      "INFO:tensorflow:loss = 0.16727, step = 26900 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 768.783\n",
      "INFO:tensorflow:loss = 0.026858501, step = 27000 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 806.951\n",
      "INFO:tensorflow:loss = 0.020017473, step = 27100 (0.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 792.151\n",
      "INFO:tensorflow:loss = 0.0063375104, step = 27200 (0.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 743.85\n",
      "INFO:tensorflow:loss = 0.038745653, step = 27300 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 763.696\n",
      "INFO:tensorflow:loss = 0.07117924, step = 27400 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 718.526\n",
      "INFO:tensorflow:loss = 0.23104118, step = 27500 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 691.104\n",
      "INFO:tensorflow:loss = 0.045554783, step = 27600 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 708.787\n",
      "INFO:tensorflow:loss = 0.054858394, step = 27700 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 749.613\n",
      "INFO:tensorflow:loss = 0.1919387, step = 27800 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 676.335\n",
      "INFO:tensorflow:loss = 0.035167128, step = 27900 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 756.045\n",
      "INFO:tensorflow:loss = 0.06809587, step = 28000 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 745.169\n",
      "INFO:tensorflow:loss = 0.020983905, step = 28100 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 731.403\n",
      "INFO:tensorflow:loss = 0.009154754, step = 28200 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 761.517\n",
      "INFO:tensorflow:loss = 0.013444103, step = 28300 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 735.488\n",
      "INFO:tensorflow:loss = 0.019219223, step = 28400 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 701.509\n",
      "INFO:tensorflow:loss = 0.0007980311, step = 28500 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 672.906\n",
      "INFO:tensorflow:loss = 0.037370995, step = 28600 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 693.067\n",
      "INFO:tensorflow:loss = 0.034441676, step = 28700 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 715.073\n",
      "INFO:tensorflow:loss = 0.019205665, step = 28800 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.271\n",
      "INFO:tensorflow:loss = 0.04330594, step = 28900 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.943\n",
      "INFO:tensorflow:loss = 0.02258328, step = 29000 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 778.761\n",
      "INFO:tensorflow:loss = 0.033814937, step = 29100 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 736.08\n",
      "INFO:tensorflow:loss = 0.06819173, step = 29200 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 694.266\n",
      "INFO:tensorflow:loss = 0.03306775, step = 29300 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.808\n",
      "INFO:tensorflow:loss = 0.044940256, step = 29400 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.514\n",
      "INFO:tensorflow:loss = 0.041515682, step = 29500 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 741.502\n",
      "INFO:tensorflow:loss = 0.0137351295, step = 29600 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 726.773\n",
      "INFO:tensorflow:loss = 0.058922045, step = 29700 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 772.537\n",
      "INFO:tensorflow:loss = 0.004406413, step = 29800 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 719.1\n",
      "INFO:tensorflow:loss = 0.02618828, step = 29900 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 723.709\n",
      "INFO:tensorflow:loss = 0.035493106, step = 30000 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 762.042\n",
      "INFO:tensorflow:loss = 0.032594115, step = 30100 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 774.994\n",
      "INFO:tensorflow:loss = 0.012513466, step = 30200 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 727.626\n",
      "INFO:tensorflow:loss = 0.030434133, step = 30300 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 751.067\n",
      "INFO:tensorflow:loss = 0.009930114, step = 30400 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 740.545\n",
      "INFO:tensorflow:loss = 0.012347832, step = 30500 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 768.107\n",
      "INFO:tensorflow:loss = 0.008713552, step = 30600 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 737.775\n",
      "INFO:tensorflow:loss = 0.01328897, step = 30700 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 700.529\n",
      "INFO:tensorflow:loss = 0.005648759, step = 30800 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 741.498\n",
      "INFO:tensorflow:loss = 0.045800988, step = 30900 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 759.22\n",
      "INFO:tensorflow:loss = 0.0049998416, step = 31000 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 778.944\n",
      "INFO:tensorflow:loss = 0.025347317, step = 31100 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 756.877\n",
      "INFO:tensorflow:loss = 0.02644661, step = 31200 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 722.356\n",
      "INFO:tensorflow:loss = 0.012870766, step = 31300 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 659.329\n",
      "INFO:tensorflow:loss = 0.030599056, step = 31400 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 724.764\n",
      "INFO:tensorflow:loss = 0.09482564, step = 31500 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 738.252\n",
      "INFO:tensorflow:loss = 0.019195255, step = 31600 (0.135 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 724.457\n",
      "INFO:tensorflow:loss = 0.011149004, step = 31700 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 749.343\n",
      "INFO:tensorflow:loss = 0.011274818, step = 31800 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.722\n",
      "INFO:tensorflow:loss = 0.016880032, step = 31900 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 686.074\n",
      "INFO:tensorflow:loss = 0.034965005, step = 32000 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 747.228\n",
      "INFO:tensorflow:loss = 0.0711512, step = 32100 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 758.061\n",
      "INFO:tensorflow:loss = 0.020935431, step = 32200 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.156\n",
      "INFO:tensorflow:loss = 0.05625681, step = 32300 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 752.545\n",
      "INFO:tensorflow:loss = 0.008945845, step = 32400 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 774.541\n",
      "INFO:tensorflow:loss = 0.0025368293, step = 32500 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 778.982\n",
      "INFO:tensorflow:loss = 0.025532966, step = 32600 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 749.828\n",
      "INFO:tensorflow:loss = 0.024839351, step = 32700 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 740.441\n",
      "INFO:tensorflow:loss = 0.05547411, step = 32800 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 721.51\n",
      "INFO:tensorflow:loss = 0.09110242, step = 32900 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 763.617\n",
      "INFO:tensorflow:loss = 0.027063178, step = 33000 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 704.674\n",
      "INFO:tensorflow:loss = 0.14633262, step = 33100 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.149\n",
      "INFO:tensorflow:loss = 0.007330508, step = 33200 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 733.544\n",
      "INFO:tensorflow:loss = 0.008549547, step = 33300 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 750.367\n",
      "INFO:tensorflow:loss = 0.024134047, step = 33400 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 756.464\n",
      "INFO:tensorflow:loss = 0.0074961022, step = 33500 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 778.015\n",
      "INFO:tensorflow:loss = 0.018352449, step = 33600 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 790.108\n",
      "INFO:tensorflow:loss = 0.016640741, step = 33700 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 781.526\n",
      "INFO:tensorflow:loss = 0.0017197947, step = 33800 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 785.147\n",
      "INFO:tensorflow:loss = 0.0111332275, step = 33900 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 760.909\n",
      "INFO:tensorflow:loss = 0.040406324, step = 34000 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 772.981\n",
      "INFO:tensorflow:loss = 0.012423429, step = 34100 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 781.531\n",
      "INFO:tensorflow:loss = 0.015810683, step = 34200 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 720.78\n",
      "INFO:tensorflow:loss = 0.0105865635, step = 34300 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 734.963\n",
      "INFO:tensorflow:loss = 0.022520129, step = 34400 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 687.737\n",
      "INFO:tensorflow:loss = 0.007853265, step = 34500 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 755.826\n",
      "INFO:tensorflow:loss = 0.017153783, step = 34600 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 733.16\n",
      "INFO:tensorflow:loss = 0.04800478, step = 34700 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 687.501\n",
      "INFO:tensorflow:loss = 0.022419866, step = 34800 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 614.493\n",
      "INFO:tensorflow:loss = 0.0017927951, step = 34900 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 745.438\n",
      "INFO:tensorflow:loss = 0.04672495, step = 35000 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 757.189\n",
      "INFO:tensorflow:loss = 0.0890024, step = 35100 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 751.348\n",
      "INFO:tensorflow:loss = 0.0034388623, step = 35200 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 737.144\n",
      "INFO:tensorflow:loss = 0.033513878, step = 35300 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 730.169\n",
      "INFO:tensorflow:loss = 0.0026553052, step = 35400 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 716.332\n",
      "INFO:tensorflow:loss = 0.040838942, step = 35500 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 693.586\n",
      "INFO:tensorflow:loss = 0.023297546, step = 35600 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 766.726\n",
      "INFO:tensorflow:loss = 0.039266217, step = 35700 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 800.56\n",
      "INFO:tensorflow:loss = 0.006094549, step = 35800 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 779.842\n",
      "INFO:tensorflow:loss = 0.024597563, step = 35900 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 791.645\n",
      "INFO:tensorflow:loss = 0.0005974211, step = 36000 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 760.571\n",
      "INFO:tensorflow:loss = 0.008686724, step = 36100 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 779.966\n",
      "INFO:tensorflow:loss = 0.003942257, step = 36200 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 786.328\n",
      "INFO:tensorflow:loss = 0.012746293, step = 36300 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 609.898\n",
      "INFO:tensorflow:loss = 0.03244071, step = 36400 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 701.438\n",
      "INFO:tensorflow:loss = 0.0060920594, step = 36500 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 789.399\n",
      "INFO:tensorflow:loss = 0.02364352, step = 36600 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 798.938\n",
      "INFO:tensorflow:loss = 0.03801505, step = 36700 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 643.811\n",
      "INFO:tensorflow:loss = 0.016948957, step = 36800 (0.155 sec)\n",
      "INFO:tensorflow:global_step/sec: 664.83\n",
      "INFO:tensorflow:loss = 0.017786317, step = 36900 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 543.587\n",
      "INFO:tensorflow:loss = 0.058662392, step = 37000 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 632.46\n",
      "INFO:tensorflow:loss = 0.014718251, step = 37100 (0.158 sec)\n",
      "INFO:tensorflow:global_step/sec: 775.433\n",
      "INFO:tensorflow:loss = 0.047604885, step = 37200 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 786.68\n",
      "INFO:tensorflow:loss = 0.002843613, step = 37300 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 799.111\n",
      "INFO:tensorflow:loss = 0.023884777, step = 37400 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 756.131\n",
      "INFO:tensorflow:loss = 0.017163115, step = 37500 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 633.787\n",
      "INFO:tensorflow:loss = 0.026272407, step = 37600 (0.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 723.891\n",
      "INFO:tensorflow:loss = 0.0028121169, step = 37700 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.086\n",
      "INFO:tensorflow:loss = 0.016791945, step = 37800 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 701.247\n",
      "INFO:tensorflow:loss = 0.047742378, step = 37900 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 682.181\n",
      "INFO:tensorflow:loss = 0.04361719, step = 38000 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 786.376\n",
      "INFO:tensorflow:loss = 0.038325347, step = 38100 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 804.283\n",
      "INFO:tensorflow:loss = 0.010711223, step = 38200 (0.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 770.032\n",
      "INFO:tensorflow:loss = 0.11263705, step = 38300 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 759.344\n",
      "INFO:tensorflow:loss = 0.06331622, step = 38400 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 793.153\n",
      "INFO:tensorflow:loss = 0.03488349, step = 38500 (0.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 748.129\n",
      "INFO:tensorflow:loss = 0.024869196, step = 38600 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 760.015\n",
      "INFO:tensorflow:loss = 0.049668543, step = 38700 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 795.537\n",
      "INFO:tensorflow:loss = 0.024653997, step = 38800 (0.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.715\n",
      "INFO:tensorflow:loss = 0.0064418544, step = 38900 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 805.211\n",
      "INFO:tensorflow:loss = 0.025671637, step = 39000 (0.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 763.917\n",
      "INFO:tensorflow:loss = 0.007240239, step = 39100 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 756.067\n",
      "INFO:tensorflow:loss = 0.0044421586, step = 39200 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 778.837\n",
      "INFO:tensorflow:loss = 0.024619173, step = 39300 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.662\n",
      "INFO:tensorflow:loss = 0.012689214, step = 39400 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 746.614\n",
      "INFO:tensorflow:loss = 0.018057879, step = 39500 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 694.068\n",
      "INFO:tensorflow:loss = 0.02785282, step = 39600 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 654.979\n",
      "INFO:tensorflow:loss = 0.012543851, step = 39700 (0.152 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 731.693\n",
      "INFO:tensorflow:loss = 0.034165256, step = 39800 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 712.66\n",
      "INFO:tensorflow:loss = 0.018817844, step = 39900 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 767.888\n",
      "INFO:tensorflow:loss = 0.064586915, step = 40000 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 682.678\n",
      "INFO:tensorflow:loss = 0.039944634, step = 40100 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 637.928\n",
      "INFO:tensorflow:loss = 0.006592812, step = 40200 (0.158 sec)\n",
      "INFO:tensorflow:global_step/sec: 654.42\n",
      "INFO:tensorflow:loss = 0.0012000705, step = 40300 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 735.869\n",
      "INFO:tensorflow:loss = 0.0021813547, step = 40400 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 739.684\n",
      "INFO:tensorflow:loss = 0.020016534, step = 40500 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 722.897\n",
      "INFO:tensorflow:loss = 0.010720066, step = 40600 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 750.822\n",
      "INFO:tensorflow:loss = 0.05658368, step = 40700 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 761.754\n",
      "INFO:tensorflow:loss = 0.013007379, step = 40800 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 721.558\n",
      "INFO:tensorflow:loss = 0.0246561, step = 40900 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 658.625\n",
      "INFO:tensorflow:loss = 0.01263381, step = 41000 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 691.157\n",
      "INFO:tensorflow:loss = 0.004255068, step = 41100 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 710.883\n",
      "INFO:tensorflow:loss = 0.015441139, step = 41200 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 797.458\n",
      "INFO:tensorflow:loss = 0.010338774, step = 41300 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 747.132\n",
      "INFO:tensorflow:loss = 0.021103038, step = 41400 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 750.845\n",
      "INFO:tensorflow:loss = 0.025617313, step = 41500 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.583\n",
      "INFO:tensorflow:loss = 0.013572226, step = 41600 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 756.386\n",
      "INFO:tensorflow:loss = 0.0033336552, step = 41700 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 782.906\n",
      "INFO:tensorflow:loss = 0.053168457, step = 41800 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 719.6\n",
      "INFO:tensorflow:loss = 0.029688315, step = 41900 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 788.136\n",
      "INFO:tensorflow:loss = 0.01654516, step = 42000 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 738.526\n",
      "INFO:tensorflow:loss = 0.015659308, step = 42100 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 777.578\n",
      "INFO:tensorflow:loss = 0.01840079, step = 42200 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 786.416\n",
      "INFO:tensorflow:loss = 0.021663003, step = 42300 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 763.347\n",
      "INFO:tensorflow:loss = 0.011599571, step = 42400 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 762.321\n",
      "INFO:tensorflow:loss = 0.0044469903, step = 42500 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 768.549\n",
      "INFO:tensorflow:loss = 0.0019147585, step = 42600 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 771.429\n",
      "INFO:tensorflow:loss = 0.0054854164, step = 42700 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 793.871\n",
      "INFO:tensorflow:loss = 0.0017117725, step = 42800 (0.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 770.1\n",
      "INFO:tensorflow:loss = 0.012048513, step = 42900 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 744.636\n",
      "INFO:tensorflow:loss = 0.06634566, step = 43000 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 696.882\n",
      "INFO:tensorflow:loss = 0.0003919307, step = 43100 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 705.516\n",
      "INFO:tensorflow:loss = 0.06582007, step = 43200 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 699.244\n",
      "INFO:tensorflow:loss = 0.0038124803, step = 43300 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 792.079\n",
      "INFO:tensorflow:loss = 0.003364585, step = 43400 (0.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 753.586\n",
      "INFO:tensorflow:loss = 0.00725976, step = 43500 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 720.951\n",
      "INFO:tensorflow:loss = 0.024148291, step = 43600 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 770.384\n",
      "INFO:tensorflow:loss = 0.013779048, step = 43700 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 799.363\n",
      "INFO:tensorflow:loss = 0.014951154, step = 43800 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 791.774\n",
      "INFO:tensorflow:loss = 0.0015594304, step = 43900 (0.126 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 44000 into /tmp/tmpuflzeb_h/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.0012097486.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.estimator.canned.dnn.DNNClassifier at 0x7f4f62b23be0>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_cols = [tf.feature_column.numeric_column(\"X\", shape=[28 * 28])]\n",
    "dnn_clf = tf.estimator.DNNClassifier(hidden_units=[300,100], n_classes=10,\n",
    "                                     feature_columns=feature_cols)\n",
    "\n",
    "input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x={\"X\": X_train}, y=y_train, num_epochs=40, batch_size=50, shuffle=True)\n",
    "dnn_clf.train(input_fn=input_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2018-05-18-19:12:49\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /tmp/tmpuflzeb_h/model.ckpt-44000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2018-05-18-19:12:50\n",
      "INFO:tensorflow:Saving dict for global step 44000: accuracy = 0.9798, average_loss = 0.10096103, global_step = 44000, loss = 12.779877\n"
     ]
    }
   ],
   "source": [
    "test_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x={\"X\": X_test}, y=y_test, shuffle=False)\n",
    "eval_results = dnn_clf.evaluate(input_fn=test_input_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.9798,\n",
       " 'average_loss': 0.10096103,\n",
       " 'global_step': 44000,\n",
       " 'loss': 12.779877}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /tmp/tmpuflzeb_h/model.ckpt-44000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'class_ids': array([7]),\n",
       " 'classes': array([b'7'], dtype=object),\n",
       " 'logits': array([ -3.809414 ,  -4.1564407,  -0.426081 ,   3.2636993, -11.065331 ,\n",
       "         -8.790985 , -10.436305 ,  19.935707 ,  -6.9282775,   2.2807484],\n",
       "       dtype=float32),\n",
       " 'probabilities': array([4.8710768e-11, 3.4428106e-11, 1.4354495e-09, 5.7469666e-08,\n",
       "        3.4389070e-14, 3.3431518e-13, 6.4506329e-14, 1.0000000e+00,\n",
       "        2.1533745e-12, 2.1505466e-08], dtype=float32)}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_iter = dnn_clf.predict(input_fn=test_input_fn)\n",
    "y_pred = list(y_pred_iter)\n",
    "y_pred[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Using plain TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def neuron_layer(X, n_neurons, name, activation=None):\n",
    "    with tf.name_scope(name):\n",
    "        n_inputs = int(X.get_shape()[1])\n",
    "        stddev = 2 / np.sqrt(n_inputs)\n",
    "        init = tf.truncated_normal((n_inputs, n_neurons), stddev=stddev)\n",
    "        W = tf.Variable(init, name=\"kernel\")\n",
    "        b = tf.Variable(tf.zeros([n_neurons]), name=\"bias\")\n",
    "        Z = tf.matmul(X, W) + b\n",
    "        if activation is not None:\n",
    "            return activation(Z)\n",
    "        else:\n",
    "            return Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = neuron_layer(X, n_hidden1, name=\"hidden1\",\n",
    "                           activation=tf.nn.relu)\n",
    "    hidden2 = neuron_layer(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                           activation=tf.nn.relu)\n",
    "    logits = neuron_layer(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,\n",
    "                                                              logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 40\n",
    "batch_size = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shuffle_batch(X, y, batch_size):\n",
    "    rnd_idx = np.random.permutation(len(X))\n",
    "    n_batches = len(X) // batch_size\n",
    "    for batch_idx in np.array_split(rnd_idx, n_batches):\n",
    "        X_batch, y_batch = X[batch_idx], y[batch_idx]\n",
    "        yield X_batch, y_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Batch accuracy: 0.9 Val accuracy: 0.9146\n",
      "1 Batch accuracy: 0.92 Val accuracy: 0.936\n",
      "2 Batch accuracy: 0.96 Val accuracy: 0.945\n",
      "3 Batch accuracy: 0.92 Val accuracy: 0.9512\n",
      "4 Batch accuracy: 0.98 Val accuracy: 0.9558\n",
      "5 Batch accuracy: 0.96 Val accuracy: 0.9566\n",
      "6 Batch accuracy: 1.0 Val accuracy: 0.9612\n",
      "7 Batch accuracy: 0.94 Val accuracy: 0.963\n",
      "8 Batch accuracy: 0.98 Val accuracy: 0.9652\n",
      "9 Batch accuracy: 0.96 Val accuracy: 0.966\n",
      "10 Batch accuracy: 0.92 Val accuracy: 0.9688\n",
      "11 Batch accuracy: 0.98 Val accuracy: 0.969\n",
      "12 Batch accuracy: 0.98 Val accuracy: 0.967\n",
      "13 Batch accuracy: 0.98 Val accuracy: 0.9706\n",
      "14 Batch accuracy: 1.0 Val accuracy: 0.9714\n",
      "15 Batch accuracy: 0.94 Val accuracy: 0.9732\n",
      "16 Batch accuracy: 1.0 Val accuracy: 0.9736\n",
      "17 Batch accuracy: 1.0 Val accuracy: 0.9742\n",
      "18 Batch accuracy: 1.0 Val accuracy: 0.9746\n",
      "19 Batch accuracy: 0.98 Val accuracy: 0.9748\n",
      "20 Batch accuracy: 1.0 Val accuracy: 0.9752\n",
      "21 Batch accuracy: 1.0 Val accuracy: 0.9752\n",
      "22 Batch accuracy: 0.98 Val accuracy: 0.9764\n",
      "23 Batch accuracy: 0.98 Val accuracy: 0.9752\n",
      "24 Batch accuracy: 0.98 Val accuracy: 0.9772\n",
      "25 Batch accuracy: 1.0 Val accuracy: 0.977\n",
      "26 Batch accuracy: 0.98 Val accuracy: 0.9778\n",
      "27 Batch accuracy: 1.0 Val accuracy: 0.9774\n",
      "28 Batch accuracy: 0.96 Val accuracy: 0.9754\n",
      "29 Batch accuracy: 0.98 Val accuracy: 0.9776\n",
      "30 Batch accuracy: 1.0 Val accuracy: 0.9756\n",
      "31 Batch accuracy: 0.98 Val accuracy: 0.9772\n",
      "32 Batch accuracy: 0.98 Val accuracy: 0.9772\n",
      "33 Batch accuracy: 0.98 Val accuracy: 0.979\n",
      "34 Batch accuracy: 1.0 Val accuracy: 0.9784\n",
      "35 Batch accuracy: 1.0 Val accuracy: 0.9778\n",
      "36 Batch accuracy: 0.98 Val accuracy: 0.978\n",
      "37 Batch accuracy: 1.0 Val accuracy: 0.9776\n",
      "38 Batch accuracy: 1.0 Val accuracy: 0.9792\n",
      "39 Batch accuracy: 1.0 Val accuracy: 0.9776\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"Batch accuracy:\", acc_batch, \"Val accuracy:\", acc_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_final.ckpt\") # or better, use save_path\n",
    "    X_new_scaled = X_test[:20]\n",
    "    Z = logits.eval(feed_dict={X: X_new_scaled})\n",
    "    y_pred = np.argmax(Z, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted classes: [7 2 1 0 4 1 4 9 5 9 0 6 9 0 1 5 9 7 3 4]\n",
      "Actual classes:    [7 2 1 0 4 1 4 9 5 9 0 6 9 0 1 5 9 7 3 4]\n"
     ]
    }
   ],
   "source": [
    "print(\"Predicted classes:\", y_pred)\n",
    "print(\"Actual classes:   \", y_test[:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow_graph_in_jupyter import show_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script src=&quot;//cdnjs.cloudflare.com/ajax/libs/polymer/0.3.3/platform.js&quot;></script>\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.2851015593374667&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 784\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        unknown_rank: true\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0714285746216774\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 5\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/mul&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;dnn/hidden1/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;dnn/hidden1/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;dnn/hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;dnn/hidden1/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\000d\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.1154700517654419\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 21\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/mul&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;dnn/hidden2/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;dnn/hidden2/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;dnn/hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;dnn/hidden2/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;d\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.20000000298023224\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/outputs/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 37\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/outputs/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/outputs/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/outputs/truncated_normal/mul&quot;\\n  input: &quot;dnn/outputs/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;dnn/outputs/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;dnn/outputs/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;dnn/outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;dnn/outputs/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/add&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/grad_ys_0&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;train/gradients/Shape&quot;\\n  input: &quot;train/gradients/grad_ys_0&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/Fill&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;message&quot;\\n    value {\\n      s: &quot;Currently there is no way to take the second derivative of sparse_softmax_cross_entropy_with_logits due to the fused implementation\\\\\\'s interaction with tf.gradients()&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 10\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 100\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 300\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden1/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden1/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden2/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden2/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/outputs/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/outputs/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden1/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden1/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden2/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden2/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/outputs/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/outputs/kernel/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2&quot;\\n  op: &quot;InTopKV2&quot;\\n  input: &quot;dnn/outputs/add&quot;\\n  input: &quot;y&quot;\\n  input: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/in_top_k/InTopKV2&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^dnn/hidden1/bias/Assign&quot;\\n  input: &quot;^dnn/hidden1/kernel/Assign&quot;\\n  input: &quot;^dnn/hidden2/bias/Assign&quot;\\n  input: &quot;^dnn/hidden2/kernel/Assign&quot;\\n  input: &quot;^dnn/outputs/bias/Assign&quot;\\n  input: &quot;^dnn/outputs/kernel/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;dnn/hidden1/bias&quot;\\n        string_val: &quot;dnn/hidden1/kernel&quot;\\n        string_val: &quot;dnn/hidden2/bias&quot;\\n        string_val: &quot;dnn/hidden2/kernel&quot;\\n        string_val: &quot;dnn/outputs/bias&quot;\\n        string_val: &quot;dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;dnn/hidden1/bias&quot;\\n        string_val: &quot;dnn/hidden1/kernel&quot;\\n        string_val: &quot;dnn/hidden2/bias&quot;\\n        string_val: &quot;dnn/hidden2/kernel&quot;\\n        string_val: &quot;dnn/outputs/bias&quot;\\n        string_val: &quot;dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;save/RestoreV2:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;save/RestoreV2:2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;save/RestoreV2:3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;save/RestoreV2:4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;save/RestoreV2:5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.2851015593374667&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using `dense()` instead of `neuron_layer()`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: previous releases of the book used `tensorflow.contrib.layers.fully_connected()` rather than `tf.layers.dense()` (which did not exist when this chapter was written). It is now preferable to use `tf.layers.dense()`, because anything in the contrib module may change or be deleted without notice. The `dense()` function is almost identical to the `fully_connected()` function, except for a few minor differences:\n",
    "* several parameters are renamed: `scope` becomes `name`, `activation_fn` becomes `activation` (and similarly the `_fn` suffix is removed from other parameters such as `normalizer_fn`), `weights_initializer` becomes `kernel_initializer`, etc.\n",
    "* the default `activation` is now `None` rather than `tf.nn.relu`.\n",
    "* a few more differences are presented in chapter 11."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\",\n",
    "                              activation=tf.nn.relu)\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                              activation=tf.nn.relu)\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")\n",
    "    y_proba = tf.nn.softmax(logits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Batch accuracy: 0.9 Validation accuracy: 0.9024\n",
      "1 Batch accuracy: 0.92 Validation accuracy: 0.9254\n",
      "2 Batch accuracy: 0.94 Validation accuracy: 0.9372\n",
      "3 Batch accuracy: 0.9 Validation accuracy: 0.9416\n",
      "4 Batch accuracy: 0.94 Validation accuracy: 0.9472\n",
      "5 Batch accuracy: 0.94 Validation accuracy: 0.9512\n",
      "6 Batch accuracy: 1.0 Validation accuracy: 0.9548\n",
      "7 Batch accuracy: 0.94 Validation accuracy: 0.961\n",
      "8 Batch accuracy: 0.96 Validation accuracy: 0.962\n",
      "9 Batch accuracy: 0.94 Validation accuracy: 0.9648\n",
      "10 Batch accuracy: 0.92 Validation accuracy: 0.9656\n",
      "11 Batch accuracy: 0.98 Validation accuracy: 0.9668\n",
      "12 Batch accuracy: 0.98 Validation accuracy: 0.9684\n",
      "13 Batch accuracy: 0.98 Validation accuracy: 0.9702\n",
      "14 Batch accuracy: 1.0 Validation accuracy: 0.9696\n",
      "15 Batch accuracy: 0.94 Validation accuracy: 0.9718\n",
      "16 Batch accuracy: 0.98 Validation accuracy: 0.9728\n",
      "17 Batch accuracy: 1.0 Validation accuracy: 0.973\n",
      "18 Batch accuracy: 0.98 Validation accuracy: 0.9748\n",
      "19 Batch accuracy: 0.98 Validation accuracy: 0.9756\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "n_batches = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_valid = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"Batch accuracy:\", acc_batch, \"Validation accuracy:\", acc_valid)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script src=&quot;//cdnjs.cloudflare.com/ajax/libs/polymer/0.3.3/platform.js&quot;></script>\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.7224827313584268&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 784\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        unknown_rank: true\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 5\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;hidden1/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\000d\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 22\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;hidden2/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;d\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 39\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;outputs/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/Softmax&quot;\\n  op: &quot;Softmax&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/grad_ys_0&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;train/gradients/Shape&quot;\\n  input: &quot;train/gradients/grad_ys_0&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/Fill&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;message&quot;\\n    value {\\n      s: &quot;Currently there is no way to take the second derivative of sparse_softmax_cross_entropy_with_logits due to the fused implementation\\\\\\'s interaction with tf.gradients()&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2&quot;\\n  op: &quot;InTopKV2&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  input: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/in_top_k/InTopKV2&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^hidden1/bias/Assign&quot;\\n  input: &quot;^hidden1/kernel/Assign&quot;\\n  input: &quot;^hidden2/bias/Assign&quot;\\n  input: &quot;^hidden2/kernel/Assign&quot;\\n  input: &quot;^outputs/bias/Assign&quot;\\n  input: &quot;^outputs/kernel/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n        string_val: &quot;hidden1/kernel&quot;\\n        string_val: &quot;hidden2/bias&quot;\\n        string_val: &quot;hidden2/kernel&quot;\\n        string_val: &quot;outputs/bias&quot;\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n        string_val: &quot;hidden1/kernel&quot;\\n        string_val: &quot;hidden2/bias&quot;\\n        string_val: &quot;hidden2/kernel&quot;\\n        string_val: &quot;outputs/bias&quot;\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;save/RestoreV2:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;save/RestoreV2:2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;save/RestoreV2:3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;save/RestoreV2:4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;save/RestoreV2:5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.7224827313584268&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Exercise solutions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. to 8."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "See appendix A."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Train a deep MLP on the MNIST dataset and see if you can get over 98% precision. Just like in the last exercise of chapter 9, try adding all the bells and whistles (i.e., save checkpoints, restore the last checkpoint in case of an interruption, add summaries, plot learning curves using TensorBoard, and so on)._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First let's create the deep net. It's exactly the same as earlier, with just one addition: we add a `tf.summary.scalar()` to track the loss and the accuracy during training, so we can view nice learning curves using TensorBoard."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\",\n",
    "                              activation=tf.nn.relu)\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                              activation=tf.nn.relu)\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "    loss_summary = tf.summary.scalar('log_loss', loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    accuracy_summary = tf.summary.scalar('accuracy', accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we need to define the directory to write the TensorBoard logs to:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "def log_dir(prefix=\"\"):\n",
    "    now = datetime.utcnow().strftime(\"%Y%m%d%H%M%S\")\n",
    "    root_logdir = \"tf_logs\"\n",
    "    if prefix:\n",
    "        prefix += \"-\"\n",
    "    name = prefix + \"run-\" + now\n",
    "    return \"{}/{}/\".format(root_logdir, name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "logdir = log_dir(\"mnist_dnn\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can create the `FileWriter` that we will use to write the TensorBoard logs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Hey! Why don't we implement early stopping? For this, we are going to need to use the validation set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "m, n = X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0 \tValidation accuracy: 92.180% \tLoss: 0.30208\n",
      "Epoch: 5 \tValidation accuracy: 95.980% \tLoss: 0.15037\n",
      "Epoch: 10 \tValidation accuracy: 97.100% \tLoss: 0.11160\n",
      "Epoch: 15 \tValidation accuracy: 97.700% \tLoss: 0.09562\n",
      "Epoch: 20 \tValidation accuracy: 97.840% \tLoss: 0.08309\n",
      "Epoch: 25 \tValidation accuracy: 98.040% \tLoss: 0.07706\n",
      "Epoch: 30 \tValidation accuracy: 98.140% \tLoss: 0.07287\n",
      "Epoch: 35 \tValidation accuracy: 98.280% \tLoss: 0.07133\n",
      "Epoch: 40 \tValidation accuracy: 98.220% \tLoss: 0.06968\n",
      "Epoch: 45 \tValidation accuracy: 98.220% \tLoss: 0.06993\n",
      "Epoch: 50 \tValidation accuracy: 98.160% \tLoss: 0.07093\n",
      "Epoch: 55 \tValidation accuracy: 98.280% \tLoss: 0.06994\n",
      "Epoch: 60 \tValidation accuracy: 98.200% \tLoss: 0.06894\n",
      "Epoch: 65 \tValidation accuracy: 98.260% \tLoss: 0.06906\n",
      "Epoch: 70 \tValidation accuracy: 98.220% \tLoss: 0.07057\n",
      "Epoch: 75 \tValidation accuracy: 98.280% \tLoss: 0.06963\n",
      "Epoch: 80 \tValidation accuracy: 98.320% \tLoss: 0.07264\n",
      "Epoch: 85 \tValidation accuracy: 98.200% \tLoss: 0.07403\n",
      "Epoch: 90 \tValidation accuracy: 98.300% \tLoss: 0.07332\n",
      "Epoch: 95 \tValidation accuracy: 98.180% \tLoss: 0.07535\n",
      "Epoch: 100 \tValidation accuracy: 98.260% \tLoss: 0.07542\n",
      "Early stopping\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10001\n",
    "batch_size = 50\n",
    "n_batches = int(np.ceil(m / batch_size))\n",
    "\n",
    "checkpoint_path = \"/tmp/my_deep_mnist_model.ckpt\"\n",
    "checkpoint_epoch_path = checkpoint_path + \".epoch\"\n",
    "final_model_path = \"./my_deep_mnist_model\"\n",
    "\n",
    "best_loss = np.infty\n",
    "epochs_without_progress = 0\n",
    "max_epochs_without_progress = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    if os.path.isfile(checkpoint_epoch_path):\n",
    "        # if the checkpoint file exists, restore the model and load the epoch number\n",
    "        with open(checkpoint_epoch_path, \"rb\") as f:\n",
    "            start_epoch = int(f.read())\n",
    "        print(\"Training was interrupted. Continuing at epoch\", start_epoch)\n",
    "        saver.restore(sess, checkpoint_path)\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        sess.run(init)\n",
    "\n",
    "    for epoch in range(start_epoch, n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val, loss_val, accuracy_summary_str, loss_summary_str = sess.run([accuracy, loss, accuracy_summary, loss_summary], feed_dict={X: X_valid, y: y_valid})\n",
    "        file_writer.add_summary(accuracy_summary_str, epoch)\n",
    "        file_writer.add_summary(loss_summary_str, epoch)\n",
    "        if epoch % 5 == 0:\n",
    "            print(\"Epoch:\", epoch,\n",
    "                  \"\\tValidation accuracy: {:.3f}%\".format(accuracy_val * 100),\n",
    "                  \"\\tLoss: {:.5f}\".format(loss_val))\n",
    "            saver.save(sess, checkpoint_path)\n",
    "            with open(checkpoint_epoch_path, \"wb\") as f:\n",
    "                f.write(b\"%d\" % (epoch + 1))\n",
    "            if loss_val < best_loss:\n",
    "                saver.save(sess, final_model_path)\n",
    "                best_loss = loss_val\n",
    "            else:\n",
    "                epochs_without_progress += 5\n",
    "                if epochs_without_progress > max_epochs_without_progress:\n",
    "                    print(\"Early stopping\")\n",
    "                    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.remove(checkpoint_epoch_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_deep_mnist_model\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, final_model_path)\n",
    "    accuracy_val = accuracy.eval(feed_dict={X: X_test, y: y_test})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9796"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "nav_menu": {
   "height": "264px",
   "width": "369px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}