Deep learning loss functions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep learning loss functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inline plots:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We want to use `Theano` so that we can use it's auto-differentiation, since I'm too lazy to work out the derivatives of these functions by hand! :)\n",
    "\n",
    "We also want to avoid the overhead of using the GPU for such small tasks, so tell Theano to use the CPU:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch, torch.nn as nn, torch.nn.functional as F\n",
    "from matplotlib import pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set()\n",
    "\n",
    "EPSILON = 1.0e-12\n",
    "\n",
    "SAVE_PLOTS = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classification: softmax non-linearity with negative log loss\n",
    "\n",
    "Define a convenience function for computing the gradient of negative log loss with softmax. We use PyTorch to do it as it handles computing the gradient for us."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Softmax function\n",
    "def f_softmax(logits, axis=1):\n",
    "    ex = np.exp(logits)\n",
    "    return ex / ex.sum(axis=axis, keepdims=True)\n",
    "\n",
    "# Classification loss: negative log of softmax\n",
    "def f_clf_loss(logits, axis=1):\n",
    "    t_logits = torch.tensor(logits, requires_grad=True)\n",
    "    # Compute negative log-softmax\n",
    "    return -F.log_softmax(t_logits, dim=axis).detach().numpy()\n",
    "\n",
    "# Gadient of classification loss\n",
    "def f_clf_loss_grad(logits, target, axis=1):\n",
    "    t_logits = torch.tensor(logits, requires_grad=True)\n",
    "    t_targets = torch.tensor(target, dtype=torch.int64)\n",
    "    # Compute cross_entropy loss\n",
    "    loss = F.cross_entropy(t_logits, t_targets, reduction='sum')\n",
    "    # Sum and compute gradient\n",
    "    loss.backward()\n",
    "    return t_logits.grad.detach().numpy()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUEAAAFKCAYAAACDye3yAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VNX9//HXvXe27PsmkIRAWJR9UUTcKooVERdEEAEBEVAU0apUrd9aaUXbilYUUXCpYl2qiP4qVsW2iuDCJrIlYQlrQkJIQrZZ7/39MSEkyhommUnu5/l4zGNm7p078zlJeHPunXvPUQzDMBBCCJNSg12AEEIEk4SgEMLUJASFEKYmISiEMDUJQSGEqUkICiFMTUJQCGFqEoJCCFOTEBRCmJqEoBDC1CQEhRCmJiEohDA1S7A++Hdf/JkERwKjO1+PRW3aMhISIikpqWzSzwgmaV/L1prb15xtU1WFuLiI094uaCE46KwBvPLTPyipLmVKj1uxa7Ym/Txdb92D5Uj7WrbW3L5Qb1vQdof7pfRmbNeR5JZu5/n1C6nxOoNVihDCxIJ6TPC8tL5M7DaGnYd389y6l6nyVAezHCGECQX9i5E+yT24vfs49lXu59l1C6hwt85jI0KI0KQEa2TpkpLKBscKthzKZcGG10lwxHF379uJsUcH7LOSkqIoLq4I2PuFGmlfQz6fl9LSYrxedxNWFTiqqqLrerDLaBJN1TaLxUZcXBKadvRrDVVVSEiIPO33CpkQBMgr3c78Da8SbYvi7t63E++IC8hnSUi0bKfbvoMHC3A4womIiEZRlCasLDAsFhWvt3WGYFO0zTAMqqoO43RWk5iYVre8sSEY9N3h+rLjOnBXr8lUeqqYu/ZFiqtLgl2SaIG8XneLCUBx+hRFISIiOmA9/ZAKQYD2MRnc3ft2XD4Xc9fOp7CqKNgliRZIArB1C+TvN+RCECA9qi339J6Kjs7ctfPZV1kQ7JKEEK1USIYgwFmRqczsMw2LauHZtQvYdXhPsEsSQrRCIRuCACnhSczsMw2Hxc7f1r3MjvL8YJckhGhlgnbZ3KlKDItnZp9p/G39Szy3fiHTetxKp7iOwS5LtBDf/FTAig1NczhlUI80LuiedvIXAgsWPE9ubg7V1ZWUlpbRpk1b/vSnp7DbHU1Smzh1Id0TPCLOEcs9vaeR4IjjhR9fYVNJTrBLEuK0bN26Ga/Xy9NPP8/ixe9RU1PNl19+EeyyBC2gJ3hEjD2Ke3pPZd76l1mw4TUmdRtDz6RuwS5LhLgLup96b60p5eZu5fnnFxIWFgZAVlYHysvLglyVgBbSEzwi0hbB3b2nkB7VhoUb32TNgfXBLkmIk9q/fx9ut4fMzPZ1yzZv3kSXLmcHsSpxRIsKQYBwaxjTe91GVkwGr276B6sKVge7JCFOKCdnC263i4KC/QAsXfoBERER9OrVJ8iVCWhBu8P1OSwO7uw5iQUbXufNLe/i8Xm4qO35wS5LiGPaunUL118/kiee+AOlpYfIyurAH//4VLDLErVaZAgC2DQbU3vcyqJNb/JO7hK8uodfpV8U7LKE+IWcnC2MHTuBGTPuC3Yp4hha3O5wfVbNym3dxtI7uQfvb/t/fJq/PNglCfELeXk5dO7cNdhliONosT3BIyyqhQlnj8aqWvh4x79x+zwMyxoi146KkPGvf8l/zqGsxYcggKZqjO06Eqtq5d+7vsSje7i+49UShEKIk2oVIQigKiqjO1+PVbXw5Z6v8eheRnYajqq06D1+IUQTC0gIPvnkk5SWljJnzpxAvF2jKYrCiOxrsGk2Ptv1Hzw+D2O6jghqTUKI0HbG3aRVq1axZMmSQNQSEIqicE3WlQxtfznfFq7mtU3/wKv7gl2WECJEnVEIlpWVMXfuXKZOnRqoegJCURSuan8513a4ijVFPzJ35ct4dG+wyxJChKAz2h1+9NFHmTlzJgUFoTno6eUZl2DVrLyXu5SqGieTu4/DplmDXZYQIoQ0OgTfe+890tLSOP/88/nggw9Oe/vGTIjSGDcmXUlCTBQLfljMwi2v8+CgaTisrW/4oqSkqGCX0KROp31FRSoWS8v6Qqyl1Xs6mqptqqoG5O++0bPNTZgwgeLiYjRNo7y8nOrqaq699loeeuihU9r+WLPNNZWkpCg++ekr/r7lHTKi2nJHz4mEW8Ob5bObg8w211Bh4S5SUzOasKLAktnmGufnv+fGzjbX6J7gq6++Wvf4gw8+4Pvvvz/lAAyG/qm9sWlWXtm4mGfWLeCuXpOJsjVPb1QIEbpabx/8GHomdWNqzwkUVx9k7tr5lDplPDdhHp988jHjxt3EhAk3s3t3Pm+88Vqz1/DQQ/ezZcsmANatW8Mtt9zIkZ3R0tJDjBw5nK+//i+TJo2lsrKyWWoKSAhef/31QT9H8FR1je/Enb1uo9xVwdNr58vcxsIUvF4v8+c/x7PPzufVV9+iurqaFSv+16w1bNq0Eaezhq5dzwGgd+++REVF8dVX/8HlcvLgg/dyww0jufDCSxgy5Ne8887iZqmr0ccEz1RzHxP8+TGl3RV7mbd+IRZFY3qvyZwVmdostTQFOSbYUP1jRZ7cb/DkfNUkdVk7X4S10wWn9NqysjKefvpJdu7cTlhYOOnpGWRldeDmm8c16rjZ999/y6JFC3C7XdTUOJk69U4uueQySktLeeaZP5ObuxVVVRk16hYGDbqYadMmUlJSQps2bbnooktYuvQDDMMgPj6BwYOv4JZbbuWFF56lpKSEsrIy8vN30K/fuQwZchWvv/4Ku3fnM2zYtUyceDsbN/7E888/g8vlpLq6mpEjb+b6628E4E9/eozU1DQmTrydPXt288AD9/D73/+Jzp278MQTf6BXrz78+tdX17Vj5coVvPLKS6SmphIfn8C99z4IwKFDJUyaNJYlSz457s8g6McEW7r0qLbM7DON59a9xDPrXmR6z9tIj24b7LJEK2QYBr/97b0MHXoNf/jDE+zZs5ubb76BZ555ocHrHnnkAfbu3XvM91iw4JW6SZkMw+CPf/w9r732D+Li4vB6vbhcTgzD4MEHZ3LDDSN57LE/UV5exujRNzBo0EVMnHg7P/zwHQ8//HvAHzKdO3dl2LBr6z4jJyeH6Oho5sz5Kx6Pm6uvvoKYmFjmzp1HYWEBkyaNZeLE20lPz2DevJfQNI3KykpGjBhWF4KTJ09j0qSxDBp0EX/4w+94+OH/o3PnLoB/9/fmm8c1aNfAgYOYO/cpoqKieOyxJ+qWx8cnYLFY2bUrn4yMzDP6+Z+MaUMQIC0ihXv73sHf1r3Es+teYlrPCXSMbX/yDUWLYe10wSn31prK6tXfoesGV1/tD5x27dKx2+106tSlwetmzz71gVZjY2OZO/cpLrvsCgYMGEhERCQ//PAdFouFIUOuAiAmJpb4+HjKysrIydlSF0bgH+Nw2LDrGrzntm05vPLKYqxWK7ruQ9NUbr31NlTVf9QsOjoagFWrVvDxxx9SUVGBrvvq1gMkJSVz5ZVDufPOycye/RS9evWu6+UWFxcRFxff4DM/+OA93G4XHo8HTdMarEtISKCo6ICEYFNLDEvw9wjXv8y89QuZ0n08XRM6Bbss0Yrk5Gyla9ej84ns2pVPQkIiUVENz3E71Z6goigsWvQma9eu5vPPP2X+/Of4xz/eZ9u2vAbBWllZSWnpIdq0aUtOzlYuueQywH98cPfuXWRldah7bUHBfsLDI0hJ8R8WysvLo0OHbMLD/aeS5eZupVOnLnzzzdd8+OE/mT37KRISEvnyyy9YuvToecKlpYf49ttvcDjCSE5OadAGu92O2+2ue75ixVe8++5bLFq0mOnTb2fDhvX06NGrbr3b7cJut5/CT/jMmD4EwT+l58w+05i3fiEvbniVCd3G0EtmshMBEhsbyw8/fI9hGDidTp5++qljDrJ6qj3BXbvyadcunXPPHUBKSiozZkxDURSSk5P59tuV6LqOruvMnfsUw4ffgNVqZdu2PLKzOwP+HllERCRW69Grp3Jzt/6ip9ilS9d6z/3r/a/rSkJCIsXFRbz88gtcdNGlAFRUVHDffXczceLtlJWVMn/+czz99LN175GV1ZHdu/NJTExk69YtPP30kzz99DwSExMZM2Ycb7zxKn/+s//1Pp+P/fv3kZXV9HOMm+oUmROJskUyo/cU2kW1ZdHGN/m+cG2wSxKtxGWXDUHTVEaPvp5HH51FQkJCg8A5XW+/vZibbx7BxIljePLJ2cye/SQAl1xyGWlpaYwbdxPjx48iNTWN226byp49u0hMTMLh8Pckk5KSycjIZOzYkSxatACA3NyGo1/7d5+PPvev78KQIVexdu1qbr31ZhYufJG0tDZ07twVp9PJAw/cw/XXj+CSSy7j6quvZe/e3axZ80Pde1x88aV8//23FBTs5+GH7+eRRx6rm4HvyiuHsn37NvLycgH46acfOfvsbkRGNv25vKb9dvh4nF4XC356nbzS7dzU+VoubBP6EzjJt8MNhfoVI/feO52bbx5Hv37nAua5YqSqqpI77riNl156rW7X/nh+//uHGTr0Gvr3P++4rwnUt8PSE/wZh8XOHT0m0C2xC2/nLOHzXf8NdkmilcnJ2fKLL0XMICIikunT72H//v0nfJ3b7aZXr94nDMBAkp7gcfh0H69vfps1RT9yZeZlXN3+ipAdrl96gg2Fek/w58zSEww0OU+wiWmqxq3njMau2fk0fzkur4sbsoeFbBAKIRpHQvAEVEXl5i434LDY+XLP17h8LkZ3uUHmLRGiFZEQPAlFUbi+49XYNTvL8r/A6XMx/uxRWFT50QnRGsi/5FOgKApXZ12Bw2JnybZ/4fa5mdRtrIxSLUQrIPt1p2Fw+sWM7nw9m0pymP/jKzi9zmCXJIQ4QxKCp2lQmwGMO/smtpXv5G/rXqbSUxXskoQQZ0BCsBHOTe3D7d3Hsa+qgLlrZHBWIVoyCcFG6p54NtN7TqLMVc7Ta+dzoLo42CUJIRpBQvAMZMd1YEafKbh9bp5e8wJ7KvYFuyQRgtauXc306bcHu4wmU1p6iOuuu+qUXlt/eP3jcbvdLW94fTNLj2rLvX2mYVWtPLN2AdvKdga7JCGa1ZYtm09pQIifD69/PDabrVmH15dTZAIgJSKZ+/rewXPrFzJv/ctM6nYL3RPPPvmGosl9V7CGVQU/nPyFjXB+Wn/OS+t7Wtv8/e+v8Nlny1BVlf79B3DHHXdz6FAxjz76MDU1NaiqwowZ99OtW3eKig7whz/87hfL66uurmbhwvmsX78Wt9tDVlYH/vCHJ47z6U1j69bNdOni/3t/7rm5uFxOiouLyMvLpUOHjjzxxF+xWCx89NEHXH75lXXbnWjKgcGDhzBp0lgmTZrS5PVLCAZInCOWe/tM4/kfF/HST39nbNeRnJvaJ9hliRCyatU3rFjxFQsXvoHFYuGRRx7gww/fp6KinIEDB3HzzeP49tuVbNiwnm7duvP//t/SYy4/wjAMHnroN5xzTncWLXoTRVE4eLDhselTHaj1TGzduoXrrhsBQF5eDjExsTz++JNYLBZuueVG8vN30rFjdoPh9U825YAMr99CRdoimNH7dhZseJ3XN79NtaeGS9oFd2h3szsvre9p99aaypo1PzB48JC6cf2GDr2GZcv+xZgxY5k16zfk5uYwcOAgbrhhJAD9+p3Lww8/8IvlR6xe/R1lZWXcdtvUumvaExOTGrzmdIbsr2/ixDEcOFB4zHUfffRZg6Hw/QOw+nuCeXm5vPrqW9hsNsA/ivWRYfnrD69/KlMOyPD6LZTD4uCOnhN5ZdNbvJe3lCpPFVe1v1wGXhAYhv6z5+DzeenZsxdvvvkuK1euYPnyz/jkk4955pkX6NHj2MuPyMnZSvfuPU/4t9XYnuArr5za8biDB4uxWCzExcWxf/8+IiIiSE31D9FfUVFBTU1N3TD79YfXP5UpB2R4/RbMqlm5rdstvLX1fT7J/4Iqbw0jsofJwAsm16dPf15/fRHDh1+Hpln45JOP6NOnH8899wwJCUmMHDma3r37MXHiGABeeOFZEhOTf7H8iMTEJFat+gav14vFYqGk5CAJCYkNXtPYnuCp2rJlU90w/Hl5OT8bkr/h5E71h9c/2ZQDzTm8voRgE9FUjTFdRxBuDePLPV9T7almbNeRaKp28o1Fq3TBBReSl5fDpEnj8Pm8nHvuAG644SbKyw/xu989xCeffIyqqjzyyGMA3HDDTTz22CO/WH7E4MFDWLt2NbfcMhKHw0G7duk8/vicZm1TTs7Wul3hYw/RfzQEjwyv36dPPy67bAhffvkFo0dfT7t26SQkJNChw9HAk+H1AyyYg44ahsG/d/2Hj3d8SreELk0y8IIMqtqQDKoaOk51eP2fTzkgw+u3IoqicGXmr7ip03VsKslh3vqF1Hhrgl2WEM3uRMPr159yQIbXbwKh0lNafWA9r29+m7SIFO7oOZFYe0xA3jdU2tdUpCfYcrWE4fWlJ9iM+qX0YlqPCRTXlPDXNS9woKoo2CUJYXoSgs3s7ITO3NPbf73xX9e+wM7y3cEuSQhTkxAMgozodtzX9w4cmoO/rVvAppKtwS5JCNOSEAyS5PAk7ut7J8nhSby44TW+LVgd7JKEMCUJwSCKsUdxT5+pdIzN4o0t7/LZrv8QpO+phDAtCcEgC6u9zK5vck+Wbl/G+3kfoxut85tCIUKRXDESAqyqhVvPGU20PYr/7FnBYXcFY8++CatM6ylEk5N/ZSFCVVRu6DiMGFs0H27/hEpPFZO7jyPMcuZDHQkhjk92h0OIoihcnnEJ47reRF7ZDp5d+yLlrtZ7ErRomQzD4L777g52GQEjPcEQdF5aXyJtESz86Q3+uuZ57uw5kZSI5GCXJUzMMAzmz/8bqqrhdNbQpk0bAF555SWqqqqorq7innt+E5BBWpub9ARD1DkJXZjRZwoun4u/rnlB5i4RAPzxj7/nk08+/sXypp7M6dNP/0Xnzmczdep0oqKi6d69J5999inJySncdddMHnzwkRYZgCAhGNIyo9O5v990ImzhPLf+ZdYcWB/skoRJbdy4oW5YrC1bNtG9e082bFhH3779g1zZmZPd4RCXGJbAfX3v5KUNr/PKprc45CxjcPrFMlJ1C/Pii/P473+XExMTS0JCIoMGXURqahrz5/8NXddJT8/E4XCwY8d2Dh06RMeOHfn97/+IzWZn3ry5fPPNChITE9F1nd69TzxdwLEmcyopOXjMSZtOZTIngPPPH8T8+X8jPT2ToqIDpKamcfHFl/Lii/NISUnhggsuomfP3k3142tSEoItQKQ1grt6TeaNLe/y4fZPOOg8xMjs4TJA6yk4vPIbyld81STvHTPoIqIHnnwOmRUrvmLDhvW88ca71NTUMGnSLQwadBEAe/bs5sMP/8XWrTksX/45Cxa8iq7r3H33VFat+gbDMMjNzeHNN9+loqKCW28ddcLPOt5kTuXlZY2azOmIQYMuqqt5ypQ7AejffwD9+w843R9byJEQbCGsmpVbzxlNvCOOz3f/lzJnGRPOGYPD0vRzMIgzs3r1d/zqV4OxWq1YrVYuvPDiunXt2mUQGRlFr159iI6O4f3332X37nz27t1DTU0NW7Zs4uKLL62bx2PAgBOH7vEmcxo1aswxJ2062WROZiAh2IKoisq1Ha8iISyed3KW8My6F5nWYwJJRJ18Y5OKHnjBKfXWmpKqqscdO/PIREIrVvyPhQsXcOONo7jqqmsoKyvDMAwURaH+lZT1Z3k7luNN5nS8SZtONpnToEH9Gtlqv2+/XXtG2zcHCcEW6MI2A4izx7Bo02L+vHoej0TdhUOCMGT163ceixe/znXXjcDtdrFy5Qqyszs3eM3q1d/zq18NZujQa9i3by/r1q2hf/9z6dfvXN566w2GD78ep9PJd9+tolu3Hsf9rONN5nS8SZtONpnTihWtf2APCcEWqltiV2b2mcr8H1/ld8v/wm3njKVzfNPPzCVO38CBg9i4cQMTJowhOjqaxMQkbLaGhzGGDbuOxx57mC+++DcWi5Xu3Xuwf/9+rr76WrZs2cy4cTcRH59AZmbWCT/reJM5lZQcPOakTSebzKk+wzD4zW9m8Ne//o3HHnuEKVOmY7FoPPXUn/jtbx8lLi7uzH9YQXBGw+vPmzePZcuWAXDxxRfzwAMPnPK2Zhxevykccpby0sbX2FdxgJs6XcugNi3/QPXPtfTh9Tdu3MCePbv59a+vxuv1MmXKBH7720fp2DEbCO3h9X9+krSu69x774Ps3p3PokULcLlc/OY3D5GYmHjM7Vv18PorV65kxYoVLFmyhA8//JBNmzbx+eefN/btRCPFO+J4fPD9dInP5h85H/DP3I9kFJoQk56eweef/5vx40czceIYLrvsiroADHXHOkkaIDY2jo0bf+KGG24iMTGRjRt/Yvr02/F6vUGu+PQ1enc4KSmJWbNmYbPZAOjQocMxZ5ESTS/cGsbU7reyZPu/+M+eFRyoKWbiOWNk8IUQER0dw9NPPxfsMhpl48YNjB49FvCfJD106DVUVFTw5JOz+b//m80HH7xH//7n0a1b95OevxiqGh2C2dlH/yfLz89n2bJl/OMf/zjl7RvTbT0TSUmt+4uD1JRYpqWMoeO2dF5Z+zbPrJ/PgxdOIyUyKdilBcTp/P6KilQslpZ1MVSo1jto0IW8+OJzZGRkUlxcRFxcLI8//n/cddc9tG3bjlWrVrB160a6deuBqipYLL/82TdV21RVDci/6zOecjMvL48pU6Zw1113cd11153ydnJMMHB+3r6cQ9tYuPENVEVlcvdxdIxtH8TqzlxLPyZ4MqF8TPBU7dqVz7PP/pVLL72MYcOurVveqo8JAqxZs4Zbb72V++6777QCUDStzvEdub/fdMKtYfxt3UuskvlLRBPLyMjk6aefaxCALUWjQ7CgoIA777yTv/zlLwwdOjSQNYkASA5P4v6+0+kY2543t7zLh9s+kS9MhDiGRh8TXLRoES6Xizlz5tQtGzVqFKNHjw5IYeLMhVvDubPnJN7L+4jPd/+XfVUFTDh7NOHW8GCXJkTIOONjgo0lxwQD51Ta9/W+b3kvdylxjlimdB/PWZGpzVTdmZNjgi1Xqz8mKFqOC9sMYEbvKbh9bv68Zh7rin4KdklChAQJQRPpEJvJg/3vpk1EKgs3vsHS7cta7XFCmb+5dQvk71dC0GRi7THM6DOVgWnn8tmu/zB/w6tUe6qDXVZAWSw2qqoOSxC2UoZhUFV1GIvFFpD3k2OCrUBj29dSjhOebvt8Pi+lpcV4ve4mrCpw/ENttc4eeVO1zWKxEReXhKYd/W63sccEJQRbgTNp3/ayfBZufAOn18moztdzXlroXfokv7+WqznbJl+MiEbpEJvJrP4zSI9uy9+3vMNbW/+J2+cJdllCNBsJQUGMPZq7e93OFRmX8s3+7/nrmucpqj4Y7LKEaBYSggIATdUY3uHXTOsxgUPOUp784Vk5jUaYgoSgaKBbYldm9b+H1IgUFm58g3/mfoRXb3ljxAlxqiQExS8khMUxs89ULm07iP/sXcHTa+bL7rFotSQExTFZVAsjOl3D5G5jKa45yJwfnmFVwWo59060OhKC4oR6JXfnoXNnkh7Vlje3vMsrmxZT7akJdllCBIyEoDipOEcsd/e+nWFZV7K+eCN/+n4u28p2BrssIQJCQlCcElVRuTLzV9zX9w40VeOZtS/y8Y5/49N9wS5NiDMiIShOS2Z0Or/tP4NzU/vwaf5y/rxmHvsrC4NdlhCNJiEoTpvD4mDc2TdxW7exlDrLePKHZ/l3/pfSKxQtUqNHlhaid3J3Osa2552cJXy041N+LN7EuLNHkhqREuzShDhl0hMUZyTKFslt3ccy8ZwxHHSW8MQPz/L5rv+22nEKResjPUEREH1TepIdl8XbOUv4cPsnrC36kdFdbiA9qm2wSxPihKQnKAIm2hbF5G5jmXjOzZS5DvPUD8/xft7HOL2uYJcmxHEFbTzBF95bR3ll8wx66XBYcTpb7/BQodg+Ly4KrGs5aMnBakTQznMeMb70Rr1XKLYvkFpF+wwDFR+q4UNF998bOmF2BbfTjYqOgo5iGCgY9R7r/uf1Hx/zdcfYpnY7AAWD8KhIfn3r+NMuPWi7wzv2H6ak3Nksn6VqKrqv9R6jCt32dcXhSMGVup4d9i/RKtKwFXdD9USc1ruEbvsCoynap+HDjhsbHuy4sdfeWxUvFrxY8WLFV/fYUvvciheL4sVS+9iKFw0fGjoaOip6g+dHHqtK8C+ntChJwOmHoIws3QqEevt8uo/le75i2c4v0DEY3O4irsj8FXbt1OaICPX2nanjtc/wujFcVRjOSgxXpf++/uPae9w1GB4nhrsGjtyfzsg/igoWG4rFBpoVxWIHixVFs4HFBpoFRbWAZgHVgqJqtY+1nz23oGj+ZagaimohKjaSikq3/7WKiqKooCr+z6x/U2vX1d0UUNWj29R7Xd362nsFBRQF1aKRmJp02j9/+WJENDlN1bgi41L6p/Tmw+2f8OmuL/m2cA3DO/ya/im9URQl2CU2K8NdjV5ZilFdhlFdRtk2J86iA/7nNeXo1WUY1eXgOcGeksWGYo9EcUSi2MJRIxPA6kCxhaFYHWALQ7GGodgcYA07utxqrw23emGnNl0MRCVF4Wym/8AUtXF/R9ITbAVaWvu2l+Xzz7yl7K7YR/voDEZ0GkZm9PGPF7ak9hmGAa4q9IqD6BXFGJUl6BUH/feVB9ErDoL7GANQWOwo4bGo4TEo4TEo4bEojqjaW23YHQk9e4S/19YCtIQ5RiQEW4GW2D7d0Pm2YA0fbV9GhaeSXkndGZY1hNSI5F+8NhTbZ3hd6OVF6OUF6GWF6OUH/I/LD4CrquGLrQ7UqESUyATUyET/44i42tCLJSm9LSWHW+fAtS0hBGV3WASFqqgMPKs/fZK7s3zP1yzf/T9+LN7I+Wn9uKr95cQ5YoNdIgCGz4NeVoBesgffob3opXvRD+3DqDrU4HVKRDxqTArWDuehRqegRCfWBR628BPu8qv2MCC0Qt5MJARFUDmbiW9EAAAZjUlEQVQsDoa2v5yL2pzPv/O/5Ot9q/j+wDoubjOQyzMuIcp2+v+zN5Zecxi9eCe+kt3oh/b6b2WFYNReE61aUOPOQkvrjBqbhhqTihqTghqTimK1N1udIrAkBEVIiLJFMqLTNVzabhD/2vk5X+75mq/2rWLQWedxU8RQQAvo5xnOSnwH8/EV5/uDr3hng96dEpWIGtcWW0Zv1IR2qPFtUWNSmvRLBBEc8hsVISUhLJ5xZ9/EkIxL+WzXf/nfvpV8vf9bBqT25fKMS0kMiz/t9zR0H3rJbnwFufiKtuM7mI9xuKhuvRKdgpaajZbUHjWpPVpCOxRbeCCbJUKYfDHSCrTm9pXUHOLrom/4z46V6Bj0Te7Jpe0GkRHd7rjbGB6XP+wKc/EV5uE7sA1qL91TIhNqwy4TLSkLLTEDxX56J28HWmv+/ckXI0KcoYSweCb3u5lLUi9i+e6vWLn/e344sI6smEwubTeInonnoLiq8B3I8/f0CvPQD+6qPY6noCa0xdp5EFpqJ7TUTqgRccFukggxEoKiRYi1x3BD9jB+nTmYb/P/y3/3f8eijW8S64PzSyvpW1FDJBpaUha2nr+uDb2OslsrTkpCUIQ0Q9dxFe7EvWU9vsJc9MJc+leX0RfYEhPFNwkxLEuM5N+JUfRIPJsL2gygS3w2qiIDJIlTIyEoQorhdeMr3nn0eF5hHpW1U3wqEfFoaV38X2KkdeK8uDYMUFQKqw7wzf7v+a5wDesPbiLeEceAtH70T+lNcnhikFskQp18MdIKtOT2Gc5KfAe2HQ294p11F/+rcWehpXYiNrsH1RHpKJEJJzzp2KN72VC8iZX7v2draR4AGVHt6Jfai77JPYmxRzdLm05XS/79nUxL+GJEQrAVaCntMwwDo6K4rofnO5CLXrrfv1LR/N/YpmZjSe2MlpqN4vD/QTemfaXOMtYU/cjqwnXsqdyPgkJ2bBa9k3vQPbFryFyRAi3n99cYEoInICEYOKHavrrz8wrz6np6Rk25f6UtDC0l279rm5KNltzeP6rJMZxp+wqrilh9YD1ritZTVH0QgPSoNvRIPIfuiWfTJjItqCPZhOrvLxAkBE9AQjBwQqF9db28oh34infWXoWRDz7/6OFKZELtN7b+4FPj2vjHiTsFgWqfYRgcqC5iQ/FmNhzcTP7h3RgYxNlj6RKfTZf4bDrHdWzWS/UgNH5/TaUlhKB8MSIape462yOhV7QDw1XpX6lZURMzsHa9GC2lI1pKNmrk6V/pEWiKopAakUJqRApXZF7KYXcFGw9uYVPJVtYXb2RVwQ8AtIlMo0tcNtlxWWTFZBJhldNsWjMJQXFChu5DLy9EL9nj37Ut2Y1esufobq2ioMa2wZLZGzUpCy05CzW+TYu4xjbaFsXAs85l4Fnnohs6eyr2seVQHjmH8vjf3m9YvucrAFLCk8mKySArJpOsmAxSwpNMNxBsaxb6f6miWRi6jlF5EL1sv3/oqNL9/qGjDu0FX+0kQKqGGtcGrV03tPh01MQMtKRM/4jFLZyqqGREtyMjuh1XZv4Kt8/NrsN72FG+ix3lu9hQvKmupxhmcdA28izaRp1Fu8g2tI06i9TwZDQ1sIM8iOYhIWgihmFgOCswKorRDxfVhZ1eVoh+uBB8Rwf2VMKiUePbYj3nMrT4dqgJ6aixaSiaOf5kbJqN7LgOZMd1AI4cTyxmR3k+uyr2srdiPyv2fYdH9/8HYVEtnBWRQkp4CqkRyaSGJ5EakUxiWAKWFtArNjP57bQihqH7J+CpKkWvLKkNu2L/MO8VtUO7158DWFFQopNRY9KwtuuOFndW3Th5R05PEX7+44nJpEYkM5BzAf/o2Aeqi9lTsY+9FfvZV1nAtrId/HBgbd12qqKSGBZPclgSCWHxJDjijt474gm3hgWrSaKWhGALYPg8tTONVfjvq8vQq8owqksxqkrZ567AXX4Qo7oMdF/Dja0O1Kgk1OhktDbnoEYn+Yd3j0pGjUlG0azBaVQroCoqaREppEWkcG5qn7rlTq+LoupiCquLOFBVRGF1EcU1JWwr24HT13Ai+jCLg+TIRCK1SGJsUUTbo4mxRRNjjyKm9nG0LUp2tZvQGYXgxx9/zPz58/F6vYwfP54xY8YEqq5Wx/B5MdzVtVMiVmO4a/xTI7prjj53VR0Nunr3x511zBqGGhGLNTbRP9px7bwVSngsalQialQS2CPkIH4zc1jspEe3JT26bYPlhmFQ7a2hxHmIkprSuvtK/TDFlaXsqdhHhbsSg1+eOhZuCSPCGk6ENaL2vvZmiSDS5l8ebgnDYbHj0Ow4LA7smh27ZpPrqE+i0SF44MAB5s6dywcffIDNZmPUqFGcd955dOzYMZD1BZRhGP6eku4FnxdD99Y+92H4vHXL0X1H1x15ndeN4XWDz4PhdR19/vN7X/3nrrqQq3+87bgsdpSwozOM+XdLoxosU+yRqOGxKBGxdV9ItObzzFoTRVHqwis96mhA1v/9+XQfFZ5KDrsqKHcfptx1mHJ3BZXuKqo8VVR5qjnsrqCg6gBVnipctedhnohds+HQHLUB6cBu8YejTbViVa1YNStW1eJ/rtmwqZbaZf6brfaxRbWgKRoWVUNTNLQj94pau05Frbe+pfzn2+gQXLlyJQMGDCA21n/50ZAhQ/j000+ZPn36KW3vXLkYX2UZGDoYBhg6Rr3HGDroOmCAfmTdz9bXPjd+9toG63QflYYPw+s9OldEoFhs/qscNKt/CkSLHaV2meKI8q+3hfmHc7I6UGzhKLYw/5ywdbfwo3PEmuRLB3F8mqoRa48h1h5zSq/36F6qPdVU1d5cPhdOnwun11l77/Iv8zqp8blweV04fU6qPFV4dA8enxeP7sGte/D4PMfshTaWqqhYVA2V2mBEQUVFQ0Ez/DcV/00zFP96A9S6xwoq/GyZ/7ECtdsffR4VHsvNI+447Tob/a+uqKiIpKSjs70nJyezYcOGU3+Dsr0oVWX+GeZVFag/C71/9nnFojacuf7Ia2tnn2/4WDs6I33dTPUKimbx31QLHHmsWVBUzR86dcus/vdo8Jra9aqGYrWjWu0oVltt0NlC6n+6pKSoYJfQpKR9J3LigWINnw9fjRNfTTU+pwvd7UZ3+e99rvqPXficTjyuGjxOJ15XDV6XC5/bjc/jRvd6Mbze2nv/3pPh9YG3tpOh6+D1gc+/TNF18OkoPh2lGa4OsycnwYjT367RIajreoMQMAzjtELBcdWDzXLZnAEkBmJ3UQdctTfctbfQ0Np3h83ePkPX0aur8VVW4quq9N9XVqIfeVxTg+F04nPWoNfUoDudtff+x4bLddz3Pi5NQ7Va/f/pW60oFguq5UjHQUOxWP17Lw4NRdNQLBb/rW69BcWiEREVTo3L5+/UqPU6OGq9To+qoqj+5UdfV+81qgqqUre8/nvVf0/N3rgJ6RsdgqmpqaxevbrueXFxMcnJv5w4WwjxS4au+wOsvIzSPW7KdxXgLS/DV16Gt7wcb3k5vorD/rCrrvYfBjoWVUUNC/PfHGGoDgdaZBS2pCT/MrujwTrFYUe12lBsNlSbzR9yNmu9xzZ/+FkCc2imua8dboxGt3TgwIE899xzHDp0iLCwMD777DMef/zxxr6dEK2K7vHgPVSCp6QE78GDeA4dPPq4pARveZl/t/Fn1LAwtJgYLDGx2NtloEVGHr1F+O/ViCPLIlDDTjyxuzi5RodgSkoKM2fOZNy4cXg8HkaMGEGPHj0CWZsQIc3wevEcLMZdWIj7QCGeA4X+x0UH8JWVNXyxomCJi8eakEBY585Y4+Lrwi4x8ywqDCuW6BhUu0zi3tzOqM87bNgwhg0bFqhahAhJuseDp7AA1969uPbtxb1/nz/0iotrz0rw0yKjsKamEnF2N6xJSVgTErEkJGBNSMASG3fcXczopChcrfiYZ6iTczKEqGUYBt7SUlx7duPet9cfenv34D5QWLfrqlgsWFNSsbdLJ6rfudhSUrGmpmJLTkGLlEsNWyIJQWFa3vJynLt24srPx5m/E2f+TnyHD9ettyQkYG/TlshevbG1bYu9TTtsKSkB+9JAhAb5bQpT0F0unDu2U7Njuz/0du3Ee+iQf6WiYEtLI6Jbd+yZ7XG0y8DWpg1auAymagYSgqJV8lYcpiYvD2deLtV5ubh276o7fmdNSSGsYyccme2xZ2biSM9AdbT8MRFF40gIihbPMAw8B4upyc2lZlsuNXm5eAoLAf8xPEdWB+KvvIqwTp1wZHVAC48IcsUilEgIihbH0HVce/dQk5dLTV4eNXm5+Mr9p6So4eGEdcwm5oILCcvuhD0jE9Uqw4WJ45MQFCFPd7upzs2hJjeHmm15OLdvQ6+pAcASH094ly6EdexEWKdO2NLOqr0WXYhTIyEoQo6vqoqabXm1Pb1c8nbl+y/QB2xntSHq3PMIy+5EWHYnrAmJQa5WtHQSgiLoPCUltcfy/MHn3rfXv0LTcGS256xhQzHaZBLWMVvOxRMBJyEomtWR43nObXm1vb08vKX+U1VUhwNHh45E9T+XsGz/t7eq3d7qR5ERwSUhKJqU7nLh3LnDv2t75Hie0z9dgCUujrCO2Tg6ZhPWMRt723YomsylIZqXhKAIGEPXcRcW4Ny5E2f+Dpw7duDas9t/fp6i+I/nnXc+YdnZhGV3whKfICOgiKCTEBSN4r/O9pA/8HbuwJm/E1f+zrpenupwYM9s7z8/L7sTjg5yfp4ITRKC4qQMnw93YSGuPbtw7dmDa+8eXHt2H73OVtP8AwqcPxBHZhaO9lnYUlPlVBXRIkgIijpHenfuggLcBfv9Ybd7N+79++pOUVEsFmxntSGiWw/sGRk42nfA3q4tqrVxQ5sLEWwSgiaku1z+wUALCnAX+gPPXVCA+0Bhg/kotKgo7O3Sib1sMPa26djbtcOWmiajqIhWRf6aWyHD58N76BCeg8V4iov99wcP1t4XNxguCvxDRtlS04jp1BlbWhq2VP9Ni46WLy5Eqych2IIYuu6fbaysHG95Kd7SUrxlZZQ7q6gsLPI/Ly/zh1z9iXlUFWt8ApbERCJ69MKamIg1KdkfeCmpMqS7MDUJwSAwfD7/lIg1Nfhqqo8+PjKFYkVF7bSKFQ0eH2/WMWtMNGp0DJbYOOwZGVhi47AmJGBNTMKamIglLl7OvxPiOCQE6zEMA3z+iaQNr7ducmnD68XwuNHdbgz30fu6ZS43uudn69xudLcLvbo26I7MCVtTg+E+yZzFmoYWFYUWGYUWGYkjPR219rEWGYklJhZLbCyWuDi06BhSzoqXKyqEaKSghWDZf77EU1EBhoGh62DooB99bOiG/yTbeo/96+o/rl1nHGNd7XPD56NAMXDXuDB8PgyfF7y+2oDz+pfV3h9rCsTToiiodrt//la7f15XNSwcLTISa2ISWni9+WHDwlHDHP71R+aNjYjAEhWFYnfIsTghmknQQrD035/gKio+uuDIzPS19/4Z6JV6s9Qrv5x5vsEs9MeY4V5RUCwWtDAHFnsYimZBsWhgsaBoWu1z/zJFs4Cm1T63+NdpWt06xVZvwuq6e3vtvRXVZvdvL+ElRIsStBDMePwJ/+GtI2HVhOEhF+ALIY4naCGoWq3o+i8P8gshRHOS65qEEKYmISiEMDUJQSGEqUkICiFMTUJQCGFqEoJCCFOTEBRCmJqEoBDC1CQEhRCmJiEohDA1CUEhhKlJCAohTE1CUAhhahKCQghTkxAUQpiahKAQwtQkBIUQpiYhKIQwNQlBIYSpSQgKIUxNQlAIYWqNDsE1a9YwYsQIhg8fzvjx49m3b18g6xJCiGbR6BC8//77mT17NkuXLmXYsGHMnj07kHUJIUSzaFQIut1uZsyYQZcuXQDo3LkzBQUFAS1MCCGaQ6NC0GazMXz4cAB0XWfevHkMHjw4oIUJIURzUAzDME70gmXLlvHEE080WJaVlcVrr72G2+1m1qxZlJeX8+KLL2K1Wpu0WCGECLSThuDxVFVVMW3aNGJjY/nLX/6CzWY7re1LSirR9UZ99GlLSoqiuLiiWT4rGKR9LVtrbl9ztk1VFRISIk9/u8Z+4P33309GRgbPPPPMaQegEEKECktjNtq8eTPLly+nY8eOXHfddQAkJyfz8ssvB7Q4IYRoao0KwbPPPpucnJxA1yKEEM1OrhgRQpiahKAQwtQkBIUQpiYhKIQwNQlBIYSpSQgKIUxNQlAIYWoSgkIIU5MQFEKYmoSgEMLUJASFEKYmISiEMDUJQSGEqUkICiFMTUJQCGFqEoJCCFOTEBRCmJqEoBDC1CQEhRCmJiEohDA1CUEhhKlJCAohTE1CUAhhahKCQghTkxAUQpiahKAQwtQkBIUQpiYhKIQwNQlBIYSpSQgKIUxNQlAIYWoSgkIIU5MQFEKYmoSgEMLUJASFEKYmISiEMDUJQSGEqUkICiFMTUJQCGFqEoJCCFOTEBRCmJqEoBDC1CQEhRCmJiEohDA1CUEhhKlJCAohTO2MQ3Dz5s1069YtELUIIUSzO6MQrKmp4fHHH8fj8QSqHiGEaFZnFIJz5sxh/PjxgapFCCGanaWxGy5fvhyn08mVV17ZqO0TEiIb+9GNkpQU1ayf19ykfS1ba25fqLftpCG4bNkynnjiiQbLsrKyqKys5LXXXmv0B5eUVKLrRqO3Px1JSVEUF1c0y2cFg7SvZWvN7WvOtqmq0qjOlWIYxmkn0XvvvceCBQuIiIgAYOvWrXTp0oXFixcTGXlqRUgIBo60r2Vrze1rCSHYqN3hG2+8kRtvvLHueefOnVm6dGlj3koIIYJKzhMUQphaQEIwJycnEG8jhBDNTnqCQghTkxAUQpiahKAQwtQkBIUQpiYhKIQwNQlBIYSpSQgKIUxNQlAIYWoSgkIIU5MQFEKYmoSgEMLUJASFEKYmISiEMDUJQSGEqUkICiFMTUJQCGFqEoJCCFOTEBRCmJqEoBDC1CQEhRCm1qgpNwNBVZVW/XnNTdrXsrXm9jVX2xr7OY2afF0IIVoL2R0WQpiahKAQwtQkBIUQpiYhKIQwNQlBIYSpSQgKIUxNQlAIYWoSgkIIU5MQFEKYmulCcPPmzXTr1i3YZQTUmjVrGDFiBMOHD2f8+PHs27cv2CUFxMcff8xVV13FFVdcweLFi4NdTsDNmzePoUOHMnToUJ566qlgl9MknnzySWbNmhXsMk7IVCFYU1PD448/jsfjCXYpAXX//fcze/Zsli5dyrBhw5g9e3awSzpjBw4cYO7cubz11lt8+OGHvPPOO2zbti3YZQXMypUrWbFiBUuWLOHDDz9k06ZNfP7558EuK6BWrVrFkiVLgl3GSZkqBOfMmcP48eODXUZAud1uZsyYQZcuXQDo3LkzBQUFQa7qzK1cuZIBAwYQGxtLeHg4Q4YM4dNPPw12WQGTlJTErFmzsNlsWK1WOnTowP79+4NdVsCUlZUxd+5cpk6dGuxSTso0Ibh8+XKcTidXXnllsEsJKJvNxvDhwwHQdZ158+YxePDgIFd15oqKikhKSqp7npyczIEDB4JYUWBlZ2fTq1cvAPLz81m2bBkXX3xxkKsKnEcffZSZM2cSHR0d7FJOKmhDaTWVZcuW8cQTTzRYlpWVRWVlJa+99lpwigqQ47Xttddew+12M2vWLLxeL1OmTAlShYGj6zqKcnRoJMMwGjxvLfLy8pgyZQoPPPAAmZmZwS4nIN577z3S0tI4//zz+eCDD4JdzkmZYiit9957jwULFhAREQHA1q1b6dKlC4sXLyYyMjLI1Z25qqoqpk2bRmxsLH/5y1+w2WzBLumMLVmyhNWrV/PHP/4RgOeffx7DMJg+fXqQKwucNWvWcPfdd/PQQw8xdOjQYJcTMBMmTKC4uBhN0ygvL6e6upprr72Whx56KNilHZthQp06dQp2CQE1bdo045FHHjF8Pl+wSwmYwsJC49JLLzVKSkqM6upq45prrjF+/PHHYJcVMPv37zfOO+88Y+XKlcEupUm9//77xoMPPhjsMk6o1e0Om83mzZtZvnw5HTt25LrrrgP8x89efvnlIFd2ZlJSUpg5cybjxo3D4/EwYsQIevToEeyyAmbRokW4XC7mzJlTt2zUqFGMHj06iFWZkyl2h4UQ4nhM8+2wEEIci4SgEMLUJASFEKYmISiEMDUJQSGEqUkICiFMTUJQCGFqEoIiZOzevZtevXpRVFRUt+yjjz5i0KBBrWJkHBGaJARFyEhPT+eSSy7h9ddfB2DdunU8/vjjvPDCC6SlpQW5OtFaSQiKkDJ58mTeeecdcnNzueuuu3jsscda1eVyIvRICIqQcs4559CjRw9GjhzJqFGjuOqqq4JdkmjlJARFSNF1HVVVURSF22+/vW55RUUFI0aMoHfv3uTm5gaxQtHaSAiKkDJnzhwqKirIzMzko48+qlvucDh46aWXGDJkSBCrE62RhKAIGW+//TZffPEFzz//PJMnT2bRokUcGeTIarUSHx8f5ApFayQhKELCypUrmTt3LvPnzycxMZEhQ4bg8XhYvnx5sEsTrZyEoAi67du3M3PmTJ566ik6d+4MgKZpTJgwocUPDitCnwyqKlqUWbNmMXHiRDp16hTsUkQrISEoWozJkyezZcsW2rRpw0033cT1118f7JJEKyAhKIQwNTkmKIQwNQlBIYSpSQgKIUxNQlAIYWoSgkIIU5MQFEKYmoSgEMLUJASFEKb2/wFLIDW7mfX0FwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Compute the range of values that we wish to explore\n",
    "xs = np.arange(-5.0, 5.001, 1.0/128.0).astype(np.float32)\n",
    "\n",
    "# Build an array of logit vector, where each logit vector is for a 2-class problem with the values [0, x[i]]\n",
    "logits = np.stack([np.zeros_like(xs), xs], axis=1)\n",
    "\n",
    "# Use softmax to compute predicted probabilities:\n",
    "clf_q = f_softmax(logits)\n",
    "\n",
    "# Compute negative log loss of softmax:\n",
    "clf_loss = f_clf_loss(logits)\n",
    "\n",
    "# Compute gradient of negative log loss of softmax with respect to the logits:\n",
    "clf_loss_grad = f_clf_loss_grad(logits, np.ones_like(xs))\n",
    "\n",
    "plt.figure(figsize=(5, 5))\n",
    "plt.xlim(-5.0, 5.0)\n",
    "plt.ylim(-5.0, 5.0)\n",
    "line_p, = plt.plot(xs, np.ones_like(xs), label=r'$p$')\n",
    "line_q, = plt.plot(xs, clf_q[:, 1], label=r'$q = softmax(X)$')\n",
    "line_loss, = plt.plot(xs, clf_loss[:, 1], label=r'loss $c =-ln(q)$')\n",
    "line_loss_grad, = plt.plot(xs, clf_loss_grad[:, 1], label=r'grad loss $\\frac{dc}{dX_1}$')\n",
    "plt.legend(handles=[line_p, line_q, line_loss, line_loss_grad])\n",
    "plt.xlabel(r'$X_1$')\n",
    "plt.show()\n",
    "\n",
    "if SAVE_PLOTS:\n",
    "    plt.figure(figsize=(5, 5))\n",
    "    plt.xlim(-5.0, 5.0)\n",
    "    plt.ylim(-5.0, 5.0)\n",
    "    line_p, = plt.plot(xs, np.ones_like(xs), label=r'$p$')\n",
    "    line_q, = plt.plot(xs, clf_q[:, 1], label=r'$q = softmax(X)$')\n",
    "    plt.legend(handles=[line_p, line_q])\n",
    "    plt.xlabel(r'$X_1$')\n",
    "    plt.savefig('clf_loss_0.png', dpi=600)\n",
    "    plt.close()\n",
    "\n",
    "    plt.figure(figsize=(5, 5))\n",
    "    plt.xlim(-5.0, 5.0)\n",
    "    plt.ylim(-5.0, 5.0)\n",
    "    line_p, = plt.plot(xs, np.ones_like(xs), label=r'$p$')\n",
    "    line_q, = plt.plot(xs, clf_q[:, 1], label=r'$q = softmax(X)$')\n",
    "    line_loss, = plt.plot(xs, clf_loss[:, 1], label=r'loss $c =-ln(q)$')\n",
    "    plt.legend(handles=[line_p, line_q, line_loss])\n",
    "    plt.xlabel(r'$X_1$')\n",
    "    plt.savefig('clf_loss_1.png', dpi=600)\n",
    "    plt.close()\n",
    "\n",
    "    plt.figure(figsize=(5, 5))\n",
    "    plt.xlim(-5.0, 5.0)\n",
    "    plt.ylim(-5.0, 5.0)\n",
    "    line_p, = plt.plot(xs, np.ones_like(xs), label=r'$p$')\n",
    "    line_q, = plt.plot(xs, clf_q[:, 1], label=r'$q = softmax(X)$')\n",
    "    line_loss, = plt.plot(xs, clf_loss[:, 1], label=r'loss $c =-ln(q)$')\n",
    "    line_loss_grad, = plt.plot(xs, clf_loss_grad[:, 1], label=r'grad loss $\\frac{dc}{dX_1}$')\n",
    "    plt.legend(handles=[line_p, line_q, line_loss, line_loss_grad])\n",
    "    plt.xlabel(r'$X_1$')\n",
    "    plt.savefig('clf_loss_2.png', dpi=600)\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Probability regression: sigmoid non-linearity with binary cross-entropy\n",
    "\n",
    "First define functions for computing sigmoid and binary cross-entropy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sigmoid definition\n",
    "def f_sigmoid(x):\n",
    "    return 1.0 / (1.0 + np.exp(-x))\n",
    "\n",
    "def f_prob_regr_loss(q, p):\n",
    "    return \n",
    "\n",
    "# Binary cross-entropy of sigmoid\n",
    "def f_prob_regr_loss(logits, target):\n",
    "    t_logits = torch.tensor(logits, requires_grad=True)\n",
    "    t_target = torch.tensor(target, requires_grad=True)\n",
    "    loss = -F.binary_cross_entropy_with_logits(t_logits, t_target)\n",
    "    return loss\n",
    "\n",
    "# Gadient of binary cross-entropy of sigmoid\n",
    "def f_prob_regr_loss_grad(logits, target, axis=0):\n",
    "    t_logits = torch.tensor(logits, requires_grad=True)\n",
    "    t_target = torch.tensor(target)\n",
    "    # Compute binary cross-entropy of sigmoid\n",
    "    loss = -F.binary_cross_entropy_with_logits(t_logits, t_target)\n",
    "    # Sum and compute gradient\n",
    "    loss.sum().backward()\n",
    "    return t_logits.grad.detach().numpy()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUEAAAFGCAYAAAD0Cy2JAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8FPX9x/HXzB7Z3HdIhHAEEg65xQPEmwqKiHiCB4ioYMWrra0/66+/Wg/UnxVbUYuKYhVttaLUKlbEX2sRREAUJeTgPnKf5Nh75vfHhCVBjpBsMsnu5/l47GNnZ2dnP7Nh33xnZuf7VXRd1xFCiDClml2AEEKYSUJQCBHWJASFEGFNQlAIEdYkBIUQYU1CUAgR1iQEhRBhTUJQCBHWJASFEGFNQlAIEdYkBIUQYU1CUAgR1iQEhRBhzWrWG1dXN6BpndOBTXJyDJWV9Z3yXmaQ7eveQnn7OnPbVFUhMTH6pF9nWghqmt5pIXjo/UKZbF/3Fsrb19W3TXaHhRBhTUJQCBHWTNsd7u50Xae+vhansx5N85taS1mZiqZpptbQkWT7uq+O2jar1U5iYioWS/sjTEKwjaqry1EUhaSkHlgsVhRFMa0Wq1XF5wvNLxHI9nVnHbFtuq7T0HCQ6upyUlIy2r0+2R1uI4/HRUJCMlarzdQAFCLcKIpCdHQcPp8nKOuTEGwzHUWRj08IMwSz4SHfYiFEWJMQFEKENQlBIURYk7PDIWbx4ucpKMinsbGe6uoaevbsxeOPP0VEhMPs0oTokiQEg+TL74tZs6W4Q9Y9fngGZw9r3U8B8vJy0TSdZ555Hrvdzl13zeXzzz/jkksu65DahOjuJARDTEFBHs8//wqRkZEAZGX1p7a2xuSqhOi6JASD5OxhrW+tdZSiogN4PF769u0XmJebu5UJEyaaWJUQXZucGAkh+fnb8HjcFBcXAbBixXKio6MZOXK0yZUJ0XVJSzCE5OVt48orr2XBgt9RXV1FVlZ/HnvsKbPLEqJLkxAMIfn527jpptncc8/PzS5FiG5DdodDSGFhPgMHDja7DCG6FWkJhpCPPlptdglCdDvSEhRChDUJQSFEWAtKCD755JM88MADwViVEEJ0qnaH4Lp163j//feDUYsQQnS6doVgTU0NCxcuZN68ecGqRwghOlW7QvA3v/kN9913H3FxccGqRwghOlWbfyLz7rvvkpGRwdixY1m+fPlJvz45Oaatb90mqamxQV1fWZmK1dp1zit1pVo6gmxf99VR26aqalC+14qu620aHn727NmUl5djsViora2lsbGRK664ggcffLBVr6+srO+0kelTU2MpL68L6jpLSvaQnt4nqOtsq1AerQxk+7qzjty2I7+Dqqq0qXHV5pbga6+9Fphevnw5X3/9dasDUAghuorQbYOLDlVdXc3Pf3636e9VW1vD5MkXBR4/+OD9bNu2NfB48+ZN3HjjNRza4amuruLaa6fyn//8izlzbqK+vr5jixddXlBC8Morr+SJJ54IxqpEN5GYmMjvf/9H098rN3dr4HrprVt/wOVyMnjwqYHnR406jdjYWL744v9wu1386lc/46qrruWcc85n4sRL+Otfl3XKNoiuS64dDhJvwZd487/okHXbBp6LLefsVi1bU1PDM888ya5dO4iMjKJ37z5kZfXn+utntum9v/76K5YsWYzH48bpdDFv3p2cf/5FvPDCH0hKSmb69Buprq7m979fwL59e0lMTKJXr96ccsopXH/9TF544Q9UVlZSU1PD7t07GTPmDCZOvJTXX3+VvXt3M2XKFdxyy+1UV1fz7LP/S0FBHqqqMn36jUyZcgUAzz33LImJSUyffmPTz7KeYufO7aSkpJKV1T8Qgn//+3J+8pNJP9qGm266hVdffYlVqz5h0KDBXHfdDQBMmDCROXNuYs6cuW36bERokBAMIbqu81//9TMmT76c3/3OCKXrr7+KZ599ocVyDz30S/bv33/UdSxe/GpgUCZd13nssd+ydOnbJCYm4vP5cLtdAOTn5zNr1i0APPLIf3PhhRN49NGnAu/5hz+8GFguLi6OJ574PV6vh8suu5j4+AQWLlxESUkxc+bcxOzZt/GrX93HVVddy8MPP05tbQ0zZlzF+PHnkpiYRH5+HjNn3tJi+x5++HH27dvLDTdczSOPPAkYu75HC/tx48azcOFTxMbG8vDDCwLzk5KSsVpt7Nmzmz59+rbvwxfdloRgkNhyzm51a62jbNy4Hk3TuewyowWVmdmbiIgIcnIGtVju0Udb39FqQkICCxc+xUUXXcxZZ40jOto4+7ZjRwE5OYPYsGE99fX1Ld7T4XAwcKDxntu35/Pqq8uw2Wxomh+LReXmm29FVY0jMXFxcWzc+DVWq5WJEy8FID4+gaSkJGpqakhMTGL7duO9Nm5cj8/n/9H2DRpktATLy8tITEz60TYsX/4uHo8br9eLxWJp8VxycjJlZaUSgmFMQjCE5OfnMXjwkMDjPXt2k5ycQmxsy99StbYlqCgKS5a8yTffbGTVqk948cXnePvt9ygpKSY6OoaYmBjy87cFQghg797dpKSkEh0dQ3FxEVFR0fTokQ5AYWEh/ftnExUVBRiDQuXkDGL79sIWQV1fX091dRU9e/aiuLiImJhD75XHkCGHj/ft2LEdhyMysP6IiAg8Hk+L7Vmz5gveeectlixZxvz5t7Nly7cMHz4y8LzH4yYiIqL1H7IIORKCISQhIYENG75G13VcLhfPPPPUUTtZbW1LcM+e3WRm9uaMM86iR4907rnnDhRFCYSX8Z6Jgff0eNw8++zTDBpkBHFBQV6gRQj8KDDz843n09LS+OqrtWiahqZpLFz4FFOnXoXdbm9ax+DA9h1azuPx8Ic/PN1ifVlZA5pCOAUwhht45pkneeaZRaSkpHDDDTN5443X+N///QMAfr+foqIDZGUNOJmPWYQYCcEQctFFE/n888+YMeNKMjN7k5ycTP/+bf+C/+Uvy9i8eRNRUZE4HJE8+qhx7K2gID8QghdddDGff76KG2+8hszM3kRGRnLaaacHlmsewvn52xg16rTA44KCfKZPv57Ro09nw4b1zJx5Hbquc/75FwVOVhjrOPRexvZdf/3VJCUlNe0KH275nnfeBXz99VeMHj2G4uIifv3r+3nooYcDo+9NmjSZ1157mcLCArKzc/j+++8YMmQoMTGde/WS6FrafMVIe8kVI8FzrF/l/+xn87n++pmMGXNGh713Y2MDUVHRgNG9/6OP/g/PP/9KUIOltVcdNDTU89Of3spLLy0N7NIfz29/+2smT76c008/MxhltplcMdI2pl8xIrq+/PxtPzopEmwffLCcjz5aQUREBHFx8fz3fz9iWssqOjqG+fPvpaioiH79so67rMfjYeTIUaYHoDCftATbqDu0BEOFbF/31R1agnLZnBAirEkICiHCmoSgECKsSQgKIcKahKAQIqxJCAohwpqEoBAirEkICiHCmoRgCPnmm43Mn3+72WV0mOrqKqZNu7RVyx7Zzf7ReDyeTulivz11L1r0LNdccznjx49h587tHVXiMT+LrlZPR5AQFN3Gtm25LXqlOZajdbN/NHa7vVO62G9P3eeccz6LFr1EenpGR5Z41M+iq9XTUeTa4RD15z+/yqefrkRVVU4//Sx++tO7qays4He/+2+cTieqqnDPPfczdOgwyspKjzq/ucbGRl555UW+/fYbPB4vWVn9+d3vFhzj3TtGXl5uoNeY555biNvtory8jMLCArKy+vPEE89gtVp/1M3+8YYcOFoX+4sXP09BQT6NjfVUV9fQs2cvHn/8qVZ1ytCWup9++llAPerwACNGjDzKGtvuZD6LzqjneJ91Zw1/ICEYJOuLN7GueEOHrHtsxumcmXHaiRdssm7dl6xZ8wWvvPIGVquVhx76JR988B61tTWMGzee66+fyVdfrWXLlm8ZOnQY//jHiqPOP0TXdR588Beceuowlix5E0VRqKgob/Gere2otT3y8rYxbdrVgNFjTXx8Ao888iRWq5Ubbria3bt3MWBAdotu9k805MDRutjPy8tF03SeeeZ57HY7d901l88//4xLLrmsQ+retWsn/foNOObwAMFysp9FR9cDx/+sO2v4AwnBELRp0wYmTJiIw2EEz+TJl7Ny5UdMn34Dv/71LykoyGfcuPFcddW1AIwZc8ZR5x+yceN6ampquPXWeSiKAkBKSmqLZU6my/7mbrnlBkpLS4763N///mmL7vCNTlmNFlVhYQGvvfYWdrsdAJ/PT1xcHNCym/3WDDlwZBf7BQV5PP/8K0RGRgKQldWf2tqaDqs7Pj7+R3WfjNbWcrKfRXDqUYDDHaUc+dmc6LPujOEPJASD5MyM006qtdaRdF074jH4/T6GDx/Jm2++w9q1a1i9+lM+/vhDnn32hWPOPyQ/P49hw0YEAvBo2toSfPXV1h3zqagox2q1kpiYSFHRAaKjo0lPN7rVr6urw+VykpbWA2jZzX5rhhxo3sV+UdEBPB5voCNWMIb1nDBhYofW7fNpRx0eoDVaW8vJfhbBqOd4vci05rPujOEPJARD0OjRp/P660uYOnUaFouVjz/+O6NHj+GFF/5ASkoa1147g1GjxnDLLcbQk8eaf0hKSirr1n2Jz+fDarVSWVlBcnJKi2Xa2hJsrW3btga60i8szD+im/6W/SY272b/REMOHNnFfn7+NjweN8XFRWRknMKKFcuJjo5m5MjRnVp3RzjZz6Kj6znRZ91Zwx9ICIags88+h8LCfObMmYnf7+OMM87iqquuo7KygocffoiPP/4QVVV56KGHAbjqquuOOv+QCRMm8s03G7nxxmtxOBxkZvbmkUee6NRtys/PazZ2yY+77c/JGRh43Lyb/RMNOXBkF/t5edu48sprWbDgd1RXV5GV1Z/HHmt7wLe17kOeffZ/+fe//4+qqkruvfdO4uLiefPNd9pUy8l+Fh1dz4k+684a/kA6VW0j6VS185zs9h2vm/0jhxw4sov9e+/9KTfdNDswTkpnOLR9Jzs8QHud6LMIRj3H+9ud6LM+0fAH0qmqEMfQvJv9IzXfBT1aF/uFhflHHaGvMxyv7o5wos+io+s53mfdmcMfSEuwjaQl2Hlk+7ov6V5fCCG6OAlBIURYkxAUQoQ1CUEhRFiTEBRChDUJQSFEWJMQFEKENQlBIURYkxAUXY6u6/z853ebXYYIE9KBgugSdF3nxRf/iKpacLmc9OzZE4BXX30Jp7OR+vp67r33F51yTa0IL9ISFEf12GO/5eOPP/zR/I4azOmTTz5i4MAhzJs3n9jYOIYNG8Gnn35CWloP7rnnZ/zqVw9JAIoOISEouoQfftgSGIxo27atDBs2gi1bNndqby4iPMnucIj5058W8a9/rSY+PoHk5BTGjz+X9PQMXnzxj/j9Gn369MXhcLBz5w6qqqoYMGAAv/3tY9jtESxatJAvv1xDSkoKmqYxatTxe8oO5mBOY8eO58UX/0jv3n0pKyslPT2D8867gD/9aRHp6emMG3cOI0aM6uiPT4QhCcEQsmbNF2zZ8i1vvPEOTqeTOXNuZPz4cwHYt28vf/vbP9i+vYDVq1exePFraJrG3XfPY926L9F1nYKCfN588x3q6uq4+ebpx32vYA/mNH78uYFa5869E4DTTz+L008/K6R7WRHmkxAMkoNrv6R2zRcdsu748ecSN+7sEy63ceN6LrxwAjabDZvNxjnnnBd4LjOzDzExMYwcOZq4uHjee+8d9u7dzf79+3A6nWzbtpXzzrsgMB7GWWcd//2CPZiTEGaREAwhqqoes4/GQ4PVrFnzb155ZTHXXDOdSy+9nJqaGnRdR1EUmvcs2XxEsKMJ9mBO48ePOfItjmvNmo0ntbwQxyIhGCRx485uVWutI40ZcybLlr3OtGlX4/G4Wbt2DdnZA1sss3Hj11x44QQmT76cAwf2s3nzJk4//QzGjDmDt956g6lTr8TlcrF+/TqGDh1+zPcK9mBOxws12R0WHUlCMISMGzeeH37YwuzZNxAXF0dKSip2e8vhCqdMmcbDD/+azz77J1arjWHDhlNUVMRll13Btm25zJx5HUlJyfTtm3Xc9wr2YE7N6brOL35xD7///R/b/6EIcQLt6l5/0aJFrFy5EoDzzjuPX/7yl61+rXSvHzyHWko//LCFffv2cskll+Hz+Zg7dzb/9V+/YcCAbLNLPKEjfyytaRrz5t3FQw/9iqefXsiTTz7O1Vdf96OWbSgI5ZZuSHevv3btWtasWcP777/PBx98wNatW1m1alVbVyeCoHfvPqxa9U9mzZrBLbfcwEUXXdwtAhCO/mPpqKgoLr54Ej/96W1ceeW1IRmAwnxt3h1OTU3lgQcewG63A9C/f/9OGyVLHF1cXDzPPPOc2WW0yQ8/bGHGjJsA48fSkydfjtfrZdOmDURERJCenm5yhSJUtTkEs7MPtzB2797NypUrefvtt1v9+rY0W9sjNTU2qOsrK1OxWrvOBTddqZa2GD/+HP70p+fo06cv5eVlpKensWDBo1x33Qw0TeO1117iF794wOwyO0x3//sdT0dtm6qqQflet3vIzcLCQubOnctdd93FtGnTWv06OSYYPKF8TAlk+7qzkD4mCLBp0yZuvvlmfv7zn59UAAohRFfR5t3h4uJi7rzzThYuXMjYsWODWZMQQnSaNofgkiVLcLvdPPHEE4F506dPZ8aMGUEpTAghOkO7jwm2lRwTDJ5QPqYEsn3dWcgfEwxvyo+unxVCdI5gtt0kBNvIbndQU1OBz+cN6h9ECHF8uq7T0HAQq9UelPXJtcNtlJiYSn19LVVVpWia39RajN5jQrdVKtvXfXXUtlmtdhITU4OzrqCsJQwpikJsbAKxsQlml9Ihxzy7Etm+7qs7bJvsDgshwpqEoBAirEkICiHCmoSgECKsSQgKIcKahKAQIqxJCAohwpqEoBAirEkICiHCmoSgECKsSQgKIcKahKAQIqxJCAohwpqEoBAirEkICiHCmoSgECKsmdap6gevfEhtbQO6YuSwrqjoimLcaLpXVAhMG4/1Zsu2eI4fvx5FQVNULBF23D5AUcza3A5ls1vweszt3bojyfZ1Q7qOgk6EHTSPFwUNFR1F11HQaPrGYnxbtaZ743n1iHlqi+e1Zq9reYuNi+aG268/6VJNC8ER2/+Nu6y8U99TUxQ0xYKuqGiqiqZY0FQLmqKiBx4ffb5fteK3WPGrVnwWW4vHh+8Pz/epNuPeYsdrtaOrlk7dViHQdSz4sePBpnuw6x5seLHqPqz4sODDphv3h+ZZm08HlvOj6n5UNGMazZjW/agcmq+1WMZCU5f6jZ23uVZPKnDyIWjakJvFuTvwu93omga6DpoWmNY1DZpP603PaXqLaV1vWq7ZtK7pTa81pnW/j2iHlfqDjeg+n3Hz+9B9fnS/D5ruDz/nN25Nj/H70L0+NJ8X3e1B93rQ3G6j5pOg2O2oDgdqZBRqZCSWyEhURyRqZCRqlDFtiYlpusUat1hjWo2IOO66u0MX5u0Rztunaxq6pwFc9eiuenR38/sGY9rTiO51gdeF7nGie53oHid4XKCfZAtTtYLVjmK1g8WGYo0AixUsVhTVYjyvWlAsxr3xuOk5i6XZtDE/JjaKBqfPWFZRQFFBUVEUFVQ18PjHzynNnmu6qU3PNX+NqgLGtMVmJbVf/5P+/E1rCdp79Oi24w7rug5+P5rHg+7xNN270Tzepvum+W43msuJ5mx+a8TvdKE5G/EdPIjmbDTmu1zHDFbFbm8WjjFY4uOxxidgTUzEmpBARN+eeInAGh+PYpVhY7o6XdfB3YDmrEVvrKWuxIWntASt0XisN9agN9aguerA3Qgc43uiWFAc0WCPQrFHotgcqHGpYDOmFXsk2B0otkhj2uYw5ltsh4OuRdjZUNTgniZISI3F20n/galq2w53yTemDRRFAasVi9UKUVFBWaeuaWiNjfjr6/HX1x2+r6vHX3+w6b4Of30dntISfDU14Df+ly9uth5LbBzWhASsycnYUlKbbinYUo3pE7UqRfvpumaEWV0FWn0lWn1FYFqvMx7j8wSWdx6asNpRohJQoxJQkzOxOOJQHDHGLSK66T4mMA9bpPFvUbSLhGAXoahqYHcY0k+4vK5p+Bvq8VVXE627qdxThL+mBl9NDb7qKrxlZTTmbkX3eFq8zhIbawRjWhr29AzsGRnY0zOw9eiBagvOOK7hQNd1dFcdWm0Jek0JWm0JWm2pcX+wFPy+FssrETEoscmoCRlYeg1FjUlCiUpAiYonuVdPalw2o6UmodbpJAS7KUVVscbGYY2NIyk1Fn+fnB8to+s6/ro6vBXleCvK8VVUGNPlFTi3F1L39frDu+CKYrQYexjBGJFxCvZemUT07Bn2rUfNeRCtan/TbR/+qgNotSXgaXbUX7WgxqWhxqdjyRxmTMcko8SkoMYmo9gcx1y/PTkWJYSPeXZ1EoIhTFEUrHFxWOPiiMz68QFjze3GW1aKp7gYT0nTrbiY2oK8wy1IRcHWowcRvTKNW2ZvIjIzsSYmhVyrRdd8aNVFaBV78DcLPd15MLCM4ohFTeqFbcBZqPHpxi0hHSUm2TghILodCcEwpkZENIVa7xbzdU3DW1mBe98+3Pv24tm/H/ee3dRv3HD4tdHROPr0xdEvy7j17Yc1wfyB6FtL1zS0mmK0il34y3fhL9+NVrkX/F5jAYsdNaknlswRWJJ6oR66RcWbW7gIOglB8SOKqmJPTcOemkbs6NMC8/1OpxGI+/fi3rcX165dVK38yPhJEmBNTMLRr18gGCP69MUSGWnWZrSgNdbgLynAX7oDrXwX/oo94HMbT9ocWFL6YBtyIZbUflhS+qLEpQX9TKnomiQERatZIiOJzM4mMjs7ME9zu5sCcSeuXbtw7dpJ/TebjCcVBXvPXkQOyCZywAAis3OwJiV3+G60ruvotSX4SgrwlxTiLylAP1jWtBF21JTe2AadiyWlL2pqP9T4dAm8MCYhKNpFjYhoCrnDweivr8e1exeunTtw7thO3Vdrqf3X5wBYExOJHJCNo+k1Eb0yUSztO5ama37jOF5T4PlLCtBdxokGxRGLJT0by5ALsKQPRE3pjaLKP3txmPxrEEFniYkheugwoocOA4zjb+79+3BuL8S1vRBnYSF1G74GQIlwEJnVv6mFmYMjq/8Jz0brXhf+sp34i/PxlxbiL90R2LVVYlOxZA7HkpGDJT0bNT4j5E7giOCSEBQdTlFVHL374OjdBy6cAIC3shLn9kKc2wtwbS+k8sMVxs91VJWI3n2Iys7BkZ1D5IBs/FEa3l2bA608rWIP6BqgoCZnYhs4Hkv6QCP0ohPN3VjR7UgIClPYkpOxJScTd+ZZAPgbG3Ht3E5jQQHOvK3UfP4Z+qp/ArDHAfZYsMepRPbrg33EpVgzcrD0GIBiD84VOyJ8SQgKU+ma3/gBckkhakkBkY2FOFJq0JPA63Xg0xLxN6o0FlXiLHdRu2MXlvhqIrNLm245xnFFObEh2khCUHQq3ePEX7bDOIlRuh1/2Q7wugBQYpKxnDLYOJGRPhA1MQNFUUlNjaWstBZPcRHOwoKmW2Hgd4uqw4Gjv3H2OTI7B0e/LFS7XAIoWkdCUHQorb7KOHlRYty0qr1Nl+opqMm9sGWf3RR62agxycdcj6KqRPTsRUTPXiScfyFw6LiiEYjOwgIqP1huLGyx4Ojbzzhr3XRc0bgmW4gfkxAUQaN7Xfgr9qCV7TSuwijbgV5faTxptWNJ64991BQs6TlY0vob3Tu1g3FccSxxZ44FjJ/mOHdsD7QWqz/7lOp/rgTAfsopgUB09MvCltZDdqEFICEo2kjXfGhV+/GX7TSuwCjbhVZzINAhgxKbgiUtC8uwiUYrLzmzw3+fZ4mJIWbESGJGjARA83hw7d4V2H2u+3o9tf/+F2DsQkf06Yujb18cffoR0bcfttRU+TlNGJIQFCeku+rxV+1Dq9yHv3IfWtVetOoDge6ilIgY1LQs7P1Ow5LWDzU1CzUyzuSqQbXbicoZSFTOQMD4vaKn6ACuPbtx7d6Fe/dualZ/ZvQgDqhRxvXQEX36EJGZSUTPXtjTM6Sj2hAnf10RoHvdRn94NcVGK68p+PSGqsAySmQcalImtlMnGNfZpmahxKZ0ixaUoqqB3nDizz4HAN3nw110IBCKrt27qF71z0CHtVgs2NMzjOORvXphb7q3JibJ7nSIkBAMM7qmoTdWox0sM8Ku2S1w/A5AUY0OQDNyUJN6Y0nORE3ORI3qPj3FtIZitR7+Ife5xjzd58NTWoJ7/348B/YbV7vsKKTu668Ov85ux5bWA3uPHth7pGPrkY49PR17j3Q5CdPNSAiGGF3XjMF36irQ6srR6srRDx6arkCvrwCt2eA7NocRduk5qAkZh29xacYYFGFIsVoDZ6Kb8zc24jlwAHfRfrwlJU1BuY/6zd8EetIBo5sxW2qaceImKRlrcorRYW1yMtbkZCxR0Z29SeI4JAS7CV3zo7sb0F11xvgVDdVojdXoDTWU+utxVpUHBuhpEXIYnQgosSlYUvqg9jsNJS4NNTYVNfEUo4v3brAr2xVYoqJ+1IsOGC1Hb0UFntISvKVGOHorKnAf2E/Dlu/Qvd4Wy6uRkViTkrHGx2OJj6chIw2PLRJrfAKWhASsTQNpqY5j90YtgqddIfjhhx/y4osv4vP5mDVrFjfccEOw6gpJxih1nqbhEI1hEY1bo/H4UMi56o3xK5pNH3PUMVskxCejRMShZgxCjU4wBuuJSUGJS0GNTT1u1+6i/RSr1dgVTv/x2DCHhziowFdVgbeyEl9lBd6qKvwHa/GUlFC/4evAyZkW67XbsUTHYImJRo1uGo71iHs1JhpLZDRqpMMYvjXCgepwyMmck9DmT6q0tJSFCxeyfPly7HY706dP58wzz2TAgAHBrK9D6boxjjGaD7SmcYg1v3HWU/Ojaz7jOb8P3e8Fnxvd5wWfB93nBp+36d6D7vMYAefzHH7scweC7lDonXAcWNVitNwcsSiOGNTk3oHpwH1UPGpUIkp0AorNEfLj8nbBNEl6AAAYqUlEQVRnzYc4ICvrqMukpMRQursEX20Nvtpa/LU1+Gpq8dcdNEYdbKjHX1+Pe/8+tPoG/A31Jxz3OjDOtSOyabzrpjGu7REodhuKzY5qtxvL2WwoTfNVW9M8ux2lab5qs4HFgmK1olgtKBZj3GHjsdV4rhvvTbQ5BNeuXctZZ51FQlOX6hMnTuSTTz5h/vz5rXq9e/OH+BsOGiF0KIyabnrzx5oGNBtQvcXyzV/XbDD2Zsuj+XEqGn6v1wizpsAzws3PMcd0PRmK2mwc14jAeK6K1Y4SnYSaaIz7qtijwH5oOhLFFtn0OArF7kCJiJERx8KQoiiBkQaPPA55NLqmoTmdgYAMjGntch0e59rlRHO6WszzVVaguT3oXk/g/shd9TazHApHSyAcFYuVvXYbmqKiHBpMXVWNs+rN75sPtq4qhwdVD0wr6E3P6YoxT296TldoGrhdwRYXR/Its0+69DaHYFlZGampqYHHaWlpbNmypdWv929fh6+uyvggmkaTV44cZV5tPlK9EpivNBu5XrGooFiN55vPb/ogFdX4Y2CxoqhN/4Mdemyxtny+xTK2lq+x2VGsEai2iGbTdhRbhPF6k6WmxppdQoeS7TvSyY91ous6Xs2H2+fG4/fi9ntwe9x4XI24XY14XY24XU68Tic+twuvy4XP7cTvcaP5fGheH5rfh+7zoXt96H4/us8PPmMavx98fvBrxrTfh+LzoPj9TQ0WHcWro+g6aKDoh6Z1FP3QYwLT6lGnQT20nAYKgG7cR6akQGeGoKZpLVosuq6fVAsm6prH0bQgtMJaofnuok4Q2n6+pptLo9nQ2aYJ9d1h2b7DvJqPBm8DDd5G6j0NNPiM+0afE5fPhdPvwuVz4fK5cfmb7gPz3fhPdDjmKBRFwRphweKwYlUtWBQVi2rFqliwqDYsigWLaml6bMWiqE3LWYmOjMDn0VEVFbWpIaNiTKtNjRXj8XGebzZPaVru0GsURUFBMZaztW1o2DaHYHp6Ohs3bgw8Li8vJy0tra2rEyJsaZpGrfugcfMcpObQdNPjOk8d9d5GGrwNuP2eY67HqlpxWCJwWB1ENt0nOuJxWHoQaXXgsDqIsEQQYbFjV23YLLaj3NuxqTZsFqsxbbFhVdp+zK8z/wNT1bbV2OYQHDduHM899xxVVVVERkby6aef8sgjj7R1dUKELE3XqHHXUumsosJVTaWzikpXVdN9NbWeg8Zx8GYUFOLsMcRHxBFjjyE9ugcxtmiibdFE26KapqMC86Jskdhk7JQ2afOn1qNHD+677z5mzpyJ1+vl6quvZvjw4cGsTYhuxeP3UtpYTmlDKSWN5ZQ0llHaUEZZYzm+ZruhCgoJEfEkRyYyMHEAvZJ6YPNHEG+PIz7CuMXaYrDIYO6dol3/dUyZMoUpU6YEqxYhuo16TwP76g6wr/6AcV93gApnFXrTEWcFhZTIJNKj0xiSPJDUyGSSI5NIdiSR5EjA2qzVFurHPLs6aT8LcQJev5c9dfvZWbubXbV72Vd3gGp3TeD5ZEcSmbGncHr6aNKj0kiPTiMtMgWbxWZi1aK1JASFOEKj10lhzQ521OxmZ+1u9tYdCJxVTYtKoX9CXzJje9I7tie9Yk4hyiaDPXVnEoIi7Pk0H7tq95JXXUheVSF7Du5DR8eqWOgdl8mFmeeQFd+HrPi+xNil84NQIyEowtJBTx3fV+TyfUUu+dU78Pg9KCj0jctkUt8LGZiYTd/43nLGNQzIX1iEjbLGcr4r38qWilx21e5BRyfJkciZ6acxKCmbnIT+RNnaN+6J6H4kBEVIq3bVsLH0WzaUbuZAfTEAvWJO4ZJ+ExiRcio9YzLkWu0wJyEoQk6Dt5HNZVvYULqZ7TW7AOgb15ursqcwImUoyZGJJlcouhIJQRESdF1ne81O1hSt59vyH/BpPnpEpXFZv4mM6TGS1Khjj2kswpuEoOjW6jz1rC/ZxJdF6ylrrCDS6uDsU85gbMbp9Io5RXZ1xQlJCIpu6UB9MZ/v/Q8bSzfj0/1kxfdl0uCLGJU2DLslPMdGEW0jISi6DU3X2FqZx+f71lBQvR27amPcKWdybq+xZET3MLs80U1JCIouz+f3sbboa1bt/RdljRUkRMRzRf9LOfuUM+RqDdFuEoKiy/JqPtYVbWD1V/+morGKzNiezB4yg1Fpw6WHFRE0EoKiy/H4vXxZtJ5Ve/5FrecgOclZXJt9BUOSBsqJDhF0EoKiy/BrftYWb2DlrlXUeurITshi5pDrGJ8zioqKerPLEyFKQlCYTtM1Npd9zz92/pMyZwVZ8X2Yfer1ZCf2B5DWn+hQEoLCVHlVhazY8TF76w6QEd2DecNvZmjyYAk+0WkkBIUpyhsreW/7h3xfkUtiRAI3Db6WM9JHoyqq2aWJMCMhKDqVy+fin3v+j8/3foFFtTC1/yVc0Gu89MIsTCMhKDqFpmtsKNnMih0fU+up48z007i8/yQSIk5+EHEhgklCUHS4ovoS3s5fzs7a3fSJy+S2YbPoF9/b7LKEACQERQfy+L38c/dqVu39Nw5rBDcOuoYzM06T436iS5EQFB0ir6qQv+Qvp9xZyZnpp3HlgMtkfA7RJUkIiqCq9zTw3vYP+brkG9IiU7h75O0MTBpgdllCHJOEoAia78p/4O285TT6nFzS9yIm9rlQzvqKLk9CULRbg7eRdwtWsKF0M5kxp3DXqNvoGZNhdllCtIqEoGiXHyq28Vbe36jzNnBpv58wqc+F0sOL6FYkBEWbOH1O/lb4IV8Vb+SU6HTmjZhN79heZpclxEmTEBQnbUfNbpbmvk2Nu5ZJfS5kUr8JMki56LbkX65oNb/m55M9n7Ny12ckOxL52eg76Bffx+yyhGgXCUHRKpXOapbmvs3O2t2ckT6aa3OuINLqMLssIdpNQlCc0KbSb3k7fzm6rjNryHTOSB9tdklCBI2EoDgmt9/DO/kf8FXJRvrF9ebmU2eQEimDmIvQIiEojqqkoYxXfniDkoYyJvW5kEv7/UR++iJCkoSg+JGNJZtZlv8edtXGnSPnMDgpx+yShOgwEoIiwKv5eK/wQ/5zYB1Z8X2ZM/QG6e9PhDwJQQFAhbOKJT+8wd66A1zU+1ymZl0iu78iLEgICraUb+XP294BdG4fNosRqaeaXZIQnUZCMIxpusbHu1axcvdqesf2ZM7QG+Xsrwg7EoJhyulzsnTrX/ihchtnZYxhes406fZKhCUJwTBU0lDK4u9fp8JZxbU5V3Buz7Eyzq8IWxKCYea78q38Ofcv2FQbd4+8nezELLNLEsJUEoJhQtM1Vu76jI93f0af2ExuG3YTiY4Es8sSwnQSgmHA6XPxeu5f+L4il7PSxzB9oBz/E+IQCcEQV+Gs5MXvXqPMWcE1OVM5r+c4Of4nRDNtDsFNmzaxYMECvF4vCQkJPP744/Ts2TOYtYl2Kqzeycs//Bld17lr5K3kJMqob0Icqc2jYN9///08+uijrFixgilTpvDoo48Gsy7RTuuKNvDcty8TY4vm/jHzJQCFOIY2haDH4+Gee+5h0KBBAAwcOJDi4uKgFibaRtM13t/+EW/mvUt2Qha/OO1O0qJSzS5LiC6rTbvDdrudqVOnAqBpGosWLWLChAlBLUycPJfPzdLct/m+Ipdze47l6uzL5fpfIU5A0XVdP94CK1euZMGCBS3mZWVlsXTpUjweDw888AC1tbX86U9/wmaTM45mqWio4sn/vMC+g8XcPOoaJmWfb3ZJQnQLJwzBY2loaOCOO+4gISGBp59+GrvdflKvr6ysR9Pa9NYnLTU1lvLyuk55LzNUK+U8+cWL+HQfc069kcHJodX/X6j//UJ5+zpz21RVITk55qRf1+azw/fffz99+vTh4YcfRlXbfH5FtNM3ZVv4c+5fiI+I597hc0mP7mF2SUJ0K20KwdzcXFavXs2AAQOYNm0aAGlpabz88stBLU4cm67rrN73BR9s/5ic5H7cMvgmYuzRZpclRLfTphAcMmQI+fn5wa5FtJKma/yt8O/8e/9aRqUO4+fn3kpttdvssoToluSKkW7G7ffw2ta3+L4il4t6n8sV/S/FbrUDEoJCtIWEYDdS667jT1teY1/dAa7LuYJze40zuyQhuj0JwW6ipKGUF757lTpPPXOHz2JYyhCzSxIiJEgIdgOF1TtY/P2fsaoW7h09jz5xmWaXJETIkBDs4jaUbObNbe+QEpnMT0fcQnJkktklCRFSJAS7KF3XWbXnX6zYuZLshCxuHzaTKFuU2WUJEXIkBLsg4ycwH/Lv/V8ypsdIbhx8LTZV/lRCdAT5ZnUxXs3Hn3P/wjdlW7gw8xymDZiMqsgVOUJ0FAnBLsTpc/LSlj9TULODaQMmM6H3eWaXJETIkxDsImrdB3n+uyUUN5Qya8h0zkgfbXZJQoQFCcEuoLSxnOe/fYU6bwPzhs/m1OSBZpckRNiQEDTZnoP7eOG7VwG4d9Rc+Q2gEJ1MQtBEuZX5vPzDG8TaYpg/co50gy+ECSQETfJ1yTe8se0dTolO56cj5hAfEWt2SUKEJQlBE3y299+8v/0jchIHcPuwmURaHWaXJETYkhDsRIdGgvt8338YnTacmUOmy4+ghTCZfAM7iV/zsyzvb6wv2cR5vc7m6uwp8iNoIboACcFO4PF7eXXrMr6vyOWyfhOZ1PdCFEUxuywhBBKCHc7pc7F4y1K21+ySjlCF6IIkBDtQnaee579bwoH6Ym4eMp0x6aPMLkkIcQQJwQ5S5apm0bevUOWqYe6wWQxNGWx2SUKIo5AQ7AAlDWU89+3LuP1u5o+8lQEJ/cwuSQhxDBKCQbb34H6e/24JiqJw76h59Io9xeyShBDHISEYRAXVO1i8ZSnRtijmj7yNtKgUs0sSQpyAhGCQfFe+lVe3LiM1Mpn5I28lISLe7JKEEK0gIRgEXxVvZFne3+gd24ufjriFaBkLRIhuQ0KwnT7f9x/eK/yQQYnZ3DZsJg5rhNklCSFOgoRgG+m6zj92fconu1czMnUYN586Q64DFqIbkm9tG2i6xrsFK/jiwDrGZZzOjEFXyXXAQnRTEoInya/5eWPbO2wo3cxPep/P1P6XyHXAQnRjEoInwav5eO2HZXxXsZXLsyYxse+FZpckhGgnCcFWcvs9vLTldfKqC7kmZyrn9zrb7JKEEEEgIdgKTp+TF757jV21e7hx8LWMzRhjdklCiCCREDyBQz3BFNWXcMvQGxidNtzskoQQQSQheBw17lqe2/wyla4q5g6fxanJg8wuSQgRZBKCx1DhrOK5zS9R563nzhFzyE7sb3ZJQogOICF4FIe6wvL4Pdw96nb6xvU2uyQhRAeREDzCvroDLPr2FaMrrNHz6BmTYXZJQogOJCHYzM7aPbzw3RIcFgd3jbqNHlGpZpckhOhgEoJN8qoKWfz968TbY7l71O0kORLNLkkI0QkkBIHvK3J55Yc3SYtMYf7I24iPiDW7JCFEJwn7ENxYspnXt/2VzJie3DlyjvQFKESYCesQ/LJoPW/nLad/Ql/mDZ9NpNVhdklCiE4WtiH4+d4veG/7PxiSNJDbht2E3WI3uyQhhAnCLgR1Xefj3Z/x8a5VjEwdxuxTZ2CVzlCFCFth9e3XdZ33t3/E6n1fcGb6adww6GosqsXssoQQJmp3d8i5ubkMHTo0GLV0KE3XeDt/Oav3fcG5Pcdx4+BrJACFEO1rCTqdTh555BG8Xm+w6ukQfs3Pn3PfYUPpN1zc5wIuz5okvUELIYB2tgSfeOIJZs2aFaxaOoRP8/HsuiVsKP2Gy/pNlO7whRAttLkluHr1alwuF5MmTWrT65OTY9r61q3m8Xt5Zu3LfFP0PTNHXsVlAyd0+HuaJTU1tH/gLdvXfXX1bTthCK5cuZIFCxa0mJeVlUV9fT1Lly5t8xtXVtajaXqbX38izbvDv/W06YyKH015eV2HvZ+ZUlNjQ3bbQLavO+vMbVNVpU2NK0XX9ZNOonfffZfFixcTHR0NQF5eHoMGDWLZsmXExLSuiI4MQZfPxQvfvcbO2t3cMPgaLh9+Qcj+I4PQ/hKBbF931h1CsE27w9dccw3XXHNN4PHAgQNZsWJFW1YVdI3eRp7/7lX21u3n5lNnMKbHSLNLEkJ0YSH1O8E6Tz2Lvn2F4oZSbh16IyNSu/5Pd4QQ5gpKCObn5wdjNe1S6z7IH799mUpnJXOH38ypyQPNLkkI0Q2EREuw2lXDHze/RI3nIHcMv4WBSQPMLkkI0U10+xCscFbyx80v0eB1Mn/ErfRP6Gt2SUKIbqRbh2BpQxl/DAyIdBt94jLNLkkI0c102xA8UF/Mc5tfRkeXAZGEEG3WLUNwb91+Fm1+Batq5e5Rt5MenWZ2SUKIbqrbhWDzEeHuGTWX1Khks0sSQnRj3SoEC6p38OKW14izx3KPjAgnhAiCbhOC2yoLWPz9UpIdSdw16jYSIuLNLkkIEQK6RQh+X5HLK9+/QY/oNO4aeRux9o7vgUYIER66fAh+U7aF17a+JUNiCiE6RJcOwfXFm3hj2ztkxffhjhG3yJCYQoig67IhuObAV/wl/32yE/szb/jNRMiQmEKIDtAlQ/D/9q3hb4V/59TkQdw69CbsFpvZJQkhQlSXC8FPd/8fK3auZGTqUGafer2MCSyE6FBdJmF0XeejXZ+ycvdqxvQYyczB18mQmEKIDtclQlDXdd7f8RGr937B2IzTuX7QVahKu4dEFkKIEzI9BDVd492CFXxxYB3n9hzHNTmXSwAKITqNqSGo6Rpv5b3HuuINXNT7XKb1nyxjAgshOpVpIahpGq/n/oWNpd9ySd8JTO73EwlAIUSnMy0E/1r4ARtLv2Vq1iVc3PcCs8oQQoQ500KwtLGUmadex9iMMZ3yfqoa2q1M2b7uLZS3r7O2ra3v06bB14UQIlTIaVghRFiTEBRChDUJQSFEWJMQFEKENQlBIURYkxAUQoQ1CUEhRFiTEBRChDUJQSFEWAu7EMzNzWXo0KFmlxFUmzZt4uqrr2bq1KnMmjWLAwcOmF1SUHz44YdceumlXHzxxSxbtszscoJu0aJFTJ48mcmTJ/PUU0+ZXU6HePLJJ3nggQfMLuO4wioEnU4njzzyCF6v1+xSgur+++/n0UcfZcWKFUyZMoVHH33U7JLarbS0lIULF/LWW2/xwQcf8Ne//pXt27ebXVbQrF27ljVr1vD+++/zwQcfsHXrVlatWmV2WUG1bt063n//fbPLOKGwCsEnnniCWbNmmV1GUHk8Hu655x4GDRoEwMCBAykuLja5qvZbu3YtZ511FgkJCURFRTFx4kQ++eQTs8sKmtTUVB544AHsdjs2m43+/ftTVFRkdllBU1NTw8KFC5k3b57ZpZxQ2ITg6tWrcblcTJo0yexSgsputzN16lTA6KNx0aJFTJgwweSq2q+srIzU1NTA47S0NEpLS02sKLiys7MZOXIkALt372blypWcd955JlcVPL/5zW+47777iIuLM7uUEzK9e/1gW7lyJQsWLGgxLysri/r6epYuXWpOUUFyrG1bunQpHo+HBx54AJ/Px9y5c02qMHg0TWvRya6u6yHZ6W5hYSFz587ll7/8JX379jW7nKB49913ycjIYOzYsSxfvtzsck4oLLrSevfdd1m8eDHR0dEA5OXlMWjQIJYtW0ZMTIzJ1bVfQ0MDd9xxBwkJCTz99NPY7d1/oPr333+fjRs38thjjwHw/PPPo+s68+fPN7my4Nm0aRN33303Dz74IJMnTza7nKCZPXs25eXlWCwWamtraWxs5IorruDBBx80u7Sj08NQTk6O2SUE1R133KE/9NBDut/vN7uUoCkpKdEvuOACvbKyUm9sbNQvv/xy/bvvvjO7rKApKirSzzzzTH3t2rVml9Kh3nvvPf1Xv/qV2WUcV8jtDoeb3NxcVq9ezYABA5g2bRpgHD97+eWXTa6sfXr06MF9993HzJkz8Xq9XH311QwfPtzssoJmyZIluN1unnjiicC86dOnM2PGDBOrCk9hsTsshBDHEjZnh4UQ4mgkBIUQYU1CUAgR1iQEhRBhTUJQCBHWJASFEGFNQlAIEdYkBIUQYU1CUHRJTz31FHfeeWfg8ZNPPsmsWbNCri9IYT65bE50Sbfddhs/+clP2LZtG99++y1r1qzhrbfewmazmV2aCDFy2Zzosp577jk+/fRT6uvreeutt8jIyDC7JBGCZHdYdFmDBw+moKCAn/3sZxKAosNIS1B0Sfn5+cyZM4fx48dTUlLS7TvEFV2XtARFl1NaWsodd9zBww8/zP/8z/9QUFDA+vXrzS5LhCgJQdGl1NfXc9ttt3HzzTdz0UUXERkZyZw5c1i4cKHZpYkQJbvDQoiwJi1BIURYkxAUQoQ1CUEhRFiTEBRChDUJQSFEWJMQFEKENQlBIURY+3+kjFYU4xfbJAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Compute the range of values that we wish to explore\n",
    "xs = np.arange(-5.0, 5.0, 0.01).astype(np.float32)\n",
    "\n",
    "# Use sigmoid to compute predicted probabilities:\n",
    "prob_regr_q = [f_sigmoid(x) for x in xs]\n",
    "\n",
    "# Compute binary cross-entropy of sigmoid:\n",
    "prob_regr_loss = [f_prob_regr_loss(x, 1.0) for x in xs]\n",
    "\n",
    "# Compute gradient of binary cross-entropy of sigmoid with respect to xs:\n",
    "prob_regr_loss_grad = [f_prob_regr_loss_grad(x, 1.0) for x in xs]\n",
    "\n",
    "plt.figure(figsize=(5,5))\n",
    "plt.xlim(-5.0, 5.0)\n",
    "plt.ylim(-5.0, 5.0)\n",
    "line_p, = plt.plot(xs, np.ones_like(xs), label=r'$p$')\n",
    "line_q, = plt.plot(xs, prob_regr_q, label=r'$q=sigmoid(X)$')\n",
    "line_loss, = plt.plot(xs, prob_regr_loss, label=r'loss $c =-ln(q)p-ln(1-q)(1-p)$')\n",
    "line_loss_grad, = plt.plot(xs, prob_regr_loss_grad, label=r'grad loss $\\frac{dc}{dx}$')\n",
    "plt.legend(handles=[line_p, line_q, line_loss, line_loss_grad])\n",
    "plt.xlabel(r'$x$')\n",
    "plt.show()\n",
    "\n",
    "if SAVE_PLOTS:\n",
    "    plt.figure(figsize=(5, 5))\n",
    "    plt.xlim(-5.0, 5.0)\n",
    "    plt.ylim(-5.0, 5.0)\n",
    "    line_p, = plt.plot(xs, np.ones_like(xs), label=r'$p$')\n",
    "    line_q, = plt.plot(xs, prob_regr_q, label=r'$q=sigmoid(X)$')\n",
    "    line_loss, = plt.plot(xs, prob_regr_loss, label=r'loss $c =-ln(q)p-ln(1-q)(1-p)$')\n",
    "    line_loss_grad, = plt.plot(xs, prob_regr_loss_grad, label=r'grad loss $\\frac{dc}{dx}$')\n",
    "    plt.legend(handles=[line_p, line_q, line_loss, line_loss_grad])\n",
    "    plt.xlabel(r'$x$')\n",
    "    plt.savefig('prob_regr_loss_2.png', dpi=600)\n",
    "    plt.close()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regression: no non-linearity and squared error loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function for computing squared error loss\n",
    "def f_regr_sqr_loss(a, b):\n",
    "    return (a - b)**2\n",
    "\n",
    "# Gadient of squared error loss\n",
    "def f_regr_sqr_loss_grad(x_hat, x):\n",
    "    t_x_hat = torch.tensor(x_hat, requires_grad=True)\n",
    "    t_x = torch.tensor(x, requires_grad=True)\n",
    "    # Compute squared error\n",
    "    loss = -(t_x_hat - t_x)**2\n",
    "    # Sum and compute gradient\n",
    "    loss.sum().backward()\n",
    "    return t_x.grad.detach().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUEAAAFGCAYAAAD0Cy2JAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XlgFPXdBvBnZndnc9+bA3KRG8idgFyCCggSMYCoWK1Yqa9Srb7aqqi81lZb0dqXvi2t1VbL21ftQSVcAgVR0RAEckDInZCQ+77Pnd2def9AUiMh2Wx2d3Z3vp+/zO7MzjNu8jCzO/P7MaIoiiCEEJlipQ5ACCFSohIkhMgalSAhRNaoBAkhskYlSAiRNSpBQoisUQkSQmSNSpAQImtUgoQQWaMSJITIGpUgIUTWqAQJIbJGJUgIkTWlVBvu7h6EIFhnABtfXzd0dg5YZVuW8tr7eQj2c8N3V8de85wj7N9E5LZ/w1oDXv7zGaxID8HK9JAxyxoaSzDyxXvg5t8FVeQN1o46ZdZ871iWgbe365TXk6wEBUG0Wgle3Z4983ZX41x5G76zMhoMw1zzvL3v32TktH+V9d1o7RpGkI/LmMdFUcBQzt8gigzYsFS7+X9i6znpdNhORAd7oW+QR1vPsNRRiIVVNPSCZRhEzPAY87i+Jg9CZy3UqZlgWMmOXxwOlaCdiAn2BABU1PdInIRYWlVDD0ID3ODE/bvoREEAn7cPrFcQlFELJUzneKgE7USQnytcnZRUgg5ObxBQ3dSH6GCvsY9Xn4XQ3QgubT0Ylv5szYn+b9oJlmEQE+KFyvpeqaMQC6pt7QevFxD99ZE/AIiCAdq8LLA+IVBGpEuYzjFRCdqR6GAvtPUMo2dAK3UUYiFX/5H7ZgnqK3Mg9raCS18PhqE/WXOj/6N2JCbkyikSnRI7rqrGXvh7OcPTTQ0AEA16aPP3g9XMgjIsReJ0jolK0I6EBrhBrVLQKbGDEkURlQ09Y44CdeVfQuzvgDpt/biXRpHpoxK0I0oFi8iZHiinI0GH1No9jP4hHaK/PuIX9Tz4goNgA6KgCEmQOJ3johK0MzHBXmhsH8DQiE7qKMTMKr/+x+3qkaCu7CTEwS6o0zfQUaAFUQnamZgQL4gAKhvolNjRVNT3wM1ZhUAfF4h6LfiCg1DMmA3lzDlSR3NoVIJ2JmKGBxQsg4oGOiV2NGV1PYgN9QLDMNAVn4A43AcufYPUsRwelaCd4VQKhAe505cjDqajZxidfSOIC/WGyA+DP38YiuB4KAOjpY7m8KgE7VBMiBdqmvvA6wxSRyFmcvXLrtgQL/BFxyFqB6Cmo0CroBK0QzHBXjAIIqqb+qSOQsykrK4bbs4qBHkw4AuPQhmWAoV/hNSxZIFK0A5FB3uCAV007UjK63oQE+IFfdG/AH4IXPp6qSPJBpWgHXJxUiHE3w1ldd1SRyFm0NY1hI7eEcydoQJ/8RiUEfOg8A2VOpZsUAnaqbgwb1Q19kGnp88F7V1RdQcAIH44F9BpwaWtkziRvFAJ2qm4MG/oDQIuNdLngvau6FInApx4OFV/AWXUAii8Z0odSVaoBO1UTLAXGAZ0SuwALl7qwDqfckDQQ01HgVZHJWinXJyUCA90R2ktlaA96+obwXBXO+K0F6GKWQzWM0DqSLJjlhJ8/fXXsW3bNnO8FJmCuFBvVDf1YYTXSx2FmKi8rge3Ol8EAxFc6h1Sx5GlaZfg6dOnkZWVZY4sZIpmh3nDIIgoremSOgoxUV31ZSxQV0EVtxSsu0bqOLI0rRLs6enBzp078eijj5orD5mCqGBPKFgGFy91SB2FmGhG82dgGAZqOgqUzLRK8KWXXsJTTz0FDw+PyRcmZufEKTEryAOFlVSC9qirsRYJqEDXjIVgXb2ljiNbJk9eumfPHgQFBWHhwoXYu3fvlNf39XUzddMm0Wjcrbo9a0mdHYB/floJV3cnuDippI5jMY74/jUfPAAVFNAsvdMh9+8qW983k0vw8OHDaG9vR2ZmJnp7ezE0NIRf/OIXeOGFF4xav7NzwGoz02s07mhv77fKtqwt1M8FgiDi9PkGJEb6SR3HIhzx/TN0N8K1OQ9fGhLxwKwQh9u/q6z53rEsY9LBlckl+Oc//3n0v/fu3YuzZ88aXYDEfKJmekKpYFFW2+OwJeiItLlZ0IoqtAbdCJalUaOlRNcJ2jlOpUBcuDdK6aJpu2HoqIWhJhefj8xGVATdHSI1s5Tghg0bsGPHDnO8FDFBYqQf6lr6MUjzjtgFbW4W9AonfD4yB3PC6AsRqdGRoANIjNZABFBGd4/YPENbNQx153GBS4O7pyf8vJyljiR7VIIOIDbMG06cAsWXqQRtnTZ3L+Dkhn3tszAnnI4CbQGVoANQKljEhXqjuKZT6ihkAvqWChgaitAfvhx9PIs54T5SRyKgEnQYc2f5oL1nBG3dQ1JHIeMQRRH8uY/AOHuigJkLBleGQyPSoxJ0EHNnXTmqoFNi22RoKoWhuRxcyu0oqhtAaIA73Jwd9+J2e0Il6CACvJ3h6+GEYhpMweaIoght7l4wrj4QIm5EVWMvfR5oQ6gEHQTDMJg7yweltV0wCILUccg3GOovQmitApd6BypbBmEQRPo80IZQCTqQubN8MKw1oKbJMW/BskejR4HuGqhil6C4pgtKBYOoYE+po5GvUQk6kNlh3mAAFF+mU2Jboa/Nh9BxGeq0TDCsEkU1XYgN8YJapZA6GvkalaADcXNWITzIgz4XtBGiKIA/lwXGMxDKqIXo7B1BU8cgEiJ8pY5GvoFK0MHMnXVlyP0huoVOcvrqcxC6G6BOWweGVaDo6+s446kEbQqVoIOZG+4DQRRRWtsjdRRZEwUD+NwssN7BUEbOBwBcrO6Cr4caQb4uEqcj30Ql6GAiZ3pCrVKghD4XlJS+6isIvS3g0teBYVjoDQJKLnchPsIXDENDZ9kSKkEHc+UWOi9crO6EKFpn0Foylijooc3bB9Y3DMrwNADApcZejPAGxM+iU2FbQyXogBIjfdHRO4LmTrqFTgq6ilMQ+9uhnrd+9KivqKYLCpahi6RtEJWgA7o6wnThJRpQwdpEgw583n6w/hFQhCSNPn6xuhNRMz3hrDZ5MHdiIVSCDsjX0wkzNa4opKk4rU5XdhLiYBfU6XeOHgX2DmhR1zqA+Ai6S8QWUQk6qMQIX1Q29GJYq5c6imyIeh58wSEogmKhmDln9PGir6/bpOsDbROVoINKjPSFQRDpwmkr0pV8CnGoB1z6hjHfAF+s7oSnK4cQf+tOM0uMQyXooKKCr3z+RJ8LWoeoGwF//mMoZs6FMih29HGDIKC4pgvxs3zo0hgbRSXooBQsi/hZPrhY3QmBLpWxOL7oE4gj/VDP2zDm8aqGXgyO6JEcTdOh2ioqQQeWGOmL3kEeda00qowlifwQ+MIjUIQmQeEfOea5gsoOKBXM6KC3xPZQCTqwhAhfMAAKq+iU2JL4wn8B2kGo0zdc89yFqg7EhXrDiaNLY2wVlaAD83DlEB7kgcJqKkFLEUcGwF88BmV4GhR+YWOea+4cRGv3MJ0K2zgqQQeXFOmLmqY+9A3yUkdxSHzhUUA3Ai59/TXPna+6cp1mUiSVoC2jEnRwSVF+EHHltIyYlzDcB77oGJSRN0DhE3zN8xcqOxDi7wZfTycJ0hFjUQk6uNAAN/h6OKGgkkrQ3PjzHwMGHdRp6655bmBYh8rGXiRF0VGgraMSdHAMwyAlxg9FNV0Y4enuEXMRBruhK/kUyujFYL0Cr3m+8FIHRBFIoc8DbR6VoAykRmugNwh094gZ8QWHAEGAOjVz3OfPV125SyQs0N3KychUUQnKQHSIJ1ydlMivaJc6ikMQ+jugK/scqrgbwXpornlebxBQVN2JpChfsHSXiM2jEpQBBcsiOcoPF6o6oTfQnMTTxRccAMCAS1k77vNltd0Y4Q1Ijrq2IIntoRKUiZQYDYa0elTU09wj0yH0tkJXng3VnJvBuo0/KkxueTvUnAJzZ9EAqvaASlAm5s7yAadkUVBB3xJPhzZ/P8AqwSVnjPu8IIgoqGxHUqQvVEqaW9geUAnKhFqlwNxZPsivbKe5R0xk6GmCvuo0VHNvAeviNe4yFfU96B/SIS3W38rpiKmoBGUkNUaD7n4tamlABZPwufsApRpc0prrLpNX3g6VkkUCjSJtN6gEZSQpyg8swyCvnL4lnipDZz301WfBxa8E6+wx7jKCKCKvog3xs3xowAQ7QiUoI27OKsSFeeFcWRudEk8Rn5cFcM7gEldfd5mapj70DPBIp1Nhu0IlKDPzZwegrXsYda0DUkexG4b2y9BfzgeXuBqM2vW6y+WVt0PBMkiKorlE7AmVoMykxmjAMgzOlbVJHcVuaHP3AmpXcPG3XncZURSRW96GOeE+cHFSWTEdmS4qQZlxc1ZhTrg3zpW10imxEQwtlTDUF4JLygDDOV93ubrWAXT0jiAtli6QtjdUgjI0L84f7T0j9C2xEbS5e8E4e4Cbu3zC5c6UtkLBMkiNoRK0N1SCMpQSo4GCZXCulE6JJ6JvKoWhqRRccgYYlfq6ywmiiLOlrZg7ywduznQqbG+oBGXoyimxD31LPAFRFMHnZoFx9YZq9s0TLlvV0IuuPi1umBNgpXTEnKgEZWr+bH909I7gcgudEo/H0FAEQ0sFuJS1YJTchMueKW0Fp2Rp7EA7RSUoUynRflCwDM6UtEodxeaIonjls0A3X6hil064rEEQkFvWhqQoP7pA2k5RCcqUi5MKiZG+OFPSCkGgU+JvMtSeh9BeA3VqJhjFxMVWerkb/UM6OhW2Y9MqwV27diEjIwMZGRl44403zJWJWMnCuYHoHeRRUksjTl8ligK0eXvBeARAGbN40uXPlLTCWa1EQgRdIG2vTC7BnJwcZGdnIysrC/v27UNxcTGOHz9uzmzEwpKi/OCiVuJ0UYvUUWyGviYPQmc91GmZYNiJh8LS6Q3Ir2xHWowGKiWdVNkrk985jUaDbdu2geM4qFQqREZGoqmpyZzZiIWplCzmzfZHXkU7TcIEQBQE8LlZYL1nQBm5YNLlL1R1YlhroFNhO2fyJ7nR0dGj/3358mUcOXIEf/3rX41e39fXzdRNm0SjcewJb0zdv9sWR+Dk+SZUNg/glvQQM6cyH2u8f/1FX2Cgpwn+G34MtwDPSZc/d6AYPh5OuDE9FAp2enOJOPLvp63v27S/zqqsrMQjjzyCZ599FuHh4Uav19k5YLUP5DUad7S3O+6lINPZPz83Ffw8nXDsdA0SwsYfKFRq1nj/RMGAwc/+BtY3BEO+czA8yfZ6B7TIK23D6htC0dU5vcEoHPn305r7xrKMSQdX0/ogIy8vDw8++CB+9KMfYf369dN5KSIRlmGwYG4gSmq70d2vlTqOZPQVpyD2tUKdtgEMM/mfxeniVgiiiMUJ1845TOyLySXY3NyMxx57DG+++SYyMsafb4HYh0XxgRBFyPaaQdGghzZ/P1jNLCjCkidfXhRx6mIzImd6IMj3+kNrEftgcgm+++670Gq12LFjBzIzM5GZmTmlzwSJ7Qj0cUHkDA9kX2yW5W10uvIvIA50Qp2+AYwR8wRfbulHY8cgFicEWSEdsTSTPxPcvn07tm/fbs4sREI3Js3A7iNluNTYh6jgyb8UcBSingeffwCKwBgoguONWif7YjNUShbz42gEaUdAFzcRAFfuJVZzCpy80Ch1FKvSlX4GcagHXPp6o44CdXoDzpa0IjVGQ4OnOggqQQIAcOKUuGF2AM6VtWFoRB7XDIo6LfjzH0MxYzaUM2YbtU5BZQcGR/T0hYgDoRIko5YlzwCvE3CmVB5fkPDFJyAO90GdvsHodT4vaISvhxPmhNGUmo6CSpCMCg90R4i/G7447/h3/oj8MPgLH0MRkghFYPTkKwBo7hxEWV0PbkqZAXaaF0cT20ElSEYxDIOlSTNQ29qPWgcfZ5AvOgZoB6FON/761pPnm6BgGSxJnGHBZMTaqATJGAvmBkClZHHyguMeDYraQfCFR6EMT4VCM8uodXidAacuNiM1RgNP14kHWSX2hUqQjOHqpML82f44XdTisF+Q8IVHAX4YXJrxR4HnytowOKLHzSkzLZiMSIFKkFxjeVowtDoDsi82Sx3F7IThPvBFx6GMmA+Fr/EDRnxe0IggXxfEhtrm/dXEdFSC5BrhgR6ImumJT/MaIDjYHST8hcOAXgsufZ3R69S29ONSUx+WJc806lpCYl+oBMm4lqcFo61nGBcvdUodxWyEoR7oij+FMmohFF7Gf7lx7Fw91JwCS+g2OYdEJUjGlRargZcbhxN5DVJHMRv+/MeAoIc6zfijwO5+Lc6WtuLGhCC4ONFESo6ISpCMS6lgcVPKTBTVdKG5c1DqONMmDHRCV/IZVLFLwHoYf8/vp/kNEAQRK9KDLZiOSIlKkFzXsuSZUCoYHM+1/6NBvuAgAIBLzTR6Ha3OgM8LGpESo4G/t4ulohGJUQmS6/J05bAoPhDZhc3oHeSljmMyoa8NurIvoYpbBtbN+FnhcopaMDiix63zbHfaATJ9VIJkQqtvCIPBIOCT3Hqpo5hMm38AYFlwKbcbvY4gijh+rh5hge6IltHQYnJEJUgmFOjjgtRYDT7Nb8Sw1v4unhZ6WqCvPAXVnFvAunobvV5+eTtauoawen4oXRbj4KgEyaTWLAjDsFaPk3Y4sII2bx+g4MAlGz8FhCiKOJRzGQE+LphHA6c6PCpBMqlZQR6YHeaNf52rg04vSB3HaIauBugvnQEXvxKss4fR6xVe6kRd2wBuXxhGo8XIAJUgMcqaBWHoHeDt6lY6PjcLUDmBS1xt9DqiKOJgzmX4eTrRpOoyQSVIjDIn3BtRMz1xKOcydHqD1HEmZeiohf5yHriEW8E4GT8XbWltN6qb+rBmQRiUCvrzkAN6l4lRGIbB+htnobtfi8/t4LNBbe5eQO0KLnGV0euIoogD2TXwcuNoJjkZoRIkRpsd7oO4UC98fLoWWp3tHg0aWqtgqLsALvE2MJzxFzlfrO5CRUMvbl8UDpWS/jTkgt5pMiXrl0agb5DHp/m2exeJNjcLjJM7uPgVRq8jiCI+OnkJ/l7OWJpEI0fLCZUgmZLoYC/ER/jgyFd1Njnoqr65HIbGYnDJGWBUTkavd7akFfVtA1i3dBZ9Figz9G6TKbtzaSQGh3U4dPqy1FHGEEURfO5eMC5eUM25xej19AYBWV9WI8TfDfNn0zfCckMlSKYsLNAdixIC8UluPdp6hqWOM8rQWAJDczm4lNvBKI2fB+TTvAa094zgzmWRYOnuENmhEiQm2bA0EgqWxZ7PqqSOAuDKUaA29yMwrj5QxS0zer3eQR77T9UgPsIHCRE0l7AcUQkSk3i7q3HbglDklbejvK5b6jgw1F+A0FYNLvUOMAqV0et9dPISeJ2Ae5dH0z3CMkUlSEy2an4ofD3UeP94BfQG6W6nE0UR2nNZYNw1UMUuMXq96qY+ZBc2Y+W8EAT5ulowIbFlVILEZGqVAvetjEVj+yD+dbZOshz6y3kQOmuhTlsHhjVuCHyDIOD9Y+XwdOWwdlG4ZQMSm0YlSKYlOdoPabEa7M++jNbuIatvXxQE8LlZYL2CoIxaaPR6x87V43JLP+5dEQ1nNc0dImdUgmTavrMiBiolg78cLYdo5Sk69dVnIXQ3gktbB4Y17te5pWsI+76sQUq0Hw2VRagEyfR5u6uxcVkkSmu78VlBo9W2KwoGaPP2gfUJhjJinlHrCKKI3YdLoVKw+O6qWPoyhFAJEvO4KWUm4mf54O+fVqGpwzqz0+mrTkPsbQGXvh4MY9yv8rGz9aho6MU9y6Pg5aa2cEJiD6gEiVkwDIOHMmZDrVLgnQPFFh98VRT00ObtB+sXDmVYqlHr1DT34aOTl5Aao6GJ1MkoKkFiNl5uanxvTRzq2gbwz88vWXRbuvJsiP3tUKdvMOqUdlirxx/2F8HLjcP31sTRaTAZRSVIzColWoPlqcE4nluPr4pbLLINUc+Dzz8ANiAKipCESZcXRBF/OlSCzl4t/uOOuXB1Mv5iauL4qASJ2d2zPAoxIV7485Ey1Lb0m/31dWUnIQ52GX0UuO/LahRUduDuW6IQHexl9jzEvlEJErNTKlj8YF083JxV+M1HhejsHTHba4t6LfiCQ1AExUExY/aky39V3IJDObVYmhSElenBZstBHAeVILEID1cOT25MxAivx3//4zz6h3izvK6u+FOIw73g5k1+FFh4qQPvflyKmBAv3H8rXQ5DxkclSCwmNMAdT9yZiI7eEfx6zwUMjeim9XoiPwz+wmEoguOhDIyZcNmy2m78LqsIwRo3PHFnIg2USq6LfjOIRcWGemNrZjzqWgfwxocF6Bs0/YiQLzoOcaQf6vQNEy53sboTv/7nBWi8nPH0PUlwcaLb4sj1UQkSi0uO9sOTGxPR0jWE1z7IR5sJ9xiL2kHwhUehCE2Gwj/iusvlFDXjN/8sRIC3C57ZlAx3F+MHVyXyRCVIrCI+whc/2pSMgSEeP9udi8JLHVNan794DOCHoE5fP+7zeoOAv35SiT8dKkV0sCee+04qPOmOEGIEKkFiNdHBXnjpwXnw83TC/+wpxIfHKzDCTz5Zk2GoH/zFf0E5Kx0Kv7Brnm9sH8COD/JxPLceK9KC8fQ9yXQKTIw2rd+UgwcP4q233oJer8fmzZtx3333mSsXcVAaL2c8/900/POzS/gkrwHnqzqwYWkE5s8OAMuO/+1tz1f7AJ0W3LeOAgeGdThyphbHztbDWa3Eo5lzaaIkMmUml2Brayt27tyJvXv3guM4bNq0CTfccAOioqLMmY84ILVKgftujcG82f54/1gF3jlYgkOna3FT8gwsmBsIN+d/39EhDPViIPcIlFELoPCeCVEUUd82gJyiFpy80ASeN2BRfCDuviWKPv8jJjG5BHNycrBgwQJ4eV25An/VqlU4evQoHn/8cbOFI44tJsQLLz80D7llbTjyVR0+/KQSfztRhbBAN0TO8ISflzMim49Co9fhKyYdNQdLUNnQg47eEShYBmmxGqxdFI6ZGjepd4XYMZNLsK2tDRqNZvRnf39/FBYWGr2+r691f3E1Gnerbs/a7Hn/Mvw9kLE0CjVNvThV2ISiS5348mIznHT9+C+v0zjDR+CvZ3rh7T6CuHAf3B2jweKkmfBwdZwjP3t+/yZj6/tmcgkKgjDmCnxRFKd0RX5n5wAEwTqjEGs07mhvN/89rLbCUfbPTcViVVowVqUFQxRFDJz8X4hVDG556AdYynlCrVKMLqsd0qJ9SCthWvNxlPdvPNbcN5ZlTDq4Mvnb4cDAQLS3t4/+3N7eDn9/GqqcmIc40AlUfQkudikCw8PGFCAh5mRyCS5atAinT59GV1cXhoeHcezYMSxdutSc2YiM8fkHAIYBl7JW6ijEwZl8OhwQEICnnnoKDzzwAHQ6HTZu3IjExERzZiMyJfS2QleRDdXc5WDdfKSOQxzctK4TXLt2LdaupX+piXlp8/YBCiW45AypoxAZoDtGiE0xdDdCX/UVuLkrwLrQAKjE8qgEiU3h8/YBKjVUSbdJHYXIBJUgsRmGzjroq8+BS7gVrJNtX1tGHAeVILEZfG4WwLmAS1gldRQiI1SCxCYY2qqhry0Al7gajNpV6jhERqgEiU3Q5u4Fo3YDF79S6ihEZqgEieT0LZUwNBSBS14DhnOWOg6RGSpBIjk+dy8YZw+o5i6XOgqRISpBIil9YwkMTaXgUtaCUdJw+MT6qASJZERRvPJZoKsPVHHLpI5DZIpKkEjGUH8RQmvV10eBjjM2ILEvVIJEEqIoQpuXBcbdD6rYG6WOQ2SMSpBIQl9bAKG9BurUTDAKmhmOSIdKkFidKApXvhH2DIQyepHUcYjMUQkSq9NX50LoaoA6bR0YlkaMJtKiEiRWJQoC+LwssN4zoYycL3UcQqgEiXXpq05D6GkGl7YODEO/fkR69FtIrEYU9NDm7wfrGwblrDSp4xACgEqQWJGu4hTEvjao09fTUSCxGfSbSKxCNOjA5x8A6x8BRWiS1HEIGUUlSKxCV/YFxIFOqNM3gGEYqeMQMopKkFicqOfBFxyEIjAGiplzpY5DyBhUgsTidCWfQRzqATfvTjoKJDaHSpBYlKgbAX/+EBQz50IZFCt1HEKuQSVILIov/gTiSD/U6euljkLIuKgEicWI/BD4C0egCE2CIiBK6jiEjItKkFgMf/EYoB2ko0Bi06gEiUWIIwPgC/8FZXgaFH7hUsch5LqoBIlF8IVHAd0IODoKJDaOSpCYnTDcB77oOJSR86HwCZY6DiETohIkZsdfOAwYeKjT1kkdhZBJUQkSsxIGu6ErPgFl9CKwXkFSxyFkUlSCxKz484cAQYA6NVPqKIQYhUqQmI0w0Ald6UmoYm8E6+EvdRxCjEIlSMyGzz8AAOBS10qchBDjUQkSsxD62qArz4Zq9k1g3XyljkOI0agEiVlo8/YDLAsu5XapoxAyJVSCZNoMPU3QV+VANXc5WBcvqeMQMiVUgmTa+Nx9gIIDl7RG6iiETBmVIJkWQ1c99NVnwcWvBOvsIXUcQqaMSpBMC5+7D+CcwSXdJnUUQkxCJUhMZmi/DP3lPHAJq8GoXaWOQ4hJqASJybS5ewG1K7iEW6WOQojJqASJSQytVTDUF4JLWgOGc5Y6DiEmoxIkJtHm7gXj7AFu7gqpoxAyLSaXYF5eHjZu3IjMzExs3rwZjY2N5sxFbJi+qRSGxhJwyRlgVGqp4xAyLSaX4DPPPINXX30V+/fvx9q1a/Hqq6+aMxexUaIogs/NAuPiBdXsm6WOQ8i0mVSCPM/jySefRFxcHAAgNjYWzc3NZg1GbJOhsRiGlgpwKWvBKDmp4xAybSaVIMdxyMy8Ml6cIAjYtWsXVqygz4YcnSiK0J7bC8bNF6q4pVLHIcQsGFEUxYkWOHLkCF577bUxj0VERGD37t3geR7btm1Db28v/vCHP0ClUlk0LJHWYMU5tO7ZAb+MrfBIpn/0iGOYtASvZ3BwEFu3boWXlxfefPNNcNzUTo06OwcgCCZteso0Gne0t/ebSTp0AAAUsUlEQVRbZVtSsMb+iaKAob0vQ9Rp4Xr3z8GwSotu75vo/bNf1tw3lmXg6+s29fVM3eAzzzyDsLAw/PrXv55yARL7o6/Jg9BZB3VaplULkBBLM+m3uaSkBCdOnEBUVBTWr78yr6y/vz/++Mc/mjUcsQ2iIIDPywLrNQPKyAVSxyHErEwqwTlz5qC8vNzcWYiN0lefgdDdBKcVPwDD0vX1xLHQbzSZkCgYoM3bB9YnBMpZ6VLHIcTsqATJhPSVORB7W6FO3wCGoV8X4njot5pcl2jQXzkK1MyCIixZ6jiEWASVILkuXfkXEAc6oU5fD4ZhpI5DiEVQCZJxiXoefMFBKAKioQhOkDoOIRZDJUjGpSv9HOJgN7h5G+gokDg0KkFyDVGnBX/+EBQzZkM5Y7bUcQixKCpBcg2++ATE4T6o0zdIHYUQi6MSJGOI/DB0Fw5DEZIARWC01HEIsTgqQTIGX3QconaAjgKJbFAJklGidhB84REow1Kg0MySOg4hVkElSEbxhUcBfhgcHQUSGaESJAAAYaQffNFxKCPmQ+EbInUcQqyGSpAAAPjzhwG9FlzaOqmjEGJVVIIEwlAPdMUnoIxaCIX3DKnjEGJVVIIE/PmPAUEPdWqm1FEIsToqQZkTBrqgK/kMqpglYD0DpI5DiNVRCcocX3AQgAgu9Q6poxAiCSpBGRP62qEr+wKquGVg3f2kjkOIJKgEZUybfwBgWXApa6WOQohkqARlSuhpgb4yG6o5t4B19ZY6DiGSoRKUKW3+PkChApecIXUUQiRFs2jLkKGrEfqqM+CS14B19pA6jlmJooju7nbw/AgAUeo4RmlrYyEIgtQxLMIy+8aA45zg7a0xy4C/VIIyxOdlASo1uMTbpI5idgMDvWAYBgEBwXYzO55SyUKvd8wStMS+iaKAnp4ODAz0wt3da9qvZx+/JcRsDB210NfkgktYBcbJTeo4Zjc8PAB3dy+7KUAydQzDwt3dG8PDA2Z5PfpNkRlt7l5A7QoucZXUUSxCEAxQKOgEx9EpFEoIgsEsr0UlKCOGtksw1F0Al7gaDOcidRyLoYmhHJ8532MqQRnR5maBcXIHF79S6iiE2AwqQZnQN5fD0FAELnkNGJWT1HEIsRlUgjIgiiL43L1gXLygmnOL1HGIzO3e/Sfcf//d6OvrkzoKALpERhYMjSUwNJdDveh+MEq11HGIjNXUVKOxsQE/+ckreOut3+K55168ZpkLF87j8OED0Ol0cHNzw9NPP2fRTHQk6OBEUYQ2dy8YVx+oZi+TOo7s5Ofn4vHH/0PqGEZ74YVnUFpabLHXnzUrAi+++DKio2Px3HMvgud5bNnyXQwM/Ptyl6SkZDz//Et46aVX0NbWiqGhIYvlAagEHZ6hvhBC2yVwqXeAUaikjkNsWHFxEUZGhjF79lyrbZPjOKxadRv+/vcPrnkuJycbYWGz4OJi2SsZqAQd2OhRoLsGqtglUseRvb/85T3cf/9deOCBe/Db3+6EwWBAW1srtm79PrZs+S4efvgBFBVdBAC0tbXi8cf/45rHv2loaAi/+c2v8NBD9+H+++/GSy89P618Bw7sxcqVq0d//sUvfor33nsHAFBfX4dNmzagvLzM5Ne/3uutWLEKhw7tH7Ps4cMHUVpajK1bf2jy9oxFnwk6MP3lfAgdtXC66ftgWPm91acuNiO7sNkir70kMQiLE4KMXv706VPIzv4Cf/rT/0GpVGL79mexb99H6O3tweLFN2LTpu/iq69yUFh4HvHxCTh0aD8WLVqC73zngTGPXyWKIl544ceYOzcB7777PhiGQUdH+5htbt/+LBoaGsbN8/bb70GtHnuVQEFBHr7znQdGf3744a3YsuW7WLJkKX72s//C88+/hNjYOKP3+dsmej2lUoXa2ssICwvHqVNf4o9/fAuLFi3BL3/5C3z/+1vh7W25kY7k95chE6IogM/NAusZCGXUQqnjyF5e3jmsWLEKTk5Xiicj4w4cOfIxNm26Dy+++CzKysqwaNES3Hnn3QCA9PT5ePHFZ1FRUT7m8atyc8+gp6cH3//+o6MXDvv5acYs8+qrb0wpY3t7G7y9fUZ/1mj8sXp1Bh577GG8+uobSEpKvmadhx66D62tLeO+3oEDx6BU/vtkc6LX8/X1RVtbK8LCwrF48Y1YvPjGKWWfDipBB6W/dBZCdwOcbnkUDKuQOo4kFidM7WjNkkRR+NbPgMGgR2JiMv72t3/iiy++wIkTx3D48EH8+te/R2JiMt5//x/Iycke8/hV5eVlSEhImvDOiakeCarVavA8P/pzd3cXvvrqFJycnOHvP/78M++9d+1nedcz0evxvBZqtTRXLlAJOiBRMIDP2wfWOxjKyPlSxyEAUlPn4X//911kZq6HQqHE4cMHkJqajt///n8QEBCAO+/chJSUdDz00H0AgN///n/g5+ePu+++d8zjV/n5aXD69Cno9XoolUp0dnbA13fsFAlTPRKMiIhCXd1l+Pn5ob+/Hz/60RN46KH/QE9PN95667d4442dJu//RK9nMBjQ1NSIiIgok19/OqgEHZC+6jSE3hY4rfwhjaZiIxYvvhGVleXYsuUBGAx6zJ+/AHfeeQ86Ozvws59tx8GDB8CyLLZv/ykA4M4778FPf7odhw8fHPP4VStWrEJ+fi7uv/9uODk5ISQkFK+8smNaGZctuxlnz36FOXPi8eyz/4kNGzbippuWQ6/X4x//+Cvy83ORmpo+5dcdGRmZ8PUuXryAOXPi4eYmzahGjCiKkow82dk5AEGwzqY1Gne0t/dbZVtS+Ob+iYIeg39/HozaBS7rX3aIwQSm8v61tNQiMDDMwonMy1bGExwcHMAPfvB9vPPO7mtOlU1lzL69/PKLyMi4A/Pm3TCl1/72e82yDHx9p16kdJjgYHTl2RD726FO3+AQBUisx9XVDY8//p9oamqy2jZ5nkdycsqUC9Cc6HTYgYgGHfj8A2D9I6EISZQ6DrFD8+YtsOr2OI7DunUbrbrNb6MjQQeiKz0JcbAL6nl30lEgIUaiEnQQol4LvuAgFEGxUMyYLXUcQuwGlaCD0JV8CnG4Fxx9FkjIlFAJOgCBHwZ//jAUwfFQBsVKHYcQuzLtEiwpKUF8fLw5shAT9Z47DHGkH+r0DVJHIcTuTKsEh4eH8corr0Cn05krD5kiUTuI3q/2QxGaDIV/hNRxCLE70yrBHTt2YPPmzebKQkzAXzwGYWQQ6vT1UkchxC6ZfJ3giRMnMDIygtWrV0++8DhMubJ7OjQad6tuzxoMQ/2oKzoG17gFCJjt2B9JGPv+tbWxY0YusRf2mNlYlto3lmXN8nc9aQkeOXIEr7322pjHIiIiMDAwgN27d5u8Ybptbvq0Z/dA5EfgvfQeh9y/q6by/gmCYBO3oE2Frdw2NxFRFPHjHz+JX/3qN1Naz5L7JgjCmN8LU2+bm7QEb7vtNtx2221jHtuzZw/efvtt3Hffv0e2yMzMxAcffCDZTdByIwz1gi86DmXUDeA0oYADlyCRhiiKeOut34BlFRgZGcbMmTMBAO+99w4GBwcxNDSI//zPH5vtPmOpmHQ6fNddd+Guu+4a/Tk2Nhb79++fYA1ibvyFw4BBD3XqOqmjECv5+c9fRkpKGtasWTvm8fz8XLz33jvYtesds27v6NGPERs7B8uXr8S7776N0NAwHDt2FP7+Abj99kyzbktKjvtBhAMTBruhKzkBZfRisF6BUschDqqoqHB0+PvS0mIkJCShsLAAaWnzJE5mXmYZQKG8vNwcL0OMxBccBAQR6tQ7pI5CjPCHP+zC55+fgKenF3x9/bBkyVIEBgbhrbd+A4NBQHj4LKjValRXX0JXVxeioqLw8ss/B8epsWvXTpw6lQ0/Pz8IgoCUlLQJt/WXv7yHY8eOgGVZzJu3AD/4wRNfj1n4XxgeHgbLMnjyyWcQH5+AtrbWcR+/auHCJXjrrd8gNDQcbW2tCAwMwrJlN+MPf9iFgIAALF68FElJKZb+32dxNIqMnRH6O6ArOwlV3FKwHprJV5AxXcUp6Mq/sMhrq2KXQhWzeNLlsrO/QGHhefzf//0Dw8PD2LLlfixZshTAlRnX/vnPQ6ipqcSxY8fw9tt/hiAIeOKJR3H69CmIooiKinK8//4/0N/fjwcf3DThtiaazGm8SZsmm8xpyZKlo1kfeeQxAFdGmbH2SDOWRiVoZ/j8AwDDgEtZO/nCRHK5uWdwyy0roFKpoFKpcOONy0afCwkJg5ubG1JS0uDq6oGPPvoH6uouo6GhHsPDwygtLcayZTdDqVTC29sbCxZMXLqTTeb07UmbJpvMSS6oBO2I0NsKXUU2VHOXg3XzmXwFmVPFLDbqaM2SWJa97qVgVycW+uKLk3jnnbdw112bsGbNHejp6YEoimAYBt8c912hmHjCrIkmcxpv0qbJJnNasmRqQ+lnZ+dOaXlbQSVoR7T5+wFWCS45Q+ooxEjp6Tfggw/+F+vXbwTPa5GTk43o6LGDXJw7d+VoMSPjDjQ2NqCgIA/z5s1Hevp8fPjh/yEzcwNGRkZw5sxpxMdff7DciSZzGm/Spskmc7LXUpsqKkE7Yehugr7yNFSJq8G6eEkdhxhp0aIlKCoqxPe+dx88PDzg56cBx42dWjIzcz1eeukFfPLJv6BUqpCQkIimpibcfvs6lJaW4IEH7oGPjy/Cwye+N3yiyZzGm7RpssmcvsnUi6XtAU20ZCeGP/k99PWFcL33l2Cdxt4q5Aj7NxF7nmipqKgQ9fV1uO2226HX6/HII9/D88+/hKio6NFlbPWOkW9fLC0IAh599IfYvv057NjxK/zqVzuwceM91xzZfpMl940mWpIRQ2cd9NVnwcWvvKYAiW0LDQ3D8eP/wubN9+Khh+7D8uW3jilAW3b1YulHH30c7u4eSEhIgouLC269dTV++MNHsGHD3RMWoL2g02E7wOdmAZwzuETTBqsg0vHw8MR///dvpY5hkqKiQtx773cBXLlYOiPjDuh0OuTlnYNarUZgoGNcqE8laOMM7TXQ1xZcGTZf7Sp1HCIj375Y2s9Pgzfe+Dk2btwEQTDg3XffxtNPPyd1zGmjzwRt3NCRX0Foq4Hrvb8EwzmPu4w9758x7PkzQWPY6meC5kCfCZJp0bdUwlB/EVzymusWICFkeqgEbRifuxeMswdUc5dLHYUQh0UlaKP0TaUwNJWCS74djFI9+QqEEJNQCdogURTBn9sLxtUbqtk3SR2HEIdGJWiDDA0XYWitBJeyFoySkzqO3ZHouz5iReZ8j6kEbYwoitDmZoFx94MqdqnUceyOUslhcLCPitCBiaKIwcE+KM10gEDXCdoYfW0BhPYaOC3bAkZBb89UeXtr0N3djoGBHqmjGO3KSDOOeYmMpfZNqeTg7W2e8TTpr8yGiKIAPjcLjGcAlNGLpI5jlxQKJfz8gqSOMSWOfJ2nPewbnQ7bEH1NLoSueqjT1oFhJx47jhBiHlSCNkIUBPC5+8B6z4Ay4gap4xAiG5KdDrMs49Dbmyp93Xmwog7qRQ9AoZz6UaCt79900f7ZL2vtm6nbkezeYUIIsQV0OkwIkTUqQUKIrFEJEkJkjUqQECJrVIKEEFmjEiSEyBqVICFE1qgECSGyRiVICJE12ZVgSUkJ4uPjpY5hVnl5edi4cSMyMzOxefNmNDY2Sh3JLA4ePIg1a9bg1ltvxQcffCB1HLPbtWsXMjIykJGRgTfeeEPqOBbx+uuvY9u2bVLHmJCsSnB4eBivvPIKdDqd1FHM6plnnsGrr76K/fv3Y+3atXj11VeljjRtra2t2LlzJz788EPs27cPf//731FVVSV1LLPJyclBdnY2srKysG/fPhQXF+P48eNSxzKr06dPIysrS+oYk5JVCe7YsQObN2+WOoZZ8TyPJ598EnFxcQCA2NhYNDc3S5xq+nJycrBgwQJ4eXnBxcUFq1atwtGjR6WOZTYajQbbtm0Dx3FQqVSIjIxEU1OT1LHMpqenBzt37sSjjz4qdZRJyaYET5w4gZGREaxevVrqKGbFcRwyMzMBAIIgYNeuXVixYoXEqaavra0NGs2/Rw729/dHa2urhInMKzo6GsnJyQCAy5cv48iRI1i2bJnEqcznpZdewlNPPQUPDw+po0zK4UaWPnLkCF577bUxj0VERGBgYAC7d++WJpSZXG/fdu/eDZ7nsW3bNuj1ejzyyCMSJTQfQRDAMP8eGkkUxTE/O4rKyko88sgjePbZZxEeHi51HLPYs2cPgoKCsHDhQuzdu1fqOJOSxVBae/bswdtvvw1XV1cAQFlZGeLi4vDBBx/Azc1N4nTTNzg4iK1bt8LLywtvvvkmOM7+Z6jLyspCbm4ufv7znwMAfve730EURTz++OMSJzOfvLw8PPHEE3jhhReQkZEhdRyz+d73vof29nYoFAr09vZiaGgI69atwwsvvCB1tPGJMhQTEyN1BLPaunWruH37dtFgMEgdxWxaWlrEm2++Wezs7BSHhobEO+64Q7xw4YLUscymqalJvOGGG8ScnBypo1jURx99JD733HNSx5iQw50Oy01JSQlOnDiBqKgorF+/HsCVz8/++Mc/SpxsegICAvDUU0/hgQcegE6nw8aNG5GYmCh1LLN59913odVqsWPHjtHHNm3ahHvvvVfCVPIki9NhQgi5Htl8O0wIIeOhEiSEyBqVICFE1qgECSGyRiVICJE1KkFCiKxRCRJCZI1KkBAia1SCxCa98cYbeOyxx0Z/fv3117F582aHGwuSSI9umyM26eGHH8bKlStRWlqK8+fPIzs7Gx9++CFUKpXU0YiDodvmiM367W9/i2PHjmFgYAAffvghgoKCpI5EHBCdDhObNXv2bFRUVODpp5+mAiQWQ0eCxCaVl5djy5YtWLJkCVpaWux+QFxiu+hIkNic1tZWbN26FT/96U/xk5/8BBUVFThz5ozUsYiDohIkNmVgYAAPP/wwHnzwQSxfvhzOzs7YsmULdu7cKXU04qDodJgQImt0JEgIkTUqQUKIrFEJEkJkjUqQECJrVIKEEFmjEiSEyBqVICFE1v4fw0hBNZENMPUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Compute the range of values that we wish to explore\n",
    "xs = np.arange(-5.0, 5.0, 0.01).astype(np.float32)\n",
    "\n",
    "# Use squared error loss:\n",
    "regr_sqr_loss = [f_regr_sqr_loss(x, 0.0) for x in xs]\n",
    "\n",
    "# Compute gradient of squared error with respect to x-hat\n",
    "regr_sqr_loss_grad = [f_regr_sqr_loss_grad(x, 0.0) for x in xs]\n",
    "\n",
    "plt.figure(figsize=(5,5))\n",
    "plt.xlim(-5.0, 5.0)\n",
    "plt.ylim(-5.0, 5.0)\n",
    "line_loss, = plt.plot(xs, regr_sqr_loss, label=r'loss $c = (x - \\hat{x})^2$')\n",
    "line_loss_grad, = plt.plot(xs, regr_sqr_loss_grad, label=r'grad loss $\\frac{dc}{dx}$')\n",
    "plt.legend(handles=[line_loss, line_loss_grad])\n",
    "plt.xlabel(r'$x$')\n",
    "plt.show()\n",
    "\n",
    "if SAVE_PLOTS:\n",
    "    plt.figure(figsize=(5, 5))\n",
    "    plt.xlim(-5.0, 5.0)\n",
    "    plt.ylim(-5.0, 5.0)\n",
    "    line_loss, = plt.plot(xs, regr_sqr_loss, label=r'loss $c = (x - \\hat{x})^2$')\n",
    "    line_loss_grad, = plt.plot(xs, regr_sqr_loss_grad, label=r'grad loss $\\frac{dc}{dx}$')\n",
    "    plt.legend(handles=[line_loss, line_loss_grad])\n",
    "    plt.xlabel(r'$x$')\n",
    "    plt.savefig('regr_sqr_loss_2.png', dpi=600)\n",
    "    plt.close()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regression: no non-linearity and Huber loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use PyTorch `smooth_l1_loss`\n",
    "def f_regr_huber_loss(predictions, targets, delta=1.0):\n",
    "    t_predictions = torch.tensor(predictions, requires_grad=True)\n",
    "    t_targets = torch.tensor(targets, requires_grad=True)\n",
    "    # Compute squared error\n",
    "    return F.smooth_l1_loss(t_predictions, t_targets)\n",
    "\n",
    "def f_regr_huber_loss_grad(predictions, targets, delta=1.0):\n",
    "    t_predictions = torch.tensor(predictions, requires_grad=True)\n",
    "    t_targets = torch.tensor(targets, requires_grad=True)\n",
    "    # Compute squared error\n",
    "    loss = F.smooth_l1_loss(t_predictions, t_targets)\n",
    "    # Sum and compute gradient\n",
    "    loss.sum().backward()\n",
    "    return t_predictions.grad.detach().numpy()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUEAAAFGCAYAAAD0Cy2JAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XdgFGX+x/H3lmwKKYQUgvQQSCgJJICiIBZASsQI0lFQEcGCCtL0PO9sJ02x4CGeKD8VEFBAQOFArBRFSEIoIfSaQEJIz6bszvz+4OTkpKRsdjY739d/WXbm+TxJ+GR2d+YZg6qqKkIIoVNGrQMIIYSWpASFELomJSiE0DUpQSGErkkJCiF0TUpQCKFrUoJCCF2TEhRC6JqUoBBC16QEhRC6JiUohNA1KUEhhK5JCQohdM2s1cA5OUUoinMWsAkK8iU7u9ApY2lB5le7ufP8nDk3o9FAYGCdSm+nWQkqiuq0Evx9PHcm86vd3Hl+rj43eTkshNA1KUEhhK5JCQohdE1KUAiha1KCQghdkxIUQuialKAQQtekBIUQuiYlKITQNc1KsLTcrtXQQgg3dCG/pErbaVaCH65LpbjEptXwQgg3cuh0LnOX767StpqV4OnMAt5YlkRRSblWEYQQbuDAiRzeXLYbvzqWKm2vWQne3zuSU5mFzF6aREFxmVYxhBC12L7jF3hrxW6CArx4LKFtlfahWQm2bVaPCffFkH6+mNlLk8gvkiIUQlRcypHzvL0ihdBAH6aOiMXPp5YdCQJEhwfxzOAYMnOszFySSG5hqZZxhBC1RNLBLN79cg8Ng+swdUQs/lUsQHCBU2TaNKvHxCHtuZBfyszFiVX+hEcIoQ87D2Tyz9V7aRrmx5ThHfD19qjW/jQvQYDIJoE8O7QD+cVlzFicyPlcq9aRhBAu6Jd9Z3n/q300v8GfZ4d2wMeregUILlKCABGNApg8LJbiEhszlySSmVOsdSQhhAvZuieDf63dT6vGAUwa0h5vT8csjO8yJQjQvIE/U0fEUlquMGNxIhnZRVpHEkK4gB+Tz/DR16m0aRbI04Pb42Vx3J1BXKoEAZrU92PqiFgURWXmkiROZ7nnDWiEEBWzeddp/m9DGtEtgnhqUAyeHiaH7t8hJThz5kymT5/uiF0B0CjEl2kj4zAYYNaSJE6eK3DYvoUQtcfGHSdZvOkgsS2DeWJANB5mxxYgOKAEt2/fzqpVqxyR5TINguowfUQcHmYjs5cmcSwj3+FjCCFc19fbj/P5d4fpFBnCY/e2w8NcMy9cq7XX3Nxc5s6dy/jx4x2V5zL16/kwfWQc3p5m5nyexJEzeTUyjhDCdaiqypotx/jyx6N0aVOfcQltMZtq7p27au35xRdfZOLEifj7+zsqz5+E1PVm2og4/LwtzFmWzMFTuTU2lhBCW6qqsvKno6zecoyu0WE8cncbTMaa/ejCoKpqle6MvGLFCg4fPsxzzz3HypUr2bFjBzNmzHB0vkuy86z8Zf42zudZ+evDN9G+ZUiNjSWEcD5VVfl43X5W/XCY3l2a8vh97TEaDTU+bpVL8KGHHiIrKwuTyUReXh7FxcXce++9PP/88xXaPju7sNJ3ps8rKmPO0iQyc61MGBhNu/CgCm0XEuJHVpb7frgi86vd3Hl+FZ2bqqos/fYQ3+46zZ1xDRnRqxVGQ+UK0Gg0EBTkW+mMVS7BP6rKkWBVShCgoLiMNz5PJj27iMcHRNMhIvi627jzLxnI/Go7d55fReamqCqf/TuNH5LTuatzY4beGYGhkgUIVS9BlztP8Hr8fCxMHh5LoxBf3lu5h11pWVpHEkJUkaKoLPrmAD8kpxN/c9MqF2B1OKQEBw4cWKPvB/4vX28PJg+LpVkDP+av3suO1HNOG1sI4Rh2ReHDr/ezZU8G93RtxsDu4U4vQKiFR4K/8/EyM2lIByIaBbBgzT627snQOpIQooJsdoUP1uznl33nGNg9nHtv1aYAoRaXIIC3p5mJg9sT1SSQj75O5afd6VpHEkJch82u8P5X+/jtQCZD7ojg7luaaZqnVpcggKfFxNODYmgbXo9F6w/wXeJprSMJIa6i3GZn3so9JB7MYkTPlvS5qYnWkWp/CQJYPExMGBhDh4hgPtt4kI2/ndI6khDif5SW23nnyz2kHMlmVO9IenZqrHUkwE1KEMDDbOTxAe3oGBnC55sP8c0vJ7SOJIT4j9IyO2+v2M3+Yxd4qF8Ut8c21DrSJW5TggBmk5HxCW25sXUoX/xwhDVbj2kdSQjds5baeHN5Mmmncnnk7jbcGnOD1pEu47iVCV2EyWjk0f4XL7he/fMxbHaFRwe21zqWELpUaC3nzWXJHMsoYNw9bbmxdX2tI/2J25UgXDxz/OH41phNBtZtO4GHxYO7b2qs2UfwQuhRobWcf3y2i+NnC3h8QDviWrnm9f5uWYIARoOBUX2iMJuMrPrhMAUFJQzv2VKKUAgnyP/P5a0Z2cU8OTCa9hW4vFUrbluCcLEIR/Zqhb+fF6t/PILNrnB/78hKX5gthKi4vMJS5nyeTGaulb+OuYnG9by1jnRNbl2CAAaDgYf7t6W8zMbX209Qbld4qG9rpyzRI4Te5BSUMntpEhcKSnhmUAxxkaEuvziE25cgXCzCgd3DMZuMfLXlGHa7ypi7W9f4Yo1C6MmF/BJmLU0ir6iMSUM60KpxXa0jVYguShAuFmFCt+aYTQa+/PEoNkXl0f5tanTZbiH0IivXyuylSRSVlDN5aAdaNAzQOlKF6aYEfxd/czPMJiPLvjuM3a4wPqHmbuAihB6cyylm9tIkSsvsTB4WS/MGNXe7jZqgy//9vW9swsherUg6dJ73Vu2h3GbXOpIQtVJGdhEzFidSVq4wZXjtK0DQaQkC9OjYiFF9ItlzJJu3v0ihtFyKUIjKOJ1VyMwlSagqTB0RS5P6flpHqhLdliDA7R0a8nB8a1JP5PD2it2UlNm0jiRErXDyXAGzliRhMMC0ERdXeq+tdF2CAF2jGzC2fxsOnsrjzeW7sZZKEQpxLccy8pm9NAmLh5HpI+NoEFRH60jVovsSBOjSJozxCW05lp7PnM+TKSop1zqSEC7pyJk85nyehLenmWkj4qgf6KN1pGqTEvyPTlGhPD6gHacyC5i9NIlCqxShEH908FQuc5Yl4+dtYdqIOELquvaVIBUlJfgHsS1DmHBfDOnni5m1JJH8ojKtIwnhElJP5PDm8mQCfT2ZNjKOoAAvrSM5jJTg/4gOD+KZwTFk5liZuSSR3MJSrSMJoam9x7J5a8VuQgK8mTYyjkA/T60jOZSU4BW0aVaPiUPacyG/lJmLE7mQX6J1JCE0sfvwed75IoWwej5MGRFLQB2L1pEcTkrwKiKbBPLs0A7kF5cxY3Ei53OtWkcSwql2pWUxb+UeGob4MmV4LP4+7leAICV4TRGNApg8LJbiEhszlySSmVOsdSQhnGJH6jnmr95LszA/pgzrgK+3h9aRaoyU4HU0b+DPlOGxlJYrzFicSEZ2kdaRhKhR2/edZcGafbRo6M+koR3w8XLfAgQpwQppGubH1BGxKIrKzCVJnMkq1DqSEDXi55R0Ply7n8jGdZk0pAPenu6/xoqUYAU1CvFl2sg4DAaYuSSJk+dce6FIISrrh6QzfPzNAdo0C+Tpwe3xtJi0juQUUoKV0CCoDtNHxOFhNjJ7aRLHz+ZrHUkIh/h25yk++XcaMS2CeGpQDJ4e+ihAkBKstPr1fJg+Mg5vTzOzlyZz5Eye1pGEqJYNv55kybeHiG0ZzJMDo/Ew66cAQUqwSkLqejNtRBx+3h7MWZbMwVO5WkcSokrWbTvO8u8P0ykqlMfubafLldb1N2MHCQrwunj2vK8nby5PJvX4Ba0jCVFhqqqy+uejrPzpKF3a1mfcPfq91YQ+Z+0ggX4Xr6MMCfDmrS9S2Hs0W+tIQlyXqqp8+eNR1mw9TtfoMB6Jb6Prm47pd+YOElDHwpQRsYTV8+GdL1NIPnxe60hCXJWqqiz77jDf/HKC2zvcwEP95PazUoIO4O9jYcrwi6vrvrdyD7vSsrSOJMSfKKrKkk2H2PjbKXp0bMQDvSMxGvRdgCAl6DC+3h5MHhZLswZ+zF+9lx2p57SOJMQliqryyYY0NieepveNjRnRsyUGKUBAStChfLzMTBrSgYhGASxYs49tezO0jiQEiqLy8dep/LQ7nfibmzLkjggpwD+QEnQwb08zEwe3J6pJIAvXpfLz7nStIwkdsysKH67bz9a9Z7m3W3MGdg+XAvwfUoI1wNNi4ulBMbQNr8fH6w/wfeJprSMJHbLZFRZ8tY9f9p/jvtvCuadbcynAK5ASrCEWDxMTBsbQISKYTzceZONvp7SOJHSk3KYwf/VedqZlMfTOCOJvbqZ1JJclJViDPMxGHh/Qjo6RIXy++RDf/HJC60hCB8ptdt5btYekQ+cZ2asVvW9sonUklyYlWMPMJiPjE9pyY+tQvvjhCGu2HtM6knBjpeV23v4ihT1HshnVJ5IeHRtpHcnluf9iYS7AZDTyaP+2mE1GVv98DJtdYcCt8ga1cKySMhvvfJFC2slcHurXmm4xDbSOVCtICTqJ0Wjg4fjWmE0G1m07gc2mMviOFlKEwiGspTbmrtjNkTN5jO3fhi5tw7SOVGtICTqR0WBgVJ8ozCYjG3acxGZXGC4nrYpqKi4p583luzlxtoDxCe3oHBWqdaRaRUrQyYwGAyN7tcJsMrLxt1PY7Ar3y+VLoooKreW88Xkyp7MKefzedsS2CtE6Uq1TrRKcN28e69evB+C2225j6tSpDgnl7gwGA0PvjMDDbOTr7Sew2VUe7Bul+wvZReXkF5Ux5/Nkzl4oZsJ90cS0CNY6Uq1U5U+Ht23bxpYtW1i1ahWrV69m3759bNq0yZHZ3JrBYGBg93ASujVny54MPvx6P3ZF0TqWqCXyCkuZtTSJzJxinh4UIwVYDVU+EgwJCWH69OlYLBdvyNyiRQvS0+USscowGAwkdGuO2WTgyx+PYrOrPNpfv4tbiorJKbhYgLkFpTwzuD1RTQO1jlSrGVRVVau7k+PHjzN8+HCWLl1Ks2bNHBBLf1b/eJiFa/ZxU9swpo3qpLv7PIiKycwp5oX528gtLOVvj3ShbXiQ1pFqvWqX4KFDhxg3bhwTJkxgwIABFd4uO7sQRal2/1ZISIgfWVmuf4vMzbtOs3jTQWJaBPHEgHYVLsLaMr+qkvldlJlrZfaSJIpLbUwa2p4WNwQ4IV31OPNnZzQaCAryrfx21Rl0165dPPjggzz77LOVKkBxZT06NmJUn0j2HMnm7S9SKC23ax1JuIhzF4qZuTiRkjIbU4Z3qBUFWFtUuQQzMjJ44oknmDNnDvHx8Y7MpGu3d2jIw/GtST2Rw9srdlNSZtM6ktBY+vkiZixJpNymMGV4LM3C/LWO5Faq/MHIwoULKS0tZcaMGZceGzZsGMOHD3dIMD3rGt0Ak8nAh2tTeXP5biYObo+3p5zSqUenswqZszQJDAamjYilYUjlX+6Ja3PIByNVIe8JXt/OA5ksWLOPJvX9mDS0PXW8PK74vNo6v4rS6/xOnC3gjWXJmE0GpgyPpUFQHQ3SVY/bvycoalanqFAeH9COU5kFzF6aRKG1XOtIwkmOZeQze2kSFg8j00bG1coCrC2kBF1cbMsQJtwXQ/r5YmYtSSS/qEzrSKKGHT6dx5zPk/DxMjN9RBz1A320juTWpARrgejwIJ4ZHENmjpWZSxLJLSzVOpKoIWknc3hjeTJ+Phamj4wjuK631pHcnpRgLdGmWT0mDmnPhfxSZi5O5EJ+idaRhIPtP36BuSt2U8/Pk2kj4qjn76V1JF2QEqxFIpsE8uzQDuQXlzFjcSLnc61aRxIOsvfoxXNDQ+p6M3VEHIF+nlpH0g0pwVomolEAk4fFUlxiY+aSRDJzirWOJKppx76zvPNlCg3q+TB1eCwBdSxaR9IVKcFaqHkDf6YMj6W0XGHG4kROZ7rv6SPubldaJv9YtINGIb5MHh6Ln48UoLNJCdZSTcP8mDo8FkVRee6fWzmTVah1JFFJO1LPMX/1PiIa12XysFh8va98HqioWVKCtVijUF+mjYzDaICZS5I4eU6OCGuLbXszWLBmHxEN/Xn50Zvx8ZIrgrQiJVjLNQiqw+uPd8PDbGT20iSOn83XOpK4jp93p7NwXSpRTQKZOKQDPle5Ekg4h5SgG7ghxJfpI+Pw9jQze2kyR87kaR1JXMX3iaf5eP0B2javx9ODYvC0yLqRWpMSdBMhdb2ZNiIOP28P5ixL5uCpXK0jif+x6bdTfLrxIO1bBDHhvmgsHlKArkBK0I0EBXgxbWQcgb6evLk8mdQTOVpHEv+x/tcTLN18iLhWITwxMFpWDnchUoJuJtDPk2kj4wgJ8OatFbvZeyxb60i6t3brMVZ8f4QbW4cyPqGt3EPGxchPww0F1LEwZUQsYfV8eOeLFJIPn9c6ki6pqsqqn46y6udj3Nw2jLFyEy2XJD8RN+XvY2HK8Fgahfjy3so97ErL0jqSrqiqyhc/HGHttuN0i2nAmPjWmIzy380VyU/Fjfl6ezB5WCzNGvgxf/VedqSe0zqSLqiqyuebD7P+15PcHtuQB/tGYTQatI4lrkJK0M35eJmZNKQDEY0CWLBmH9v2Zmgdya0pqspnmw6yaecpenZsxAN3tcJokAJ0ZVKCOuDtaWbi4PZENQlk4bpUft6drnUkt6SoKp9sOMD3iWfoc1MThvdsiUEK0OVJCeqEp8XE04NiaBtej4/XH+D7xNNaR3IriqLy0dep/LQ7g7tvacrg21tIAdYScsGijlg8TEwYGMP81Xv5dONBbHaVXp0bax3LZaiqQvn+71ELK3dakaKq7D6cTVB2EZNa1aWl53nKduyq8PbZPhZKi93ztgnOnJvJ2xd6DK30dlKCOuNhNvL4gHYsWLOPpZsPYbMr9O3SVOtYmlMVhZIfF2I7tBWMZqjEQZzdrtJKVWntY8SYA2WVPEe9HAMqmtz0scY5c27mgBApQVExZpOR8Qlt+dfa/az44QjldoV7ujbXOpZmVMVOyff/wnbkFyydBuAZl1Ch7cptCvNX7yX58HmG3RnBXTc2qdL47nxLUWffcrMqpAR1ymQ08mj/i1cvrP75GDa7yoBbm+vufSzVbqPku/exHduJ5cYheHboV6HtysrtzFu1h71HL3D/Xa24M65RDScVNUVKUMeMRgMPx7fGbDKwbttxbHZFV2/oq/ZyrJvew34yGc+bh2OJ7l2h7UrL7LzzZQoHTuTwYN8oure/oYaTipokJahzRoOBUX2iMJuMbPj1JDabootTO1RbGdZN72I/tQfPbqOwtLmzQttZS228/UUKh07n8nB8a7pGN6jhpKKmSQkKjAYDI3u1wmwysvG3U9jsCvf3jnTbk3xVWynWf7+N/Uwqnt0fwhJ1W4W2Ky6x8daK3RxNz2ds/zZ0aRNWw0mFM0gJCgAMBgND74zAw2zk6+0nsNlVt7zcSy0vwbphLvazB/G6/RE8WnWt0HZFJeW8uSyZk+cKGZ/Qlk5RoTWcVDiLlKC4xGAwMLB7OGaTka+2HMOmKG514b9aVkzx+jdRMo/idcc4PCK6VGi7guIy3liWzJmsIh4f0I7YliE1nFQ4k5SguIzBYCChW3PMJgNf/ngUm13lUTdYAkotLaL4mzko50/i1eMxPMI7V2i7/KIy5nyexNkLVibcF0NMi6AaTiqcTUpQXFH8zc0wm4ws++4wdrvC+IR2eJhrZxEqJQVYv56DknMG77uexNw0tkLb5RaWMntpEtl5JTw9OIa2zerVcFKhhdr5Wy2coveNTRjZqxVJh87z3qo9lNvsWkeqNMWaj3XdTJTcM3j3fqrCBXghv4SZixO5kF/KxCHtpQDdmJSguKYeHRsxqk8ke45k884XKZSW154iVIpzsa6dgZKXiXefSZgbx1Rou/N5VmYuSSSvqIxJQ9sT2SSwhpMKLUkJiuu6vUNDHurXmv3Hc3h7xW5KymxaR7oupfACxWtfRynMxrvvJMwN21Rou8ycYmYuTqTQamPysFhaNqpbw0mF1qQERYV0i2nA2P5tOHgqjzeX78Za6rpFqBScp3jt66jF+fj0m4z5hqgKbZd+voiZS5IoKbMzdXgs4Tf413BS4QqkBEWFdWkbxviEthxLz2fO58kUlZRrHelPlPzMiwVYWoRP/BRMYS0rtN3R9HxmLE7ErqhMHRFH0zC/Gk4qXIWUoKiUTlGhPD6gHacyC5i9NIkCF1oHT8k9e7EAy0vwuXsaptDwCm23//gFZn+ehJfFxPP3x9E41LeGkwpXIiUoKi22ZQgT7osh/Xwxr32yizPni7SOhD3nDMVrXwe7DZ/+0zEFV2yNxB+SzzB3+W6CA7x47v6OhAb61HBS4WqkBEWVRIcHMW1ELCXldl77ZCcpR7S7t7E9+xTWtTMA8O4/HVO966+WbVcUFm86yCcb0mjdLJDnRnYk0M+zpqMKFyQlKKqsRcMAXhzdidBAb95ekcLqn49iVxSnZrCfP07xuhlgMuPT/zlMgQ2vu82F/BLmLE1m867T3NW5Mc8Mao+Pl1w3oFfykxfVUs/fi+dGduTTjWms2XqcAydyePSettTz96rxse2ZRyn+Zg4Gizc+d0/D6H/9RQ2SD53no29SKbcpPHJ3a25pJ0th6Z2UoKg2T4uJR+5uQ9tm9fhkYxovLtzBkDsjuDWmQY2tS2g7ewjr+jcwePldLEC/4Gs+v6iknBXfH+an3Rk0qe/L+IR2hNWT9/+ElKBwoJvbhRHe0J+PvznAovUH2L73LKP6RNIgqI5Dx7GlH8C6YS6GOoH4xE/F6Hv1S9pUVWVHaiZLvz1IodVG35uacO+t4bX2OmjheFKCwqHqB/owdUQsW1IyWP7dYV5cuINb29/APV2bUde3+h882E7vw/rvtzH6B+MdPxWjz9Wv6Eg7mcMXPx7hyJl8moX5MWloFE3qy/l/4nJSgsLhjAYD3dvfQPuIYNZsPcZPyels25vBHbEN6dGxEcEB3lXar+1kCtZN72AMCLtYgN5/vqJDVVUOnMhh/Y6T7D16gUA/Tx7sG0W36AZut0CscIxqleDatWuZP38+NpuN0aNHM3LkSEflEm4goI6FB+6K5K7Ojfnq52Ns+u00G387RcdWIdza/gZaNw2s8DqFtuNJWL99D2NgQ3zip2DwuvyE5oLiMn47kMmPyemcyizEz8eDQbe3oGfHRlg8TDUxPeEmqlyC586dY+7cuaxcuRKLxcKwYcO46aabiIiIcGQ+4QbqB/rw6D1tGXR7CzbvOs1Pu9PZmZaFr7cHca2CadOsHlFNAvGvY7ni9oUHtmPdNA9jcFN8+j2LwbMOiqqScb6I/cdz2HvsAvuPX8CuqDQKqcODfaO4uW19PMxSfuL6qlyC27Zto0uXLtSte/E9md69e7NhwwaefPJJh4UT7qWevxeD74jg3lvD2Xs0m19Tz/HbgUx+2p3xn3/3pGGwL/XreePvY8HXx4OgCyk0PbKCQp9G7Kw7mAvfnST9fDGnswopKbu4rFdooDe9OjemS5v6NA71dfs75QnHqnIJZmZmEhLy33sthIaGkpKSUuHtg4Kce31mSIh7vyFe2+Z3Q4MA7uoajt2ucORMHnsOn+d4Rj4nzxawdU8G1lI7nS1HGFFnG4dtoXyQ3ZXy0xkE1PGkYagvPTo3oUXDANq3DCHUDU51qW0/v8pw9blVuQQVRbnsL66qqpX6C5ydXYiiqFUdvlJCQvzIyipwylhaqO3zC/Q20z06jO7R/72FpXXf95Rv3YZaP5IO/Sczs9iOr4/Hn2/6ZLfX6rlD7f/5XYsz52Y0Gqp0cFXlk6XCwsLIysq69HVWVhahoXIbQlF9Zfu+xbb1/zA3bod//CTq1w8kwNfTbe56J1xLlX+rbrnlFrZv386FCxewWq1s3LiR7t27OzKb0KGylH9TuvUzzE1j8b7rKQzmK39YIoSjVPnlcP369Zk4cSKjRo2ivLycQYMGERNTsXs4CHElpclfU7ZjBebmnfC6czwGk5zGKmpetX7L+vfvT//+/R2VRehYaeJXlO1chblFF7zuGIvBKKe3COeQP7VCU6qqUrZzJWVJazG37IrXbWMwyHt/womkBIVmVFWl9NdllKdswCOqO563PojBIAUonEtKUGhCVVVKty+hfO8mPNrciWfX+6UAhSakBIXTqapC6ZZPKU/9Ho/o3nh2GSZXeQjNSAkKp1IVhdKfP6Y87Wcs7fthuXGwFKDQlJSgcBpVsVPyw4fYDm/HEpeApeO9UoBCc1KCwilUxUbJdx9gO7oDS6eBeMbdo3UkIQApQeEEqt1Gyeb52I7vwvOmoVja99U6khCXSAmKGqXayrB++x72k7vxvGUklna9tI4kxGWkBEWNUW1lWDe+g/30Xjy7jcLS5k6tIwnxJ1KCokao5aVY//0W9vQDeHV/GI8oWVxDuCYpQeFwapkV64a52M8dwuuOsXi0vEXrSEJclZSgcCi1rJji9W+iZB7F687xeLS4SetIQlyTlKBwGLWkkOL1b6Bkn8Sr5xN4NO+odSQhrktKUDiEUlKA9evZKDnpePeagLlpB60jCVEhUoKi2pTiPKxfz0LJz8S799OYG0drHUmICpMSFNWiFOVcLMDCbLz7TMTcsI3WkYSoFClBUWVKYTbF62ahWvPw7vss5gaRWkcSotKkBEWVKAVZFwuwpBCffpMx1Y/QOpIQVSIlKCpNyTtH8bqZqOUl+MRPwRQarnUkIapMsxJUcs9it5U7ZawytQ72nCKnjKUFZ85PLS2iZPN8sNvwuXsapuCmThlXiJqiWQla18/Blpd1/Sc6QLFTRtGOs+dn8PbHu/90TPUaOXlkIRxPsxL07DoKc1mJU8by9/cmP9/qlLG04Oz5mepHYKwT6LTxhKhJmpWguUkMRkV1yli+IX5YswqcMpYW3H1+QtQkub2XEELXpASFELomJSiE0DUpQSGErkkJCiF0TUpQCKFrUoJCCF2TEhRC6JqUoBBC16QEhRC6JiUohNA1KUEhhK5JCQohdE1KUAiha1KCQghdkxIUQuj/CQHrAAASs0lEQVSalKAQQtekBIUQuiYlKITQNSlBIYSuSQkKIXStyiW4a9cuBg0aREJCAqNHj+bMmTOOzCWEEE5R5RKcMmUKr776Kl999RX9+/fn1VdfdWQuIYRwiiqVYFlZGU8//TRRUVEAREZGkpGR4dBgQgjhDFUqQYvFQkJCAgCKojBv3jx69uzp0GBCCOEMBlVV1Ws9Yf369bz++uuXPRYeHs6iRYsoKytj+vTp5OXl8f777+Ph4VGjYYUQwtGuW4JXU1RUxGOPPUbdunWZM2cOFoulUttnZxeiKFUautJCQvzIyipwylhakPnVbu48P2fOzWg0EBTkW/ntqjrglClTaNq0KW+99ValC1AIIVyFuSob7d+/n82bNxMREcGAAQMACA0N5V//+pdDwwkhRE2rUgm2adOGtLQ0R2cRQginkytGhBC6JiUohNA1KUEhhK5JCQohdE1KUAiha1KCQghdkxIUQuialKAQQtekBIUQuiYlKITQNSlBIYSuSQkKIXRNSlAIoWtSgkIIXZMSFELompSgEELXpASFELomJSiE0DUpQSGErkkJCiF0TUpQCKFrUoJCCF2r0i03a4qqquTkZFFWVgKoDttvZqYRRVEctj9Xo9/5GbBYvAgMDMFgMDg9l3APLlWChYV5GAwG6tdvhMHguINUs9mIzea+JaHX+amqQm7ueQoL8/Dzq6tBMuEOXOrlsNVaiJ9fXYcWoHBfBoMRP79ArNZCraOIWsyl2kZR7JhMLnVwKlycyWRGUexaxxC1mEuVICDv7YhKkd8XUV0uV4JCCOFMUoJCCF2TEhQuZdGiD7n//iHk5+drHUXohJSgcBnHjh3lzJnT/O1vrzB//rtaxxE6ISV4DYmJO3nyyUe1jlEhOTkXGDgwvtLb5efnkZDQ2yEZnn9+Cqmp+6q8ffPm4fzlL3+nZctIpk37CwBlZWU8+OBICgvlNBhRM6QE3URq6n6ioto4bbvfKYqCqqrs27eXkhIrrVu3rfK+rsRisdCnTz+WLVvs0P0K8TspwQr65JOPuP/+wYwaNZR3352L3W4nM/McTz75KGPGPMDYsaPYu3cPwFUf/6Pi4mLeeecNHn54JPffP4QXX3yuWvkOHNiPv78/06ZNZODAeCZPfgqbzQbAX/86nc2bN1167j/+8RLr1n31h+0CmDjxCYYOvZe//nX6pe3y8/OZNes1xo9/mOHDB7JgwXsA/POfbzNjxitMmvQkDzwwBLvdzpo1K+nVq89lY3z00QcAnDp1kmHDBpKWduCq+a/1/F69+lzKK4SjueyZyVv3ZLAlJcMh+zIYQP3DpcjdYhrQNbpBhbffvn0rW7b8xIcfforZbOaFF6ayevWX5OXlcsst3RgxYhS//LKNlJRk2rWLZt26r674+O9UVeX55yfTtm00Cxd+hsFg4Pz5rMvGfOGFqZw+ffqKeRYs+AhPT6/LHjtwIBWLxYOXX56Bh4cHI0cO4vjxY0REtCQtLZVx45649NyDB9MYPHjYpe3Ky8uYMeNNTCYTTzwxlp9//oHbb+/B3//+PMOHP8DUqX/BZrMxbtxD3HFHD9LS0jCbzfzjH3Pw8rqYIylpFyNGjLo0xtixjzFmzAN069adl1/+K8899yKRkVFX/R5f6/lBQUGYzR6cOHGcpk2bVeAnJkTFuWwJupJdu36jZ8/el/7Dx8ffw/r1XzNs2Ej+8pepHDyYxi23dOO++4YA0KnTjVd8/Hc7d/5Kbm4ujzwy/tLJvsHBIZc959VXZ1UqY1paKgsWLMLT0xMAm82Ov78/BQUF5Ofn0bBhI+Die2xnzpymefMWl7abN++DS9u1bNmKnJwcfvvtV/bv38d77719aYzCwgIUReHw4TTef//jS98PgKysTAID6136OiQklD594nniibG8+uos2rfvcM3813t+UFAQmZnnpASFw7lsCXaNrtzR2rVUd4EBVVX+52uw223ExHTgs8+Ws23bFjZv3sg336zlrbf+edXHf5eWdoDo6PbXvNqhMkeCWVlZeHh4EBYWBkBBQQElJVZCQ+uza9dvRES0ujRWaup+mjRpitlsJifnAmVlpdxwQ8NL+9q/fx/9+vUnMXEnAwYMuuwIEiAjIx0vL28aN25y2eOenp6UlZVd+jon5wK//LIVLy9vQkPrX3WeFX1+WVnppaIWwpFctgRdSVxcZ/7v/xaSkDAAk8nMN9+sIS6uE//859sEB4cyZMhwYmM78fDDIwGu+vjvgoND2L59KzabDbPZTHb2eYKCgi97TmWOBFNT9xEV1frS12lpqbRqdfGlZG5uDv7+AQDYbDY+/fSjS/+WmrqfwsJCzp49S1hYGCtXrqBu3bq0adOO06dPsXz5UkaPHoOXlxdWq5WsrHMcO3b0srF+Fx4ewcmTxwkODqagoIBnn32Khx9+lNzcHObPf5dZs+Zeeu4rr7xI9+53cNttdwBc9/l2u5309DOEh0dU+HsiREVJCVZA1663cuhQGmPGjMJut3HjjV24776hZGef56WXXuCbb9ZiNBp54YWXALjvvqFXfPx3PXv2JjFxJ/ffPwQvLy8aN27CK6/MqHK+AwdSiYz83xKMBC6+NP/888+YNGkCN9xwA0ajicjIyEvPGzBgMK+++iK5uTlERLTipZdeB6BHj7tISUlm9OhheHv7YLFYGD/+SQ4eTLvip8m33XYHO3b8Qps27Zg69RkGDhzE7bf3wGazsXz5UhITdxIX1+k/4x5g0KChAJSUlFz3+SkpybRp0w5fX98qf4+EuBqDqqqOW720ErKzC1GUy4c+e/YEYWFNHT6WXtfbc6aiokIef/wRPvhg0Z8+tPmj/Pw8XnzxucveHriel19+gb59+9O5801X/Pea+r1xlpAQP7KyCrSOUSOcOTej0UBQUOX/UMopMsIh6tTx5cknnyE9Pf2az/P3D6hUAZaVldGhQ9xVC1CI6pKXw8JhOnfu4vB9WiwWBg4cpPmRrnBfciQohNA1KUEhhK5JCQohdE1KUAiha9Uuwf3799OuXTtHZBFCCKerVglarVZeeeUVysvLHZVHCCGcqlolOGPGDEaPHu2oLEII4XRVPk9w8+bNlJSU0KdPn+s/+QqudGZ3ZqYRs7lm3qasqf26Cj3Pz2g0EhLi58Q0jlfb81+Lq8/tuiW4fv16Xn/99cseCw8Pp7CwkEWLFlV54CtdNqcoSo2cFOsKl5XVpOvNT1VVJk9+mjfeeMeJqRznevNTFKVWX3Yml805RlUvm7tuCfbt25e+ffte9tiKFStYsGABI0f+d3WUhIQEFi9eLBe5uwhVVZk//x2MRhMlJVYaNry4XNZHH31AUVERxcVFPPPM5Gte5yuEHjhkAYXIyEjS0tIqtY3eFlB47bW/ExvbkX79+l/2eGLiTj766APmzfugyvu+0vzWr1+HxeJJjx69WLhwAU2aNEVVL67Ld/fdCVUeSwvX+/nJAgquqzYcCbr3G0k6tndvyqXl6VNT9xEd3Z6UlCQ6duyscTIhXItDFlCo7FFgbfL++/P44YfNBATUJSgomG7duhMW1oD589/Bbldo2rQZXl5eHD16hAsXLhAREcHf//4aFosn8+bNZevWLQQHB6MoCrGxHa851ieffMTGjesxGo107tyFxx9/iuzs87z88l+xWq0YjQaefnoK7dpFk5l57tLjJpORp56afNl9TG6+uRvz579DkybNyMw8R1hYA2677Q7ef38e9evXp2vX7rRvH1vT3z4hXJ7LriJTfnAr5Wk/OWRfBoOBP77q94jsjkerrtfdbsuWn0hJSebTT5djtVoZM+Z+unXrDly8I9oXX6zj8OGDbN68iQULPkZRFJ56ajzbt29FVVUOHkzjs8+WU1BQwIMPDrvmWNW5mdNvv23/082cunXrfinr70vkd+7cpUZWehGiNnPZEnQFO3f+yp139sTDwwMPDw9uvfW2S//WuHFTfH196dAhDn//AL78cjknTx7n9OlTWK1WUlP3cdttd2A2mwkMDKRLl2uXbnVu5tSt261/upmTEKJiXLYEPVp1rdDRWkVU9YMRo9H4pw9vfvf7TX+2bPmRDz9cwODBw+jX7x5yc3NRVfU/R5//fb7JZLrmWNW5mdO3325k3bo1ly1W2q1bp0rNdcuWnZV6vhDuwmVL0BV06nQTixf/HwMGDKKsrJRt27bQsmXkZc/ZuXMHd97Zk/j4ezhz5jRJSbvo3PlGOnW6kSVLPiUhYSAlJSX8+ut22rWLuepY1bmZU+fOnRk1asRl+5NSE6JipASv4ZZburF3bwoPPTQSf39/goNDsFguv+1j//4DeOmlv/Dtt//GbPYgOjqG9PR07r77XlJT9zNq1FDq1QuiWbPwa45VnZs5mUx/vpnTH9X2k6WFqElyo6Vr2Ls3hVOnTtK3793YbDbGjXuI5557kYiIlg7PWB1Xmt//niytKArjx0/ghRemMWPGG7zxxgwGDRr6pyNbVyTnCdZecp5gLdekSVM2bfo3o0cP5+GHR9Kjx10uV4BXs2HD10RGtmH8+Cfx8/MnOro9Pj4+3HVXHyZMGMfAgUNqRQEKUdPk5fA1+PsH8Oab72odo0r27k1h+PAHgIsnS8fH30N5eTm7dv2Gp6cnYWFhGicUwjVICbqp/z1ZOjg4hFmzXmPQoGEoip2FCxcwadI0rWMKoTl5T9AN6H1+8p6g65L3BIUQwsVJCQohdE1KUAiha1KCQghdc7kS1OhzGlFLye+LqC6XKkGz2UJRUb78YosKUVWVoqJ8zGaL1lFELeZS5wkGBoaQk5NFYWGuQ/d7cTUY9z2FRM/zM5stBAaGODmRcCcuVYImk5ng4AYO3687n4cFMj8hqsOlXg4LIYSzSQkKIXRNs5fDRqPBrcdzNplf7ebO83PW3Ko6jmbXDgshhCuQl8NCCF2TEhRC6JqUoBBC16QEhRC6JiUohNA1KUEhhK5JCQohdE1KUAiha1KCQghd010J7t+/n3bt2mkdw6F27drFoEGDSEhIYPTo0Zw5c0brSA6xdu1a+vXrx1133cXixYu1juNw8+bNIz4+nvj4eGbNmqV1nBoxc+ZMpk+frnWMa9JVCVqtVl555RXKy8u1juJQU6ZM4dVXX+Wrr76if//+vPrqq1pHqrZz584xd+5clixZwurVq1m2bBmHDx/WOpbDbNu2jS1btrBq1SpWr17Nvn372LRpk9axHGr79u2sWrVK6xjXpasSnDFjBqNHj9Y6hkOVlZXx9NNPExUVBUBkZCQZGRkap6q+bdu20aVLF+rWrYuPjw+9e/dmw4YNWsdymJCQEKZPn47FYsHDw4MWLVqQnp6udSyHyc3NZe7cuYwfP17rKNelmxLcvHkzJSUl9OnTR+soDmWxWEhISABAURTmzZtHz549NU5VfZmZmYSE/HfF6NDQUM6dO6dhIsdq2bIlHTp0AOD48eOsX7+e2267TeNUjvPiiy8yceJE/P39tY5yXS61srQjrF+/ntdff/2yx8LDwyksLGTRokXahHKQq81t0aJFlJWVMX36dGw2G+PGjdMooeMoioLB8N+lkVRVvexrd3Ho0CHGjRvH1KlTadasmdZxHGLFihU0aNCAm2++mZUrV2od57p0sZTWihUrWLBgAXXq1AHgwIEDREVFsXjxYnx9fTVOV31FRUU89thj1K1blzlz5mCx1P4bD61atYqdO3fy2muvAfDee++hqipPPvmkxskcZ9euXTz11FM8//zzxMfHax3HYR566CGysrIwmUzk5eVRXFzMvffey/PPP691tCtTdahVq1ZaR3Coxx57TH3hhRdUu92udRSHOXv2rHrHHXeo2dnZanFxsXrPPfeou3fv1jqWw6Snp6s33XSTum3bNq2j1Kgvv/xSnTZtmtYxrsntXg7rzf79+9m8eTMREREMGDAAuPj+2b/+9S+Nk1VP/fr1mThxIqNGjaK8vJxBgwYRExOjdSyHWbhwIaWlpcyYMePSY8OGDWP48OEaptInXbwcFkKIq9HNp8NCCHElUoJCCF2TEhRC6JqUoBBC16QEhRC6JiUohNA1KUEhhK5JCQohdE1KULikWbNm8cQTT1z6eubMmYwePdrt1oIU2pPL5oRLGjt2LL169SI1NZXk5GS2bNnCkiVL8PDw0DqacDNy2ZxwWe+++y4bN26ksLCQJUuW0KBBA60jCTckL4eFy2rdujUHDx5k0qRJUoCixsiRoHBJaWlpjBkzhm7dunH27NlavyCucF1yJChczrlz53jsscd46aWX+Nvf/sbBgwf59ddftY4l3JSUoHAphYWFjB07lgcffJAePXrg7e3NmDFjmDt3rtbRhJuSl8NCCF2TI0EhhK5JCQohdE1KUAiha1KCQghdkxIUQuialKAQQtekBIUQuvb/hzoULQ8rcAEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Compute the range of values that we wish to explore\n",
    "xs = np.arange(-5.0, 5.0, 0.01).astype(np.float32)\n",
    "\n",
    "# Use Huber loss:\n",
    "regr_sqr_loss = [f_regr_huber_loss(x, 0.0) for x in xs]\n",
    "\n",
    "# Compute gradient of Huber loss with respect to x-hat\n",
    "regr_sqr_loss_grad = [f_regr_huber_loss_grad(x, 0.0) for x in xs]\n",
    "\n",
    "plt.figure(figsize=(5,5))\n",
    "plt.xlim(-5.0, 5.0)\n",
    "plt.ylim(-5.0, 5.0)\n",
    "line_loss, = plt.plot(xs, regr_sqr_loss, label=r'loss $c = huber(x, \\hat{x})$')\n",
    "line_loss_grad, = plt.plot(xs, regr_sqr_loss_grad, label=r'grad loss $\\frac{dc}{dx}$')\n",
    "plt.legend(handles=[line_loss, line_loss_grad])\n",
    "plt.xlabel(r'$x$')\n",
    "plt.show()\n",
    "\n",
    "if SAVE_PLOTS:\n",
    "    plt.figure(figsize=(5, 5))\n",
    "    plt.xlim(-5.0, 5.0)\n",
    "    plt.ylim(-5.0, 5.0)\n",
    "    line_loss, = plt.plot(xs, regr_sqr_loss, label=r'loss $c = huber(x, \\hat{x})$')\n",
    "    line_loss_grad, = plt.plot(xs, regr_sqr_loss_grad, label=r'grad loss $\\frac{dc}{dx}$')\n",
    "    plt.legend(handles=[line_loss, line_loss_grad])\n",
    "    plt.xlabel(r'$x$')\n",
    "    plt.savefig('regr_huber_loss_2.png', dpi=600)\n",
    "    plt.close()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Show the classification logits and probabilities in tables:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>$X_0$</th>\n",
       "      <th>$X_1$</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    $X_0$  $X_1$\n",
       "0     0.0   -5.0\n",
       "1     0.0   -4.0\n",
       "2     0.0   -3.0\n",
       "3     0.0   -2.0\n",
       "4     0.0   -1.0\n",
       "5     0.0    0.0\n",
       "6     0.0    1.0\n",
       "7     0.0    2.0\n",
       "8     0.0    3.0\n",
       "9     0.0    4.0\n",
       "10    0.0    5.0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=np.array(logits)\n",
    "pd.DataFrame(columns=['$X_0$', '$X_1$'], data=data[::128])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>$X_0$</th>\n",
       "      <th>$X_1$</th>\n",
       "      <th>$q_0$</th>\n",
       "      <th>$q_1$</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-5.0</td>\n",
       "      <td>0.993307</td>\n",
       "      <td>0.006693</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-4.0</td>\n",
       "      <td>0.982014</td>\n",
       "      <td>0.017986</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>0.952574</td>\n",
       "      <td>0.047426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>0.880797</td>\n",
       "      <td>0.119203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.731059</td>\n",
       "      <td>0.268941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.268941</td>\n",
       "      <td>0.731059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.119203</td>\n",
       "      <td>0.880797</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.047426</td>\n",
       "      <td>0.952574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.017986</td>\n",
       "      <td>0.982014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.006693</td>\n",
       "      <td>0.993307</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    $X_0$  $X_1$     $q_0$     $q_1$\n",
       "0     0.0   -5.0  0.993307  0.006693\n",
       "1     0.0   -4.0  0.982014  0.017986\n",
       "2     0.0   -3.0  0.952574  0.047426\n",
       "3     0.0   -2.0  0.880797  0.119203\n",
       "4     0.0   -1.0  0.731059  0.268941\n",
       "5     0.0    0.0  0.500000  0.500000\n",
       "6     0.0    1.0  0.268941  0.731059\n",
       "7     0.0    2.0  0.119203  0.880797\n",
       "8     0.0    3.0  0.047426  0.952574\n",
       "9     0.0    4.0  0.017986  0.982014\n",
       "10    0.0    5.0  0.006693  0.993307"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=np.append(np.array(logits), np.array(clf_q), axis=1)\n",
    "pd.DataFrame(columns=['$X_0$', '$X_1$', '$q_0$', '$q_1$'], data=data[::128])"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}