additional_self-supervised_representation_learning_3_part.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Basic setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Create anaconda environment\n",
    "<br>\n",
    "```bash\n",
    "conda create -n ml python=3.7.5 jupyter\n",
    "```\n",
    "Install fastai library\n",
    "<br>\n",
    "```bash\n",
    "conda install -c pytorch -c fastai fastai\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## The universal approximation theorem"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<a href=\"https://en.wikipedia.org/wiki/Universal_approximation_theorem\">The Universal Approximation Theorem</a>\n",
    "<br>\n",
    "<a href=\"https://www.youtube.com/watch?v=Ijqkc7OLenI\">The Universal Approximation Theorem for neural networks</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Universal approximation theorem says that every compact / mesurable function can be approximated at the any $\\epsilon$ distance with three layer (shallow) wide enough neural network\n",
    "<br>\n",
    "Or with the multilayer fixed width neural network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/ua_1.jpg\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Let's images the one dimensional case\n",
    "<img src=\"images/add1/ua_wd_1.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'kiwisolver'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Input \u001b[0;32mIn [8]\u001b[0m, in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mmatplotlib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpyplot\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mplt\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/edu/lib/python3.9/site-packages/matplotlib/__init__.py:210\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    205\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m parse_version(module\u001b[38;5;241m.\u001b[39m__version__) \u001b[38;5;241m<\u001b[39m parse_version(minver):\n\u001b[1;32m    206\u001b[0m             \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMatplotlib requires \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmodname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m>=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mminver\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m; \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    207\u001b[0m                               \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124myou have \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmodule\u001b[38;5;241m.\u001b[39m__version__\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 210\u001b[0m \u001b[43m_check_versions\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    213\u001b[0m \u001b[38;5;66;03m# The decorator ensures this always returns the same handler (and it is only\u001b[39;00m\n\u001b[1;32m    214\u001b[0m \u001b[38;5;66;03m# attached once).\u001b[39;00m\n\u001b[1;32m    215\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mlru_cache()\n\u001b[1;32m    216\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_ensure_handler\u001b[39m():\n",
      "File \u001b[0;32m~/anaconda3/envs/edu/lib/python3.9/site-packages/matplotlib/__init__.py:204\u001b[0m, in \u001b[0;36m_check_versions\u001b[0;34m()\u001b[0m\n\u001b[1;32m    195\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m ft2font\n\u001b[1;32m    197\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m modname, minver \u001b[38;5;129;01min\u001b[39;00m [\n\u001b[1;32m    198\u001b[0m         (\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcycler\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m0.10\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m    199\u001b[0m         (\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdateutil\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2.7\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    202\u001b[0m         (\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpyparsing\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2.2.1\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m    203\u001b[0m ]:\n\u001b[0;32m--> 204\u001b[0m     module \u001b[38;5;241m=\u001b[39m \u001b[43mimportlib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mimport_module\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    205\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m parse_version(module\u001b[38;5;241m.\u001b[39m__version__) \u001b[38;5;241m<\u001b[39m parse_version(minver):\n\u001b[1;32m    206\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMatplotlib requires \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmodname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m>=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mminver\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m; \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    207\u001b[0m                           \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124myou have \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmodule\u001b[38;5;241m.\u001b[39m__version__\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m~/anaconda3/envs/edu/lib/python3.9/importlib/__init__.py:127\u001b[0m, in \u001b[0;36mimport_module\u001b[0;34m(name, package)\u001b[0m\n\u001b[1;32m    125\u001b[0m             \u001b[38;5;28;01mbreak\u001b[39;00m\n\u001b[1;32m    126\u001b[0m         level \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[0;32m--> 127\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_bootstrap\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_gcd_import\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m[\u001b[49m\u001b[43mlevel\u001b[49m\u001b[43m:\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpackage\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlevel\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'kiwisolver'"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.  , 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 ,\n",
       "        0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21,\n",
       "        0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32,\n",
       "        0.33, 0.34, 0.35, 0.36, 0.37, 0.38, 0.39, 0.4 , 0.41, 0.42, 0.43,\n",
       "        0.44, 0.45, 0.46, 0.47, 0.48, 0.49, 0.5 , 0.51, 0.52, 0.53, 0.54,\n",
       "        0.55, 0.56, 0.57, 0.58, 0.59, 0.6 , 0.61, 0.62, 0.63, 0.64, 0.65,\n",
       "        0.66, 0.67, 0.68, 0.69, 0.7 , 0.71, 0.72, 0.73, 0.74, 0.75, 0.76,\n",
       "        0.77, 0.78, 0.79, 0.8 , 0.81, 0.82, 0.83, 0.84, 0.85, 0.86, 0.87,\n",
       "        0.88, 0.89, 0.9 , 0.91, 0.92, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98,\n",
       "        0.99]),\n",
       " array([1.    , 1.0001, 1.0004, 1.0009, 1.0016, 1.0025, 1.0036, 1.0049,\n",
       "        1.0064, 1.0081, 1.01  , 1.0121, 1.0144, 1.0169, 1.0196, 1.0225,\n",
       "        1.0256, 1.0289, 1.0324, 1.0361, 1.04  , 1.0441, 1.0484, 1.0529,\n",
       "        1.0576, 1.0625, 1.0676, 1.0729, 1.0784, 1.0841, 1.09  , 1.0961,\n",
       "        1.1024, 1.1089, 1.1156, 1.1225, 1.1296, 1.1369, 1.1444, 1.1521,\n",
       "        1.16  , 1.1681, 1.1764, 1.1849, 1.1936, 1.2025, 1.2116, 1.2209,\n",
       "        1.2304, 1.2401, 1.25  , 1.2601, 1.2704, 1.2809, 1.2916, 1.3025,\n",
       "        1.3136, 1.3249, 1.3364, 1.3481, 1.36  , 1.3721, 1.3844, 1.3969,\n",
       "        1.4096, 1.4225, 1.4356, 1.4489, 1.4624, 1.4761, 1.49  , 1.5041,\n",
       "        1.5184, 1.5329, 1.5476, 1.5625, 1.5776, 1.5929, 1.6084, 1.6241,\n",
       "        1.64  , 1.6561, 1.6724, 1.6889, 1.7056, 1.7225, 1.7396, 1.7569,\n",
       "        1.7744, 1.7921, 1.81  , 1.8281, 1.8464, 1.8649, 1.8836, 1.9025,\n",
       "        1.9216, 1.9409, 1.9604, 1.9801]),\n",
       " 100)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.arange(0, 1, 0.01)\n",
    "f = lambda x: x**2 + 1\n",
    "y = f(x)\n",
    "x, y, len(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'plt' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [5]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mplt\u001b[49m\u001b[38;5;241m.\u001b[39mplot(x, y)\n\u001b[1;32m      2\u001b[0m plt\u001b[38;5;241m.\u001b[39mshow()\n",
      "\u001b[0;31mNameError\u001b[0m: name 'plt' is not defined"
     ]
    }
   ],
   "source": [
    "plt.plot(x, y)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "def sgm(x:np.ndarray):\n",
    "    return 1/(1 + np.exp(-x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.01798621, 0.01984031, 0.02188127, 0.02412702, 0.02659699,\n",
       "       0.02931223, 0.03229546, 0.03557119, 0.03916572, 0.04310725,\n",
       "       0.04742587, 0.05215356, 0.05732418, 0.06297336, 0.06913842,\n",
       "       0.07585818, 0.0831727 , 0.09112296, 0.09975049, 0.10909682,\n",
       "       0.11920292, 0.13010847, 0.14185106, 0.15446527, 0.16798161,\n",
       "       0.18242552, 0.19781611, 0.21416502, 0.23147522, 0.24973989,\n",
       "       0.26894142, 0.2890505 , 0.31002552, 0.33181223, 0.35434369,\n",
       "       0.37754067, 0.40131234, 0.42555748, 0.450166  , 0.47502081,\n",
       "       0.5       , 0.52497919, 0.549834  , 0.57444252, 0.59868766,\n",
       "       0.62245933, 0.64565631, 0.66818777, 0.68997448, 0.7109495 ,\n",
       "       0.73105858, 0.75026011, 0.76852478, 0.78583498, 0.80218389,\n",
       "       0.81757448, 0.83201839, 0.84553473, 0.85814894, 0.86989153,\n",
       "       0.88079708, 0.89090318, 0.90024951, 0.90887704, 0.9168273 ,\n",
       "       0.92414182, 0.93086158, 0.93702664, 0.94267582, 0.94784644,\n",
       "       0.95257413, 0.95689275, 0.96083428, 0.96442881, 0.96770454,\n",
       "       0.97068777, 0.97340301, 0.97587298, 0.97811873, 0.98015969])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s_1 = np.arange(-4.0, 4.0, 0.1)\n",
    "s = sgm(s_1)\n",
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXhV5bn+8e9DGAJJCBASphACCCIiCERQW7VHrUVrxYlWHOqEVFuH9thB2x7bqh0cTgerP5VaiwyCw1GLirNWKooQRiFMAYEECElICCEhIcPz+yOxRgiygZ2svZP7c125mr33ysp9pXvfvrzrXWuZuyMiItGvTdABREQkPFToIiIthApdRKSFUKGLiLQQKnQRkRaibVC/uHv37p6enh7UrxcRiUqLFy8udPfkxl4LrNDT09PJzMwM6teLiEQlM9t8sNc05SIi0kIcstDN7EkzyzezlQd53czsITPLNrMVZjYq/DFFRORQQhmhTwXGfcnr5wKD6r8mA48efSwRETlchyx0d58HFH3JJuOBaV5nAdDFzHqFK6CIiIQmHHPofYCcBo9z6587gJlNNrNMM8ssKCgIw68WEZHPhKPQrZHnGr3il7tPcfcMd89ITm501Y2IiByhcBR6LtC3weNUYFsY9isiIochHOvQ5wA3m9lsYCxQ4u7bw7BfEZGot7uiirySCraXVJBXspftJRWcOSSF4aldwv67DlnoZjYL+BrQ3cxygV8B7QDc/TFgLnAekA2UA9eGPaWISIQq31fN5p3lbCkqJ6f+K7d4L1t37WVr8V5KK6u/sL0ZdI/vEEyhu/vEQ7zuwA/ClkhEJMLU1Dq5xeVk5+9hQ8EeNhaUsbGwjE2FZeSXVn5h24QObUnt1onUrp0Y278bfbp2pFdiR3olxtIzMZaUhFjat22aczoDO/VfRCQS5ZdWkLVtN2vzSuu+dpSSnb+Hyura/2zTPb496UlxnD44mf7d4+iX1Im0bnVfiR3bYdbYWpGmp0IXkVYrv7SC5TklrMjdxcqtJazatvsLI+4enTtwbM/OnDIgiUE94jkmJZ5jkhNI7NQuwNQHp0IXkVahqqaWVdt2k7mpiCVbilm2ZRfbSioAiGljDEqJ56vHdGdo784c3zuRIT0T6BrXPuDUh0eFLiItUmV1DctzSvhow04WbNzJ0pxiKqrqpk1Su3YkI70bI/p2YURqIsf3TqRj+5iAEx89FbqItAjuztodpfx7XSHz1hew8NMiKqtrMYOhvTozcUwaGf26kZHelR6dY4OO2yRU6CIStcr3VfPB+kLeWZ3Pv9bls2N33fz3oJR4Lh+bxikDkhjbPyli57zDTYUuIlGluGwfb2Xt4PVVeczPLqSyupaEDm05fXAyZwxO5rTB3emV2DHomIFQoYtIxNtVvo/XVuYx95PtfLhhJzW1TmrXjlw+No2zj+vBSendmmxtdzRRoYtIRKqoquG9Nfm8uHQr763Np6rGSU/qxOTTB/DNE3pxfO/Oga33jlQqdBGJKCu3lvBsZg4vLd3K7opqUhI6cPUp6Vw4so9K/BBU6CISuLLKal5cupXZi7awcutuOrRtw7nDenLp6L6cMjCJmDYq8VCo0EUkMJ8WljH9o808tziH0opqjuvVmbvHH8/4EX1azcqUcFKhi0izcncWflrE4/M28u6afNrFGOed0IvvnpLOqLQumlI5Cip0EWkWNbXOm6vyeGzeRpbn7CIprj23nTWIK8amkdJCT/Rpbip0EWlSNbXOy8u38dC769lYUEa/pE7ce+EwLh2dSmy76D/dPpKo0EWkSexf5EN6JvDw5SM5d1gvHeRsIip0EQkrd+ed1fnc/8Ya1u3Yw5CeCTx25SjOGdqTNiryJqVCF5GwWby5iD+8toZFm4rp3z2ORy4fxbnDVOTNRYUuIkdt6669/H7ual5ZsZ3u8R2498JhfOekvrSL0en4zUmFLiJHbO++Gh6ft4HH3t+AO9x21iC+d8YAOrVXtQRBf3UROSJvZ+3gV3NWsXXXXr45vBc/P+84+nRpnVc5jBQqdBE5LHklFfx6zipeX5XH4B7xzJ58MicPSAo6lqBCF5EQ1dY60xds5oE31lJVU8tPxx3LpK8O0GVrI4gKXUQOacvOcn78/HIWflrEaYO6c++Fw+iXFBd0LNmPCl1EDqq21pn58WZ+/9oaYsy4/9LhTBidquutRCgVuog0asfuCm5/djkfZBdy2qDu3HfJcHrroGdEU6GLyAHeztrBT55fTkVVLb+9aBiXj0nTqDwKqNBF5D8qqmr43dzVTPtoM0N7deahiSM5JiU+6FgSIhW6iACwqbCMG2csZk1eKZO+2p+fjDuWDm11NcRookIXEV5fmcdPnltOTIzxj2tO4r+GpAQdSY6ACl2kFauuqeX+N9YyZd5Ghqcm8v+uGEVq105Bx5IjpEIXaaWKy/bx/ZlL+GjjTq48OY3/OX+opliiXEineJnZODNba2bZZnZHI6+nmdl7ZrbUzFaY2Xnhjyoi4bI2r5QLHvmAxVuKeXDCCO698ASVeQtwyBG6mcUAjwBfB3KBRWY2x92zGmz2S+BZd3/UzIYCc4H0JsgrIkfprawd/HD2UuI6tOWZySczMq1r0JEkTEKZchkDZLv7RgAzmw2MBxoWugOd679PBLaFM6SIHD13Z8q8jfzh9TWc0CeRKVdl0DNRN2duSUIp9D5AToPHucDY/bb5NfCmmd0CxAFnN7YjM5sMTAZIS0s73KwicoSqa2r51ZxVzPx4C+cP78WDE0boBs0tUChz6I2dHub7PZ4ITHX3VOA8YLqZHbBvd5/i7hnunpGcnHz4aUXksJVVVjN5+mJmfryFG88YyEOXjVSZt1ChjNBzgb4NHqdy4JTK9cA4AHf/yMxige5AfjhCisiRKSit5NqpC8natpt7LxzGlSf3CzqSNKFQRuiLgEFm1t/M2gOXAXP222YLcBaAmR0HxAIF4QwqIocnp6icSx/7kA35Zfztuxkq81bgkCN0d682s5uBN4AY4El3X2VmdwOZ7j4HuB34m5n9iLrpmGvcff9pGRFpJmvzSrnq7x9TWV3LjEljGd1PK1lag5BOLHL3udQtRWz43F0Nvs8CvhLeaCJyJBZvLua6qYuIbdeG5248hcE9EoKOJM1EZ4qKtCAfZhdy/VOZ9OjcgenXj6VvN53G35qo0EVaiPfXFTB5WibpSXHMmDSW5IQOQUeSZqZCF2kB3s7awfdnLuGYlHhmTBpLt7j2QUeSAOh23SJR7vWV27lxxmKO65XArBtOVpm3Yhqhi0Sx11fm8YOnlzIiNZGp142hc2y7oCNJgFToIlHqndU7uGXWEoanJjLt+rHEd9DHubXTlItIFHp/XQE3zVjCcb06M/XaMSpzAVToIlFnfnYhk6dlckxKPNOuG0NiR02zSB0VukgUWby5mElPfb40sUsnHQCVz6nQRaLEmrzdXPuPhXUnDU0ao9UscgAVukgU2FRYxlV/X0in9m2Zfv1YUhJ0Ywo5kApdJMLllVRw5d8/prqmlhmTxuh0fjkoHRoXiWAle6u4+smF7Cqv4ukbxnJMii60JQenEbpIhKqoquGGaZlsLNzD41eNZnhql6AjSYTTCF0kAtXUOv/97DIWflrEQxNH8pVjugcdSaKARugiEcbdueeVLOZ+kscvv3kcF4zoHXQkiRIqdJEI87d/b2Tqh5uY9NX+TDptQNBxJIqo0EUiyNxPtvO7uWs4f3gvfn7ecUHHkSijQheJEEu2FPOjZ5Yxul9XHpwwgjZtLOhIEmVU6CIRYMvOcm54KpOeibFMuWo0se1igo4kUUiFLhKwkvIqrp26kOpa58lrTiIpXreOkyOjQhcJUFVNLT94eglbisqZctVoBibHBx1JopjWoYsE6J5Xsvggu5AHLh3O2AFJQceRKKcRukhApi/YzLSPNjP59AFMyOgbdBxpAVToIgGYn13Ir+es4qwhKfxs3JCg40gLoUIXaWabCsv4/swlDEyO48+XnUiMlidKmKjQRZrRnspqbpiWSRuDJ757Egmxun2chI8Oioo0k9pa50fPLGNjYRnTrxtDWpKuay7hpRG6SDP5yzvreStrB7847zhO1dUTpQmo0EWawesr8/jLO+u5ZFQq134lPeg40kKp0EWa2Podpdz+7DJG9O3Cby8ahpkOgkrTUKGLNKHdFVV8b/piOraP4fErdY0WaVohFbqZjTOztWaWbWZ3HGSbb5tZlpmtMrOnwxtTJPrU1jq3P7uczUXlPHL5KHomxgYdSVq4Q65yMbMY4BHg60AusMjM5rh7VoNtBgF3Al9x92IzS2mqwCLR4tH3N/BW1g7uOn+oTuuXZhHKCH0MkO3uG919HzAbGL/fNjcAj7h7MYC754c3pkh0eX9dAQ++uZbxJ/bWQVBpNqEUeh8gp8Hj3PrnGhoMDDaz+Wa2wMzGNbYjM5tsZplmlllQUHBkiUUiXE5RObfNXsqxPRL4/cUn6CCoNJtQCr2xd6Pv97gtMAj4GjAReMLMuhzwQ+5T3D3D3TOSk5MPN6tIxKuoquH7M5dQU+s8duVoOrXXuXvSfEIp9Fyg4aXgUoFtjWzzT3evcvdPgbXUFbxIq/Kbl7P4ZGsJ/zthBOnd44KOI61MKIW+CBhkZv3NrD1wGTBnv21eAv4LwMy6UzcFszGcQUUi3XOZOcxauIWbvjaQc47vGXQcaYUOWejuXg3cDLwBrAaedfdVZna3mV1Qv9kbwE4zywLeA37i7jubKrRIpMnatptfvrSSUwYkcfvXBwcdR1opc99/Orx5ZGRkeGZmZiC/WyScdldU8a2/fkBFVQ2v3HIayQm6J6g0HTNb7O4Zjb2mIzYiR8Hd+elzK8gt3sszk09WmUugdOq/yFF4cv4mXl+Vxx3jhpCR3i3oONLKqdBFjtDizcX8fu5qzhnag0mn9Q86jogKXeRIFJXt4+anl9C7S0cemDBCJw9JRNAcushh+uzOQzvL9vHCTaeS2FG3kZPIoBG6yGF69P0NvL+ugLvOH8qwPolBxxH5DxW6yGFYsHEn//vmWi4Y0ZsrxqYFHUfkC1ToIiEqKK3k1llLSU+K43e66JZEIM2hi4Sgpn7evGRvFU9dN4b4DvroSOTRu1IkBA+/m80H2YXcd8kJHNerc9BxRBqlKReRQ/gwu5A/v7OOi0f24dsZfQ/9AyIBUaGLfIn80gpunb2Mgcnx3HvRMM2bS0TTlIvIQdTUOj+cvYw9lVXMnDRWN6uQiKd3qMhB/PXd9Xy4YSf3XzqcY3smBB1H5JA05SLSiPnZhfzlnfVcPKoPE0anBh1HJCQqdJH95JdWcNvsZQzoHsc94zVvLtFDUy4iDdTUOrfN+nzePE7rzSWK6N0q0sBD76zno42aN5fopCkXkXrzswt56N31XDIqVevNJSqp0EX4fN58YHI891x4fNBxRI6Iplyk1aupdW6dtZQ9lVU8fYPWm0v00jtXWr0/v72OBRuLeHDCCAb30Ly5RC9NuUirNm9dAQ+/l82E0alcqvXmEuVU6NJq5ZVU8MNnljE4JYG7xw8LOo7IUVOhS6tUXVPLrbOWUlFVwyNXjKJj+5igI4kcNc2hS6v0wJtrWbipiD9/50SOSYkPOo5IWGiELq3OW1k7ePz9jVw+No0LR/YJOo5I2KjQpVXJKSrn9meXMaxPZ+46f2jQcUTCSoUurUZldQ0/eHoJDvy/y0cT207z5tKyaA5dWo17X1nNitwSplw1mrSkTkHHEQk7jdClVXhp6VamL9jM5NMHcM7xPYOOI9IkVOjS4q3NK+XOFz5hTP9u/PQbxwYdR6TJhFToZjbOzNaaWbaZ3fEl211qZm5mGeGLKHLkSiuquGnGYuJj2/LwxJG0jdEYRlquQ767zSwGeAQ4FxgKTDSzA5YHmFkCcCvwcbhDihwJd+enz69gc1E5D08cSUrn2KAjiTSpUIYrY4Bsd9/o7vuA2cD4Rra7B7gfqAhjPpEj9sS/P+W1lXncMW4IYwckBR1HpMmFUuh9gJwGj3Prn/sPMxsJ9HX3V75sR2Y22cwyzSyzoKDgsMOKhOrDDYX84fU1jDu+J5NO6x90HJFmEUqhN3aHXP/Pi2ZtgD8Btx9qR+4+xd0z3D0jOTk59JQih2Hbrr3c8vRS0pM68cCE4brJs7QaoRR6LtDwflypwLYGjxOAYcC/zGwTcDIwRwdGJQgVVTXcNGMxldW1PH5VBgmx7YKOJNJsQin0RcAgM+tvZu2By4A5n73o7iXu3t3d0909HVgAXODumU2SWORL/OblVSzPLeHBCSN00S1pdQ5Z6O5eDdwMvAGsBp5191VmdreZXdDUAUVCNWvhFmYtzOH7XxvIuGE6eUhan5BO/Xf3ucDc/Z676yDbfu3oY4kcnsWbi7jrnys5bVB3bj9HJw9J66SzLCTq5ZVUcOOMJfTu0pGHJ44ipo0OgkrrpItzSVSrqKrhezMWU15ZzcxJY0nspIOg0nqp0CVquTu/fGkly3N28fhVoxncIyHoSCKB0pSLRK2pH27i+cW53HbWIL6hKyiKqNAlOv1rbT73vJLFOUN7cNtZg4KOIxIRVOgSdbLzS7nl6aUc27Mzf/rOibTRQVARQIUuUaa4bB/XP5VJh3YxPHF1BnEddBhI5DMqdIkaVTW13DRzMdt3VfD4VaPp06Vj0JFEIoqGNxIV3J1fvriSBRuL+OO3RzC6X9egI4lEHI3QJSo8+v4GnsnM4dYzj+HiUalBxxGJSCp0iXgvL9/G/a+vZfyJvfnR1wcHHUckYqnQJaIt3lzE7c8t56T0rtx3ia5tLvJlVOgSsTYVlnHDtMX0Tozl8asyiG0XE3QkkYimQpeIVFBayXefXIi78+Q1J9Etrn3QkUQinla5SMQpq6zmuqmLKCit5OkbxjIgWTeqEAmFCl0iSt1a8yVkbd/N3747mpFpWp4oEipNuUjEqK11fvZ/K5i3roDfXjiMM4f0CDqSSFRRoUtEcHfufXU1LyzZyo/OHsxlY9KCjiQSdVToEhH++m42T87/lGtOTefWs44JOo5IVFKhS+Cmzv+UP761jotH9eGu84dqrbnIEVKhS6BeWJLLr1/O4utDe3D/JcN1KVyRo6BCl8C8umI7P35uOacMSOKvE0fSNkZvR5GjoU+QBOKNVXncNnspo9K68sTVOgtUJBxU6NLs3l2zg5ufXsKwPon849qTdJMKkTBRoUuzen9dATdOX8KQnp156roxJMS2CzqSSIuhoZE0m/fW5PO9GYsZmBLP9OvHkNhRZS4SThqhS7N4Y1Uek6dncmyPBGbdMJYunXSxLZFw0whdmtyrK7Zz2+ylDOuTyFPXaWQu0lQ0Qpcm9eLSXG6ZtYSRaV00zSLSxDRClybzj/mf8puXszhlQBJPXJ2h1SwiTUyfMAk7d+dPb6/noXfWc87QHjw0caTWmYs0AxW6hFVtrfPrl1cx7aPNfDsjld9ddILOABVpJiF90sxsnJmtNbNsM7ujkdf/28yyzGyFmb1jZv3CH1UiXUVVDTfPWsK0jzbzvdMHcN8lw1XmIs3okJ82M4sBHgHOBYYCE81s6H6bLQUy3H048Dxwf7iDSmQrKtvHFU98zGsr8/jlN4/jzvOO01UTRZpZKFMuY4Bsd98IYGazgfFA1mcbuPt7DbZfAFwZzpAS2TYVlnHNPxayraSCRy4fxXkn9Ao6kkirFEqh9wFyGjzOBcZ+yfbXA68dTSiJHos2FfG96Ytxd2bdMJbR/boFHUmk1Qql0Bv7d7M3uqHZlUAGcMZBXp8MTAZIS9MtxqLd7IVb+J9/riS1ayeevOYk+nePCzqSSKsWSqHnAn0bPE4Ftu2/kZmdDfwCOMPdKxvbkbtPAaYAZGRkNPofBYl81TW13PvqaqZ+uInTByfz18tGkthJJwyJBC2UQl8EDDKz/sBW4DLg8oYbmNlI4HFgnLvnhz2lRIyisn3cMmsJ87N3Mumr/bnj3CFaySISIQ5Z6O5ebWY3A28AMcCT7r7KzO4GMt19DvAAEA88V7+yYYu7X9CEuSUAS7cU84OZSygs28cDlw5nQkbfQ/+QiDSbkE4scve5wNz9nrurwfdnhzmXRBB3Z9pHm7n31Sx6dI7lhZtOZVifxKBjich+dKaofKnSiip+/uJKXl6+jbOGpPDHb5+o+XKRCKVCl4NasqWY22YvZWvxXn7yjWO56YyBtGmjk4VEIpUKXQ5QU+s8+q9s/vT2enp2juW5G0/R+nKRKKBCly/YsrOcHz+/nIWfFvGtEb2598Jhuoa5SJRQoQtQd5XEmQu38Pu5q4kx48EJI7hkVB9dj0UkiqjQhdzicn72fyuYn72T0wZ1575LhtO7S8egY4nIYVKht2LVNbX8Y/4m/vjWOtoY/P7iE7jspL4alYtEKRV6K7U8Zxd3vvAJWdt3c9aQFH4z/nhSu3YKOpaIHAUVeitTXLaP/31rLTM/3kJyfAcevWIU44b11KhcpAVQobcS1TW1zPx4C398ax17Kqu5+pR0/vucwXSO1QoWkZZChd7CuTvz1hfyu1dXs3ZHKacOTOJX3zqeY3smBB1NRMJMhd6CLc/ZxX2vr+HDDTvp260jj105im8cr+kVkZZKhd4CZeeX8qe31/Pqiu10i2vPr741lMvHptGhbUzQ0USkCanQW5D1O0p56N1sXlmxjY7tYrj1rEHccFp/EjRPLtIqqNBbgFXbSnj0Xxt49ZPtdGwXw41nDOSG0wbQLa590NFEpBmp0KOUuzM/eyePz9vAv9cXEtc+hpvOGMgkFblIq6VCjzIVVTXMWbaNqR9uImv7bpITOvCzcUO4fGyaLqIl0sqp0KNETlE5Mz7ezDOLcthVXsXgHvH84eITuGhUHx3sFBFAhR7RKqtreCtrB88syuGD7EIMOGdoT64+NZ2TB3TT8kMR+QIVeoRxd1bklvDSsq28tHQrxeVV9OnSkdvOGsS3M/rqKogiclAq9AjxaWEZLy/fxktLt7KxsIz2MW04e2gK3zkpja8e050Y3fpNRA5BhR6g7PxS5n6Sx9xPtrMmrxSAsf27Mfn0AZw7rJduxiwih0WF3oyqa2rJ3FzM21k7eGdNPp8WlgGQ0a8r/3P+UM4d1lNTKiJyxFToTWzbrr3MW1fAv9cX8kF2ISV7q2gXY5w8IIlrTk3nG8f3pGdibNAxRaQFUKGH2c49lSzYWMSCjTv5cEMhGwrqRuE9OnfgnKE9OHNICqcNTia+g/70IhJeapWj4O5s3llO5uZiFm8uInNTMevz9wAQ1z6GjPRuTByTxumDkxmUEq9lhiLSpFToh6FwTyWfbC1hec4ulufsYkVuCTvL9gGQENuW0f26ctGoPpw8IIkT+iTSLqZNwIlFpDVRoTeiptbZtLOMtXmlrNm+m1XbdrNyWwk7dlcCYAaDUuI5c0gKJ6Z1IaNfNwalxNNGSwtFJECtutArq2vYsrOc7Pw9bCjYQ3b+Htbn1/1vZXUtAG0MBibHc8qAJIb1SeT43omckJqoOXARiTgtvpVK9laRU1RObnE5W4rKySnay6adZWwsKGNbyV7cP9+2T5eODEyJ59SBSRzbszPH9khgUI94YtvpWikiEvmittDdndLKavJ3V7JjdwV5JRXk7a5ge8letu2qYGvxXrbt2ktpZfUXfq5zbFv6d48jI70r6Ump9O8ex8DkeAYkxxGnUbeIRLGoa7BnFm3hkfc2kF9aQUVV7QGvd+3Ujl6JHUlL6sQpA5Po3SWWvl070bdb3ZcuMSsiLVVIhW5m44C/ADHAE+7+h/1e7wBMA0YDO4HvuPum8EatkxTXgRP7diEloQMpnTvQo3MsyQkd6J3YkZ6JsZoeEZFW65CFbmYxwCPA14FcYJGZzXH3rAabXQ8Uu/sxZnYZcB/wnaYIfPbQHpw9tEdT7FpEJKqFslB6DJDt7hvdfR8wGxi/3zbjgafqv38eOMt0Fo2ISLMKpdD7ADkNHufWP9foNu5eDZQASeEIKCIioQml0BsbafsRbIOZTTazTDPLLCgoCCWfiIiEKJRCzwX6NnicCmw72DZm1hZIBIr235G7T3H3DHfPSE5OPrLEIiLSqFAKfREwyMz6m1l74DJgzn7bzAGurv/+UuBddz9ghC4iIk3nkKtc3L3azG4G3qBu2eKT7r7KzO4GMt19DvB3YLqZZVM3Mr+sKUOLiMiBQlqH7u5zgbn7PXdXg+8rgAnhjSYiIodD13cVEWkhLKipbjMrADYf4Y93BwrDGCecIjVbpOaCyM0WqbkgcrNFai5oOdn6uXujq0oCK/SjYWaZ7p4RdI7GRGq2SM0FkZstUnNB5GaL1FzQOrJpykVEpIVQoYuItBDRWuhTgg7wJSI1W6TmgsjNFqm5IHKzRWouaAXZonIOXUREDhStI3QREdmPCl1EpIWI+kI3sx+bmZtZ96CzfMbM7jGzFWa2zMzeNLPeQWcCMLMHzGxNfbYXzaxL0Jk+Y2YTzGyVmdWaWeBLy8xsnJmtNbNsM7sj6DyfMbMnzSzfzFYGnaUhM+trZu+Z2er6/x9vCzrTZ8ws1swWmtny+my/CTpTQ2YWY2ZLzeyVo91XVBe6mfWl7k5KW4LOsp8H3H24u58IvALcdagfaCZvAcPcfTiwDrgz4DwNrQQuBuYFHaTBXbrOBYYCE81saLCp/mMqMC7oEI2oBm539+OAk4EfRNDfrBI4091HACcC48zs5IAzNXQbsDocO4rqQgf+BPyURq69HiR3393gYRwRks/d36y/AQnAAuouhRwR3H21u68NOke9UO7SFQh3n0cjl6YOmrtvd/cl9d+XUldQ+98IJxBeZ0/9w3b1XxHxmTSzVOCbwBPh2F/UFrqZXQBsdfflQWdpjJn91sxygCuInBF6Q9cBrwUdIkKFcpcuOQgzSwdGAh8Hm+Rz9dMay4B84C13j5Rsf6ZuUFobjp2FdLXFoJjZ20DPRl76BfBz4JzmTfS5L8vm7v90918AvzCzO4GbgV9FQq76bX5B3T+RZzZHpsPJFiFCugOXHMjM4oH/A364379UA+XuNcCJ9ceNXjSzYe4e6HEIMzsfyHf3xWb2tXDsM6IL3d3Pbux5MzsB6A8sr78XdQpeuAsAAAFUSURBVCqwxMzGuHtekNka8TTwKs1U6IfKZWZXA+cDZzX3TUgO428WtFDu0iX7MbN21JX5THd/Ieg8jXH3XWb2L+qOQwR9YPkrwAVmdh4QC3Q2sxnufuWR7jAqp1zc/RN3T3H3dHdPp+4DOKq5yvxQzGxQg4cXAGuCytKQmY0DfgZc4O7lQeeJYKHcpUsasLqR1d+B1e7+x6DzNGRmyZ+t6DKzjsDZRMBn0t3vdPfU+g67jLo7vR1xmUOUFnoU+IOZrTSzFdRNC0XKEq6HgQTgrfollY8FHegzZnaRmeUCpwCvmtkbQWWpP3D82V26VgPPuvuqoPI0ZGazgI+AY80s18yuDzpTva8AVwFn1r+3ltWPPCNBL+C9+s/jIurm0I96iWAk0qn/IiIthEboIiIthApdRKSFUKGLiLQQKnQRkRZChS4i0kKo0EVEWggVuohIC/H/AUaRtnfCZRnZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(s_1, s)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "x_1 = np.arange(0, 1, 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAASkElEQVR4nO3dX4xcd3XA8e+xN5sIkZCqNgTFNk5VR8VErUJXaSqkEhQaOXmwXyiypZRSRVjQhj6AKqWiCig8FdQioboFt0UUJAiBB1gho0ilQSCEqRcFAnaUyjV/sgpqDE3zQMrdmd3Th5ldj9bz59rMztzf7PcjWZo78/Ps+WXvObk+87vzi8xEklS+HdMOQJI0HhZ0SZoRFnRJmhEWdEmaERZ0SZoRc9P6wbt27cr9+/dP68dLUpG+853v/Cwzd/d7bWoFff/+/SwtLU3rx0tSkSLix4Nes+UiSTPCgi5JM8KCLkkzwoIuSTPCgi5JM2JkQY+IT0TE8xHxgwGvR0R8NCLOR8RTEfH68YcpSRqlzhX6J4FDQ16/FzjQ/XMc+MdfPSxJ0pUauQ49M78eEfuHDDkCfCo738N7OiJujIhXZ+ZPxxSjGiIz+dS3fszPf7ECfu2ydNXufu2r+J29N479fcdxY9HNwLM9x8vd5y4r6BFxnM5VPPv27RvDj9YkXfjZL3j/4tmN44gpBiMV7JU3XNfYgt4vrftevmXmSeAkwMLCgpd4hfm/lVUATv7x73LP626acjSSNhvHKpdlYG/P8R7guTG8rxqmancK+rXX7JxyJJL6GUdBXwTe1l3tcifwov3z2VS11gC4ds7VrlITjWy5RMRngbuAXRGxDLwfuAYgMz8GnALuA84DLwF/ulXBarqqtgVdarI6q1yOjXg9gT8fW0RqrEsF3ZaL1EReaqm2Sz10TxupicxM1WbLRWo2M1O12XKRms2Crtqqli0XqcnMTNVmy0VqNjNTta0X9PmdnjZSE5mZqq1qr3Lt3A7CL3GRGsmCrtqq1prtFqnBzE7VVrXX/B4XqcEs6KptveUiqZnMTtVWtW25SE1mdqq2Tg/dlovUVBZ01Va1V72pSGows1O12XKRms3sVG2dgm7LRWoqC7pqq1qucpGazOxUbSuuQ5cazYKu2uyhS81mdqo2byySms3sVG2uQ5eazYKu2qr2GvNeoUuNZXaqlrW1ZGXVHrrUZGanallZ7e5W5J2iUmOZnaqlarlBtNR0FnTVUrW7G0TbcpEay+xULW4QLTWf2alaNgq6d4pKjWVBVy22XKTmMztViy0XqfnMTtXiKhep+SzoqmWj5eI6dKmxamVnRByKiGci4nxEPNTn9X0R8UREPBkRT0XEfeMPVdNky0VqvpHZGRE7gRPAvcBB4FhEHNw07K+BxzLzduAo8A/jDlTTdamg23KRmqrO5dYdwPnMvJCZK8CjwJFNYxK4ofv4FcBz4wtRTVC1XOUiNV2d7LwZeLbneLn7XK8PAPdHxDJwCnh3vzeKiOMRsRQRSxcvXryKcDUtl9ahW9ClpqqTndHnudx0fAz4ZGbuAe4DPh0Rl713Zp7MzIXMXNi9e/eVR6upseUiNV+dgr4M7O053sPlLZUHgMcAMvNbwHXArnEEqGbwxiKp+epk5xngQETcEhHzdD70XNw05ifA3QAR8Vo6Bd2eygy5tA7dgi411cjszMw28CDwOPA0ndUsZyPikYg43B32XuAdEfE94LPA2zNzc1tGBVvfrSiiXwdOUhPM1RmUmafofNjZ+9zDPY/PAW8Yb2hqEjeIlprPDFUtVdsNoqWms6CrlqrlfqJS05mhqqVqr7oGXWo4M1S12HKRms+Crlo6Bd3TRWoyM1S1VC1XuUhNZ4aqlvV16JKaywxVLfbQpeazoKsWV7lIzWeGqhbXoUvNZ4aqFlsuUvNZ0FWL3+UiNZ8Zqlqq9po9dKnhzFCNlJms2HKRGs+CrpFWVt3cQiqBGaqRLu0n6ukiNZkZqpE2tp+7xpaL1GQWdI3kBtFSGcxQjWTLRSqDGaqRNlournKRGs2CrpE2Wi6uQ5cazQzVSLZcpDKYoRrpUkG35SI1mQVdI1UtV7lIJTBDNdL6Ffp19tClRjNDNZItF6kMFnSN5I1FUhnMUI3kOnSpDBZ0jbTRcrGHLjWaGaqR1lsu8zs9XaQmq5WhEXEoIp6JiPMR8dCAMW+NiHMRcTYiPjPeMDVNVXuN+Z072LEjph2KpCHmRg2IiJ3ACeAPgWXgTEQsZua5njEHgL8C3pCZL0TEK7cqYE1e1VrzA1GpAHWy9A7gfGZeyMwV4FHgyKYx7wBOZOYLAJn5/HjD1DRV7VXmLehS49XJ0puBZ3uOl7vP9boVuDUivhkRpyPiUL83iojjEbEUEUsXL168uog1cVXbK3SpBHWytF/jNDcdzwEHgLuAY8A/R8SNl/2lzJOZuZCZC7t3777SWDUlVXvN3YqkAtQp6MvA3p7jPcBzfcZ8KTNbmflD4Bk6BV4zoGqteoUuFaBOlp4BDkTELRExDxwFFjeN+SLwJoCI2EWnBXNhnIFqemy5SGUYmaWZ2QYeBB4HngYey8yzEfFIRBzuDnsc+HlEnAOeAP4yM3++VUFrsqr2qneJSgUYuWwRIDNPAac2Pfdwz+ME3tP9oxlTtdd4+bW1ThVJU+S/ozWS69ClMpilGsmWi1QGC7pGWln1Cl0qgVmqkarWmt+0KBXALNVInWWLtlykprOga6ROD91TRWo6s1RDZaY3FkmFMEs1VGs1ycTvcpEKYEHXUG4QLZXDLNVQG/uJWtClxjNLNdSlgm7LRWo6C7qGqlrdlovr0KXGM0s1lC0XqRxmqYay5SKVw4KuodZbLm4SLTWfWaqh1q/Qr7OHLjWeWaqhbLlI5bCgayhvLJLKYZZqqKrlFbpUCgu6hlpvufihqNR8ZqmGsuUilcMs1VAbH4q6ykVqPLNUQ6330Od3eqpITWeWaqiqvcrcjmDOgi41nlmqodytSCqHmaqhqvaquxVJhbCga6iq5RW6VAozVUPZcpHKYaZqqJX2mneJSoWwoGuoTg/d00QqgZmqoWy5SOWolakRcSginomI8xHx0JBxb4mIjIiF8YWoaapsuUjFGFnQI2IncAK4FzgIHIuIg33GXQ/8BfDtcQep6anaq16hS4Wok6l3AOcz80JmrgCPAkf6jPsg8CHgl2OMT1NWtdbsoUuFqJOpNwPP9hwvd5/bEBG3A3sz88vD3igijkfEUkQsXbx48YqD1eTZcpHKUaegR5/ncuPFiB3AR4D3jnqjzDyZmQuZubB79+76UWpqbLlI5aiTqcvA3p7jPcBzPcfXA7cBX4uIHwF3Aot+MDobXOUilaNOpp4BDkTELRExDxwFFtdfzMwXM3NXZu7PzP3AaeBwZi5tScSaqE4P3ZaLVIKRBT0z28CDwOPA08BjmXk2Ih6JiMNbHaCmJzNtuUgFmaszKDNPAac2PffwgLF3/ephqQnaa8lauv2cVAozVQNtbD/nKhepCBZ0DVS1uhtEuw5dKoKZqoEuXaF7mkglMFM1kC0XqSwWdA1UtbstF6/QpSKYqRqoanWv0O2hS0UwUzXQestlfqctF6kEFnQNtNFy8QpdKoKZqoE2Wi720KUimKkayFUuUlks6BrIVS5SWcxUDbRxhW4PXSqCmaqBNm79t+UiFcGCroG89V8qi5mqgVYs6FJRzFQNVLXX2LkjmNvpaSKVwEzVQO5WJJXFbNVAbhAtlcVs1UBVa80VLlJBLOgaqGqvugZdKojZqoFsuUhlMVs1UKeg23KRSmFB10CucpHKYrZqoKq1Zg9dKojZqoFsuUhlsaBrIFsuUlnMVg3kKhepLGarBvLGIqksFnQN5I1FUlnMVg1ky0UqS61sjYhDEfFMRJyPiIf6vP6eiDgXEU9FxFcj4jXjD1WT5ioXqSwjC3pE7AROAPcCB4FjEXFw07AngYXM/G3gC8CHxh2oJqu9usbqWnqFLhWkTrbeAZzPzAuZuQI8ChzpHZCZT2TmS93D08Ce8YapSXODaKk8dbL1ZuDZnuPl7nODPAB8pd8LEXE8IpYiYunixYv1o9TErRf0eXcrkopRJ1ujz3PZd2DE/cAC8OF+r2fmycxcyMyF3bt3149SE1e1VwG49hp76FIp5mqMWQb29hzvAZ7bPCgi3gy8D3hjZlbjCU/TUrXcIFoqTZ1sPQMciIhbImIeOAos9g6IiNuBjwOHM/P58YepSdvoobvKRSrGyIKemW3gQeBx4Gngscw8GxGPRMTh7rAPAy8HPh8R342IxQFvp0JstFy8QpeKUaflQmaeAk5teu7hnsdvHnNcmjJXuUjlMVvV16Ueui0XqRQWdPW1smrLRSqN2aq+Nq7QbblIxTBb1ZerXKTyWNDVl6tcpPKYrerr0hW6p4hUCrNVfV3qodtykUphQVdftlyk8pit6qtqr7EjYG5Hv+9mk9REFnT1tb5bUYQFXSqFBV19VS03iJZKY8aqLzeIlspjxqovN4iWymNBV19Ve9UrdKkwZqz6qlpr9tClwpix6suWi1QeC7r6suUilceMVV+ucpHKY8aqr6ply0UqjQVdfVVtbyySSmPGqi9bLlJ5zFj15SoXqTwWdPVVtVaZ9wpdKooZq75suUjlMWN1mfbqGu21tOUiFcaCrsusrK5vP+fpIZXEjNVlVtwgWiqSGavLVBsF3ZaLVBILui5TtbxCl0pkxuoyVXsVsIculcaM1WVsuUhlqlXQI+JQRDwTEecj4qE+r18bEZ/rvv7tiNg/7kA1ORtX6LZcpKKMzNiI2AmcAO4FDgLHIuLgpmEPAC9k5m8CHwH+ZtyBanLsoUtlmqsx5g7gfGZeAIiIR4EjwLmeMUeAD3QffwH4+4iIzMwxxgrAY2ee5Z++cWHcbysggczkpZX1HrotF6kkdQr6zcCzPcfLwO8NGpOZ7Yh4Efh14Ge9gyLiOHAcYN++fVcV8I0vu4YDr3r5Vf1djRYEBNxw3TX81k3XTzscSVegTkGPPs9tvvKuM4bMPAmcBFhYWLiqq/d7XncT97zupqv5q5I00+o0SZeBvT3He4DnBo2JiDngFcD/jCNASVI9dQr6GeBARNwSEfPAUWBx05hF4E+6j98C/PtW9M8lSYONbLl0e+IPAo8DO4FPZObZiHgEWMrMReBfgE9HxHk6V+ZHtzJoSdLl6vTQycxTwKlNzz3c8/iXwB+NNzRJ0pVwobEkzQgLuiTNCAu6JM0IC7okzYiY1urCiLgI/Pgq//ouNt2Fuk1sx3lvxznD9pz3dpwzXPm8X5OZu/u9MLWC/quIiKXMXJh2HJO2Hee9HecM23Pe23HOMN5523KRpBlhQZekGVFqQT857QCmZDvOezvOGbbnvLfjnGGM8y6yhy5JulypV+iSpE0s6JI0Ixpd0Lfj5tQ15vyeiDgXEU9FxFcj4jXTiHPcRs27Z9xbIiIjovjlbXXmHBFv7f6+z0bEZyYd41aocY7vi4gnIuLJ7nl+3zTiHKeI+EREPB8RPxjwekTER7v/TZ6KiNdf1Q/KzEb+ofNVvf8F/AYwD3wPOLhpzJ8BH+s+Pgp8btpxT2DObwJe1n38rtLnXHfe3XHXA18HTgML0457Ar/rA8CTwK91j1857bgnNO+TwLu6jw8CP5p23GOY9x8Arwd+MOD1+4Cv0Nn97U7g21fzc5p8hb6xOXVmrgDrm1P3OgL8a/fxF4C7I6LfdnilGDnnzHwiM1/qHp6ms4NU6er8rgE+CHwI+OUkg9sideb8DuBEZr4AkJnPTzjGrVBn3gnc0H38Ci7fIa04mfl1hu/idgT4VHacBm6MiFdf6c9pckHvtzn1zYPGZGYbWN+culR15tzrATr/Vy/dyHlHxO3A3sz88iQD20J1fte3ArdGxDcj4nREHJpYdFunzrw/ANwfEct09mF492RCm6orzf2+am1wMSVj25y6ILXnExH3AwvAG7c0oskYOu+I2AF8BHj7pAKagDq/6zk6bZe76PxL7BsRcVtm/u8Wx7aV6sz7GPDJzPzbiPh9Oruh3ZaZa1sf3tSMpZY1+Qp9O25OXWfORMSbgfcBhzOzmlBsW2nUvK8HbgO+FhE/otNjXCz8g9G65/eXMrOVmT8EnqFT4EtWZ94PAI8BZOa3gOvofIHVLKuV+6M0uaBvx82pR86523r4OJ1iPgs9VRgx78x8MTN3Zeb+zNxP57ODw5m5NJ1wx6LO+f1FOh+CExG76LRgLkw0yvGrM++fAHcDRMRr6RT0ixONcvIWgbd1V7vcCbyYmT+94neZ9qe/Iz4Zvg/4Tzqfir+v+9wjdJIZOr/ozwPngf8AfmPaMU9gzv8G/Dfw3e6fxWnHPIl5bxr7NQpf5VLzdx3A3wHngO8DR6cd84TmfRD4Jp0VMN8F7pl2zGOY82eBnwItOlfjDwDvBN7Z87s+0f1v8v2rPb+99V+SZkSTWy6SpCtgQZekGWFBl6QZYUGXpBlhQZekGWFBl6QZYUGXpBnx/2XgSTIHofZHAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_h = 1 * sgm(1000 * x_1 -300)\n",
    "plt.plot(x_1, y_h)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAW6ElEQVR4nO3db4xcV33G8e+zM7ubIkJS1Q6h/oOD6lBM1Cp0laZCKkGByIlUW5UosqWUUkVY0AZegCqlogoovCqIIqGagmkRBQlC4AVYyChSaSJQhGmMAgE7SuWaP7ESNYamaUWa2Z2dX1/cubPDetZzd313Zs89z0eytDNzvXtudn4/nzz33DmKCMzMLH0z0x6AmZnVww3dzKwh3NDNzBrCDd3MrCHc0M3MGqI9rR+8bdu22LNnz7R+vJlZkr73ve/9PCK2j3ptag19z549nDp1alo/3swsSZJ+utZrjlzMzBrCDd3MrCHc0M3MGsIN3cysIdzQzcwaYmxDl/QZSc9K+tEar0vSxyWdlfS4pNfVP0wzMxunygz9s8D+S7x+O7C3/+cI8A+XPywzM1uvsevQI+JbkvZc4pCDwOei+Bzek5KulvSKiHimpjHaFhERfO47P+UXv1yEpn3ssoSAN+97OTfsuGraozHbkDpuLNoBPDX0+Hz/uYsauqQjFLN4du/eXcOPtkk69/Nf8oHjpwePpSkOpmblv09nnvkfPv22hekOxmyD6mjoo8p65PQtIo4BxwAWFhYaNsVrvv9bXAbg2J/+Hre99topj6Z+f/yJR3hxaXnawzDbsDpWuZwHdg093gk8XcP3tS2m0y2a3fxsa8oj2Rzz7Rk6S71pD8Nsw+po6MeBt/VXu9wMPO/8vJnKZjffbuZq1/l2a/CPllmKxkYukr4I3AJsk3Qe+AAwCxARnwROAHcAZ4EXgD/frMHadHW6TW/oM4NzNEtRlVUuh8e8HsBf1jYi27JWGnpDI5fZFotu6JawZk61bFOsZOjNfNt4hm6pa2Zl2qbII3Jxhm7pamZl2qZofOTSbnmViyXNDd0q6yw1PHKZdeRiaWtmZdqmyCFyWVzu0ev5njdLUzMr0zZF2dDnWs1825RR0uKyZ+mWpmZWpm2KTneZ+fYMatKHuAwp/8/DObqlyg3dKuss9Robt8DKtQGvdLFUNbc6rXadbq+xn+MCK5GLL4xaqtzQrbIycmmqQeTiGbolqrnVabXrdBseufTP7UVn6Jao5lan1a7I0Bscucw6crG0uaFbZZ3ucmNvKgJHLpa+5lan1S6XyMUzdEtVc6vTalc09AZHLuUqF2folig3dKuss9TwVS5eh26Ja251Wu0WG78O3ZGLpc0N3SprfobuVS6WtuZWp9Wu8TcWlZHLkiMXS1Nzq9Nq1/h16I5cLHFu6FZZp9tjrsEz9PJjgd3QLVXNrU6rVa8XLC43O0OXxJz3FbWENbc6rVblpg9NvlMU+htFex26JarZ1Wm1KZtckzN06G8U7cjFEuWGbpWUMUSTIxfoz9AduViiml2dVpumbxBdmp+d8QzdktXs6rTaDBp6g+8UhSJyWXRDt0S5oVsleUUubuiWpmZXp9Umm8ilPeM7RS1Zza5Oq002q1xmvcrF0uWGbpUMIpcc1qG7oVuiKlWnpP2SnpR0VtI9I17fLekhSY9JelzSHfUP1aYpq8jFyxYtUWOrU1ILOArcDuwDDkvat+qwvwEeiIgbgUPAJ+oeqE3XSkNveOTSbvlOUUtWlenWTcDZiDgXEYvA/cDBVccE8LL+11cBT9c3RNsKyguFjZ+hex26JaxKde4Anhp6fL7/3LAPAndKOg+cAN496htJOiLplKRTFy5c2MBwbVpW1qE3vKE7crGEValOjXguVj0+DHw2InYCdwCfl3TR946IYxGxEBEL27dvX/9obWqyilw8Q7dEVWno54FdQ493cnGkchfwAEBEfAe4AthWxwBta8jpxqLFbo+I1XMWs62vSnU+CuyVdJ2kOYqLnsdXHfMz4FYASa+haOjOVBpkZR16wxv6rDe5sHSNrc6I6AJ3Aw8CT1CsZjkt6T5JB/qHvQ94h6QfAF8E3h6e4jRKuVuRNCqBaw5vFG0pa1c5KCJOUFzsHH7u3qGvzwCvr3dotpU0fYPo0sq+osvA7HQHY7ZOza9Qq0Wn2+wNokuDhu616JYgN3SrpLPU7P1ES+XHAztysRQ1v0KtFp3ucuPXoMPqyMUsLc2vUKtFdpGLZ+iWIDd0q6Ro6M1/uwxWuThDtwQ1v0KtFp2lTFa5zDpysXQ1v0KtFuU69KabazlysXQ1v0KtFrlk6Ff4TlFLmBu6VZLPKpcyQ3fkYulpfoVaLbJZh+5VLpaw5leo1SKXyMWf5WIpc0O3SrL5LBevcrGENb9CrRadbi+LDH2wysXr0C1Bza9Qu2wRwWImkcvMjJhreV9RS5Mbuo21uJzH5halctcis9TkUaF2WVb2E83j7TI/642iLU15VKhdlsH2c7PNj1zAG0VbutzQbaxcNoguzbedoVua8qhQuyy5RS5z7RnfKWpJyqNC7bIMIpcMVrlAES15hm4pckO3sQaRSwbr0KGMXDxDt/TkUaF2WXKLXJyhW6ryqFC7LCsNPZPIpd3ynaKWJDd0G6u8QJjNDN3r0C1ReVSoXZZyhn5FVhm6Z+iWnjwq1C5LlpGLG7olyA3dxsryxiKvQ7cE5VGhdlnyW4fuyMXS5IZuYw0il2wy9CJyiYhpD8VsXfKoULssZeRSbv7QdGW0VH5ssFkqKlWopP2SnpR0VtI9axzzVklnJJ2W9IV6h2nT1On2mGvNMDOjaQ9lIrxRtKWqPe4ASS3gKPBm4DzwqKTjEXFm6Ji9wF8Dr4+I5yRds1kDtsnrLPWyuSAKKx8T3FnqwRVTHozZOlSp0puAsxFxLiIWgfuBg6uOeQdwNCKeA4iIZ+sdpk1Tp7vMXE4NvdxX1DcXWWKqVOkO4Kmhx+f7zw27Hrhe0iOSTkraP+obSToi6ZSkUxcuXNjYiG3iOt3cZuiOXCxNVap0VHC6+vJ/G9gL3AIcBv5R0tUX/aWIYxGxEBEL27dvX+9YbUo63V42uxXBUIbuz3OxxFRp6OeBXUOPdwJPjzjmaxGxFBE/Bp6kaPDWAJ2l5bxm6P319o5cLDVVqvRRYK+k6yTNAYeA46uO+SrwRgBJ2ygimHN1DtSmJ7vIxatcLFFjqzQiusDdwIPAE8ADEXFa0n2SDvQPexD4haQzwEPAX0XELzZr0DZZne5yNneJgjN0S9fYZYsAEXECOLHquXuHvg7gvf0/1jCdbo+Xzld6qzTCIHLx57lYYvL5/2jbsOzWoTtysUTlU6W2YdlFLoOLom7olhY3dBtrcTmzGXo/Q190Q7fE5FOltmGdpV42n7QIw5GLM3RLSz5VahtWLFt05GK21bmh21hFhp7PW2XOd4paovKpUtuQiMjuxqLWjJhtyZGLJSefKrUNWVoOIsjqs1zAG0VbmtzQ7ZJy2yC6NN+e8QzdkpNXldq6DfYTzbGhO0O3xORVpbZuKw09s8hl1pGLpccN3S6p/DyTnNahgyMXS1NeVWrrlnXk4hm6JSavKrV1yzZyabecoVty3NDtksrIJadNoqGImF505GKJyatKbd3KGfoVGWbo/nAuS01eVWrrlnXk4oZuiXFDt0vyjUVm6cirSm3dyguD2c3QZ31jkaXHDd0uqYwdcrsoOtfyskVLT15VauuWbeQy23LkYsnJq0pt3QYXRTNc5dLp9oiIaQ/FrLK8qtTWrcyR51p5vVXm2zNEFB8fbJaKvKrU1q3TXaY9I9rZNfRyGzrHLpaOvKrU1i233YpKZcTkC6OWkvwq1dal013ObrciWLkI7IZuKXFDt0vqLGU6Qy8jlyVHLpaO/CrV1iXbyMUzdEtQfpVq67LY7WV3lyisZOj+gC5LiRu6XVKRoef3NllZ5eKGbunIr1JtXRy5OEO3dFSqVEn7JT0p6aykey5x3FskhaSF+oZo09TJNXIZXBT1DN3SMbahS2oBR4HbgX3AYUn7Rhx3JfAe4Lt1D9Kmp9NdznOG7nXolqAqlXoTcDYizkXEInA/cHDEcR8CPgy8WOP4bMo6S71MM3RHLpaeKpW6A3hq6PH5/nMDkm4EdkXE1y/1jSQdkXRK0qkLFy6se7A2edlHLp6hW0KqNHSNeG7wiUWSZoCPAe8b940i4lhELETEwvbt26uP0qYm28ilnKH7xiJLSJVKPQ/sGnq8E3h66PGVwA3Aw5J+AtwMHPeF0WbIdpWLM3RLUJVKfRTYK+k6SXPAIeB4+WJEPB8R2yJiT0TsAU4CByLi1KaM2CaqyNDzi1zKjwt2Q7eUjG3oEdEF7gYeBJ4AHoiI05Luk3Rgswdo0xMR2UYu7dYM7Rn5oqglpV3loIg4AZxY9dy9axx7y+UPy7aCbi/oRX7bz5Xm294o2tKSZ6VaJYPt5zJc5QLlvqJu6JYON3RbU7nCI8d16FDuK+rIxdKRZ6VaJSsz9DzfJuVG0WapyLNSrZLsI5d2yxm6JcUN3dZUxg3ZztBnHblYWvKsVKuknJ3mnaF7hm7pyLNSrZKymc218oxc5tzQLTFu6LamQeSS7Qy95cjFkpJnpVolg8gl1wzdNxZZYvKsVKvEq1wcuVha3NBtTdmvcnHkYonJs1KtksEMPdcMfdYzdEtLnpVqlQxu/c85cnGGbglxQ7c1+dZ/Ry6Wljwr1SpZzL6hz9AL6C57lm5pyLNSrZJOt0drRrRbeb5NvA2dpSbPSrVKct2tqFReO3BDt1TkW602Vq4bRJfKc3eObqnIt1ptrM5SL9sVLjAUuXiliyXCDd3W1OkuZ7sGHRy5WHryrVYby5GLIxdLS77VamMVDT3jyMUzdEuMG7qtKftVLs7QLTH5VquN1VnqZZ6hO3KxtORbrTaWIxdHLpYWN3RbU/aRi2folph8q9XGyn6VizN0S0y+1WpjZX9jkSMXS4wbuq3JNxY5crG05FutNlb2kUvbkYulpVK1Stov6UlJZyXdM+L190o6I+lxSd+U9Mr6h2qTlvsql3ZrhtaMHLlYMsY2dEkt4ChwO7APOCxp36rDHgMWIuJ3gK8AH657oDZZ3eUey73IeoYO/W3oHLlYIqpU603A2Yg4FxGLwP3AweEDIuKhiHih//AksLPeYdqk5b5BdKlo6J6hWxqqVOsO4Kmhx+f7z63lLuAbo16QdETSKUmnLly4UH2UNnFlE5vLdLei0pw3iraEVKlWjXguRh4o3QksAB8Z9XpEHIuIhYhY2L59e/VR2sSVMcP8bL4ZOnijaEtLu8Ix54FdQ493Ak+vPkjSm4D3A2+IiE49w7NpKWelztAduVg6qlTro8BeSddJmgMOAceHD5B0I/Ap4EBEPFv/MG3SBhl6xqtcoLiG4IZuqRjb0COiC9wNPAg8ATwQEacl3SfpQP+wjwAvBb4s6fuSjq/x7SwRg8gl+xm6IxdLR5XIhYg4AZxY9dy9Q1+/qeZx2ZR5lUth3hdFLSF5V6utaSVDzzxycYZuCXFDt5EWlx25QPEP2qIbuiUi72q1NQ1m6LlHLrO+U9TSkXe12pq8yqXgyMVS4oZuI3mVS6FY5eKGbmnIu1ptTSsz9LzfIsUqF0culoa8q9XWtJKhZx65+MYiS4gbuo3kyKUw327R7QXdZTd12/ryrlZbU6fbY0bQnhn12Wz5KP9BW3RDtwS4odtI5W5Fkhs6eBs6S4Mbuo3UWcp7g+hSeQ3BObqlwBVrI+W+QXRpMEP3zUWWAFesjZT7BtGl8r+BZ+iWAjd0G6nTXfYMHWfolhZXrI3UWeo5Q2fls2wcuVgKXLE2kiOXgiMXS4kbuo3kyKXgi6KWElesjeRVLoVB5OIM3RLgirWROkuOXMCRi6XFDd1G6nR9YxE4crG0uGJtJEcuhZWG7hm6bX2uWBvJq1wKg1v/naFbAtzQbaTO0jJznqEz13LkYulwxdpIjlwKsy0hOXKxNLhi7SLd5R7dXjhyASR5o2hLhhu6XaTczMGrXArz7Zb3FbUkuGLtIoveIPpXzLdnvGORJcEVaxfpDBq6IxfobxTtVS6WADd0u0jZvDxDL8y3W87QLQmuWLtIuUTPGXqhuCjqDN22PlesXcSRy6/yKhdLRaWGLmm/pCclnZV0z4jX5yV9qf/6dyXtqXugNjmDGbojF6Bc5eKGblvf2IqV1AKOArcD+4DDkvatOuwu4LmI+C3gY8Df1j1Qmxxn6L9qftaRi6WhXeGYm4CzEXEOQNL9wEHgzNAxB4EP9r/+CvD3khQRUeNYAXjg0af49LfP1f1tDQggInhhsczQHblA8Q/bE8/8L7d+9GEkAaApj8nS9p5b9/JHv/ubtX/fKg19B/DU0OPzwO+vdUxEdCU9D/wG8PPhgyQdAY4A7N69e0MDvvols+x9+Us39HdtPCEQvOyKWX772iunPZwt4fBNu5ltzRAAAUHt8xTLzFW/Nrsp37dKQx81GVn9jq5yDBFxDDgGsLCwsKGquO2113Lba6/dyF8125BbXn0Nt7z6mmkPw2ysKiHpeWDX0OOdwNNrHSOpDVwF/FcdAzQzs2qqNPRHgb2SrpM0BxwCjq865jjwZ/2v3wL862bk52ZmtraxkUs/E78beBBoAZ+JiNOS7gNORcRx4J+Az0s6SzEzP7SZgzYzs4tVydCJiBPAiVXP3Tv09YvAn9Q7NDMzWw8vNDYzawg3dDOzhnBDNzNrCDd0M7OG0LRWF0q6APx0g399G6vuQs1Ejued4zlDnued4znD+s/7lRGxfdQLU2vol0PSqYhYmPY4Ji3H887xnCHP887xnKHe83bkYmbWEG7oZmYNkWpDPzbtAUxJjued4zlDnued4zlDjeedZIZuZmYXS3WGbmZmq7ihm5k1xJZu6DluTl3hnN8r6YykxyV9U9IrpzHOuo0776Hj3iIpJCW/vK3KOUt6a//3fVrSFyY9xs1Q4T2+W9JDkh7rv8/vmMY46yTpM5KelfSjNV6XpI/3/5s8Lul1G/pBEbEl/1B8VO9/AK8C5oAfAPtWHfMXwCf7Xx8CvjTtcU/gnN8IvKT/9btSP+eq590/7krgW8BJYGHa457A73ov8Bjw6/3H10x73BM672PAu/pf7wN+Mu1x13Defwi8DvjRGq/fAXyDYve3m4HvbuTnbOUZ+mBz6ohYBMrNqYcdBP65//VXgFtV7uKbprHnHBEPRcQL/YcnKXaQSl2V3zXAh4APAy9OcnCbpMo5vwM4GhHPAUTEsxMe42aoct4BvKz/9VVcvENaciLiW1x6F7eDwOeicBK4WtIr1vtztnJDH7U59Y61jomILlBuTp2qKuc87C6Kf9VTN/a8Jd0I7IqIr09yYJuoyu/6euB6SY9IOilp/8RGt3mqnPcHgTslnafYh+HdkxnaVK239keqtMHFlNS2OXVCKp+PpDuBBeANmzqiybjkeUuaAT4GvH1SA5qAKr/rNkXscgvF/4l9W9INEfHfmzy2zVTlvA8Dn42Ij0r6A4rd0G6IiN7mD29qaullW3mGnuPm1FXOGUlvAt4PHIiIzoTGtpnGnfeVwA3Aw5J+QpExHk/8wmjV9/fXImIpIn4MPEnR4FNW5bzvAh4AiIjvAFdQfIBVk1Wq/XG2ckPPcXPqsefcjx4+RdHMm5CpwpjzjojnI2JbROyJiD0U1w4ORMSp6Qy3FlXe31+luAiOpG0UEcy5iY6yflXO+2fArQCSXkPR0C9MdJSTdxx4W3+1y83A8xHxzLq/y7Sv/o65MnwH8O8UV8Xf33/uPopihuIX/WXgLPBvwKumPeYJnPO/AP8JfL//5/i0xzyJ81517MMkvsql4u9awN8BZ4AfAoemPeYJnfc+4BGKFTDfB26b9phrOOcvAs8ASxSz8buAdwLvHPpdH+3/N/nhRt/fvvXfzKwhtnLkYmZm6+CGbmbWEG7oZmYN4YZuZtYQbuhmZg3hhm5m1hBu6GZmDfH/Ls661AWf+V8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_h = 1* sgm(1000 * x_1 -300) + -1* sgm(1000 * x_1 - 500)\n",
    "plt.plot(x_1, y_h)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAWJklEQVR4nO3da5BkZ13H8e9vrtnLmJDMRGPYzUARCkMUCWMMWmoUtEKKIi8MVqhSwIpugeKl9IUXyoioL7RKqMKkjGuR4qJyERBWKkiBXKVMYBNCyEWslVu2iGZ2Ezaz3Un3ds/fF31603a6p8/s9vTp55zfp2oqfTnT8z+Znl/+efp5zqOIwMzM0jdTdAFmZjYeDnQzs5JwoJuZlYQD3cysJBzoZmYlMVfUD15eXo7V1dWifryZWZLuuuuuYxGxMui5wgJ9dXWVw4cPF/XjzcySJOlbw57zkIuZWUk40M3MSsKBbmZWEg50M7OScKCbmZXEyECXdI6kL0r6iqT7Jf3JgGMWJb1P0hFJd0pa3YlizcxsuDwdegP4mYh4AfDDwDWSruo75kbgsYh4DvBW4C/GW6aZmY0ych56dK6vezK7O5999V9z9zrgTdntDwA3S1L42rxWAt8+XudDXz7K5mY5387nLMzy2h9bZfdCYctSbExy/QYlzQJ3Ac8BbomIO/sOuRh4CCAiWpJOABcAx/pe5wBwAGD//v1nV7nZhLz7jm/yd5//xun7UoHFjFm35Xr28h6uufyiYouxs5Yr0COiDfywpPOAf5Z0eUTc13PIoLf409qZiDgIHARYW1srZ7tjpXOy0WJlaZEvvfGlRZcydusbDX7kzz/JIxuNokuxMdjWLJeI+C7wGeCavqeOAvsAJM0B5wKPjqE+s8LVGm32LMwWXcaOOH/PAjPqBLulL88sl5WsM0fSLuClwH/2HXYIeE12+3rgUx4/t7KoN1ulHV+enREX7F10oJdEnnfpRcA7s3H0GeD9EfFRSW8GDkfEIeDtwLslHaHTmd+wYxWbTVit0WbPYjk7dIAVB3pp5Jnlci/wwgGP39Rz+0ngleMtzWw61Jstztu9UHQZO2ZlaZH1kw70MvBKUbMRas2Sd+hL7tDLwoFuNkK9Ud4xdOgE+rGTDfyxV/oc6GYj1E+Vd5YLdMbQT7WDE0+cKroUO0sOdLMR6o02uxfL3aGDpy6WgQPdbAvN1ibN9ma5O3QHemk40M228ESzDVDqMfTlvVmge6ZL8hzoZluoNVsApZ/lAu7Qy8CBbraFehboZe7Qv+ecORbmZhzoJeBAN9tCrdEZcilzhy7Jq0VLwoFutoVaBTp08GrRsnCgm22h3u3QqxDo7tCT50A320K3Q99V4mmL8NRqUUubA91sC/Vm+cfQobNa9HitSau9WXQpdhYc6GZbqDWqM4YeAY/WmkWXYmfBgW62hfrphUUl79Czuejeii5tDnSzLdSaLRbmZpifLfefileLlkO536VmZ6le4v1Ee13o1aKl4EA320KtxPuJ9jrdoTvQk+ZAN9tCveT7iXbtWphlaXHOgZ44B7rZFqrSoYNXi5aBA91sC/WS7yfaa3lpkWPu0JPmQDfbQr3ZdoduyXCgm22h3mxVYpYL4CsuloAD3WwLtZLvJ9prZWmRjSdbPHmqXXQpdoZGBrqkfZI+LelBSfdL+q0Bx1wt6YSke7Kvm3amXLPJqlSH7rnoycvTerSA342IuyUtAXdJ+kREPNB33Ocj4uXjL9GsGJubUa0x9J7VovvO311wNXYmRnboEfFwRNyd3d4AHgQu3unCzIr2xKlqXGmxyx16+rY1hi5pFXghcOeAp18s6SuSPibp+UO+/4Ckw5IOr6+vb7tYs0l66lroFenQfYGu5OUOdEl7gQ8Cvx0Rj/c9fTdwSUS8APhr4MODXiMiDkbEWkSsraysnGnNZhPx1G5F1ejQz9+zgITnoicsV6BLmqcT5v8QER/qfz4iHo+Ik9nt24F5SctjrdRswqqyn2jX/OwM5+9e8M5FCcszy0XA24EHI+ItQ475vuw4JF2Zve7xcRZqNmlV2a2ol/cWTVue1uPHgV8CvirpnuyxPwT2A0TErcD1wOsltYAngBsiInagXrOJqcpuRb28WjRtI9+pEfHvgEYcczNw87iKMpsGlezQ9y7yjWO1osuwM+SVomZDdDv0PVXr0Dca+H+w0+RANxuiKvuJ9lreu0ijtclG9h8zS4sD3WyI7iyXPRW5lgt4cVHqHOhmQ9QbbWYEi3PV+TNxoKetOu9Us22qNVvsWZgjm5FbCQ70tDnQzYaoN9rsrtAMF+i5QJcDPUkOdLMh6qfalZrhAnDurnnmZ+XVoolyoJsNUW+0Ktehz8yIZe9clCwHutkQtWarUqtEu7xaNF0OdLMh6s12Za602Mt7i6bLgW42RK1RzQ7dQy7pcqCbDdHZfq6CHfrSIsdrTdqbXv6fGge62RC1RqtSq0S7VpYWaW8Gj9WbRZdi2+RANxsgIirdoYPnoqfIgW42QLO9SWszKtuhgwM9RQ50swG6+4lWskPPVot6cVF6HOhmA5y+0mIFZ7m4Q0+XA91sgNPXQq/YSlHoXC5498KsAz1BDnSzAaq4W1EvrxZNkwPdbIAq7lbUy4uL0uRANxvgdIdewVku4OX/qXKgmw1Q9Q7dQy5pcqCbDVDF/UR7rSwt8t36KRqtdtGl2DY40M0GqPI8dHhq6uLxk17+n5KRgS5pn6RPS3pQ0v2SfmvAMZL0NklHJN0r6YqdKddsMp4acqloh+6t6JKU593aAn43Iu6WtATcJekTEfFAzzEvAy7Nvn4U+Jvsn2ZJqjdbnDM/w+xMdTaI7tXt0L1aNC0jAz0iHgYezm5vSHoQuBjoDfTrgHdFRAB3SDpP0kXZ95olp6q7FXV1A/1vP/t1/vW+/ym4mvL56eddyLU/eNHYX3db71hJq8ALgTv7nroYeKjn/tHssf8X6JIOAAcA9u/fv71KzSao3miza76a4+cAFy4tcuXq+Rx9rM7Rx+pFl1M6q8t7duR1cwe6pL3AB4HfjojH+58e8C1Puzp+RBwEDgKsra356vk2tWrNFnsrOsMFYG52hve/7sVFl2HblGuWi6R5OmH+DxHxoQGHHAX29dx/JvCdsy/PrBj1ZruS13GxtOWZ5SLg7cCDEfGWIYcdAl6dzXa5Cjjh8XNLWa3Rqux1XCxded6xPw78EvBVSfdkj/0hsB8gIm4FbgeuBY4AdeCXx1+q2eTUm22Ws6l7ZqnIM8vl3xk8Rt57TAC/Pq6izIpWa1ZzP1FLm1eKmg1Qb1RzP1FLmwPdbAB36JYiB7pZn/Zm8OSpTXfolhwHulmfeoX3E7W0OdDN+lR5P1FLmwPdrE/V9xO1dDnQzfpUfbciS5cD3axP1fcTtXQ50M361E+5Q7c0OdDN+nS3n3OHbqlxoJv16W4Q7Q7dUuNAN+tTb3QD3R26pcWBbtan5lkuligHulmferPF7IxYnPOfh6XF71izPrXsSoudvV3M0uFAN+tTb3q3IkuTA92sT837iVqiHOhmfereT9QS5UA361NrerciS5MD3axP3bsVWaIc6GZ9vJ+opcqBbtan5lkuligHulmfesOzXCxNDnSzHhHhDt2SNTLQJd0m6RFJ9w15/mpJJyTdk33dNP4yzSaj0dpkM7yfqKUpTxvyDuBm4F1bHPP5iHj5WCoyK1B3+zl36JaikR16RHwOeHQCtZgVrtbwtdAtXeMaQ3+xpK9I+pik5w87SNIBSYclHV5fXx/TjzYbn9MduuehW4LGEeh3A5dExAuAvwY+POzAiDgYEWsRsbaysjKGH202Xt6tyFJ21oEeEY9HxMns9u3AvKTls67MrADeT9RSdtaBLun7lF04WtKV2WseP9vXNStCt0PfNe8O3dIzsg2R9B7gamBZ0lHgj4F5gIi4FbgeeL2kFvAEcENExI5VbLaD6lmgu0O3FI1810bEq0Y8fzOdaY1myat1h1w8hm4J8kpRsx7dDn23O3RLkAPdrEe3Q/cYuqXIgW7Wo95ssWt+ltkZbxBt6XGgm/WoNdvs8XVcLFEOdLMe9UaL3b6OiyXKgW7Ww/uJWsoc6GY9vJ+opcyBbtaj7g7dEuZAN+tRb7R9LXRLlgPdrEet2fJuRZYsB7pZj3rTHbqly4Fu1qPWcIdu6XKgm2Va7U0arU136JYsB7pZpn6qcx0Xz3KxVDnQzTLerchS50A3y3g/UUudA90s0+3QfS0XS5UD3SzT7dC9W5GlyoFulvFuRZY6B7pZxvuJWuoc6GYZd+iWOge6WcYduqXOgW6WOd2he5aLJcqBbpapN9vMz4qFOf9ZWJpGvnMl3SbpEUn3DXlekt4m6YikeyVdMf4yzXZeZ3MLd+eWrjytyDuAa7Z4/mXApdnXAeBvzr4ss8mrNVoeP7ekjWxHIuJzkla3OOQ64F0REcAdks6TdFFEPDymGm1KRAR/9JH7+NbxetGljF0E/Of/bHDe7vmiSzE7Y+P4/8uLgYd67h/NHntaoEs6QKeLZ//+/WP40TZJj9VP8fd3fJt95+9iee9i0eWMjbJ/XnLBbl7yAxcWWovZ2RhHoGvAYzHowIg4CBwEWFtbG3iMTa9jJxsA/N41z+PlP/T9BVdjZv3G8XH+UWBfz/1nAt8Zw+valFnf6AT6Som6c7MyGUegHwJenc12uQo44fHzcjod6EsOdLNpNHLIRdJ7gKuBZUlHgT8G5gEi4lbgduBa4AhQB355p4q1YnUDfdmBbjaV8sxyedWI5wP49bFVZFNr/WSDxbkZlnytE7Op5CVxltv6RoOVpUWkQZ+Dm1nRHOiWWzfQzWw6OdAtt/WNhme4mE0xB7rltn7SHbrZNHOgWy6n2ps8Vm860M2mmAPdcnm01iTCc9DNppkD3XLxKlGz6edAt1y8qMhs+jnQLRd36GbTz4Fuuayf9HVczKadA91yWd9osHTOHOfMe0cfs2nlQLdcvErUbPo50C0XrxI1m34OdMvlmFeJmk09B7rl4iEXs+nnQLeRnmi22Wi0HOhmU86BbiN1N4f2GLrZdHOg20iPeJWoWRIc6DaSV4mapcGBbiN1V4le6A7dbKo50G2k9Y0GEpy/Z6HoUsxsCw50G2l9o8EFexaYm/XbxWya+S/URjp2ssGyx8/Npp4D3UbyoiKzNOQKdEnXSPqapCOSfn/A86+VtC7pnuzrV8ZfqhXFgW6WhrlRB0iaBW4BfhY4CnxJ0qGIeKDv0PdFxBt2oEYrUESw7uu4mCUhT4d+JXAkIr4eEU3gvcB1O1uWTYvHn2zRbG16DrpZAvIE+sXAQz33j2aP9ft5SfdK+oCkfYNeSNIBSYclHV5fXz+Dcm3STi8qcoduNvXyBLoGPBZ99/8FWI2IHwI+Cbxz0AtFxMGIWIuItZWVle1VaoXwKlGzdOQJ9KNAb8f9TOA7vQdExPGIaGR3/w540XjKs6J5L1GzdOQJ9C8Bl0p6lqQF4AbgUO8Bki7qufsK4MHxlWhF8pCLWTpGznKJiJakNwAfB2aB2yLifklvBg5HxCHgNyW9AmgBjwKv3cGabYKOnWwwPyvO3TVfdClmNsLIQAeIiNuB2/seu6nn9h8AfzDe0mwadPcSlQZ9lGJm08QrRW1LXlRklg4Hum3JgW6WDge6bcmrRM3S4UC3odqbwXFfadEsGQ50G+rRWpPN8JRFs1Q40G0orxI1S4sD3YbyKlGztDjQbSivEjVLiwPdhjqWdej+UNQsDQ50G2p9o8GehVn2LOZaUGxmBXOg21BeVGSWFge6DeVAN0uLA92GWveiIrOkONBtKHfoZmlxoNtAjVabE0+c8qIis4Q40G2gYyebgOegm6XEgW4DeVGRWXoc6DbQMQe6WXIc6DaQr+Nilh4Hug3UHXK5YI8D3SwVDnQbaH2jwTN2z7Mw57eIWSr812oDrW94UZFZahzoNpD3EjVLjwPdBvIqUbP05Ap0SddI+pqkI5J+f8Dzi5Lelz1/p6TVcRdqkxMRnUD3kItZUkYGuqRZ4BbgZcBlwKskXdZ32I3AYxHxHOCtwF+Mu1CbnFqzzROn2u7QzRKTZ+eCK4EjEfF1AEnvBa4DHug55jrgTdntDwA3S1JExBhrBeCz/7XOn330gdEH2rZ1f1mt9ibgnYrMUpMn0C8GHuq5fxT40WHHRERL0gngAuBY70GSDgAHAPbv339GBe9dnOPS7917Rt9rowkBcMX+Z/ATz10uuBoz2448ga4Bj/V33nmOISIOAgcB1tbWzqh7f9Elz+BFl7zoTL7VzKzU8nwoehTY13P/mcB3hh0jaQ44F3h0HAWamVk+eQL9S8Clkp4laQG4ATjUd8wh4DXZ7euBT+3E+LmZmQ03csglGxN/A/BxYBa4LSLul/Rm4HBEHALeDrxb0hE6nfkNO1m0mZk9XZ4xdCLiduD2vsdu6rn9JPDK8ZZmZmbb4ZWiZmYl4UA3MysJB7qZWUk40M3MSkJFzS6UtA586wy/fZm+VagVUcXzruI5QzXPu4rnDNs/70siYmXQE4UF+tmQdDgi1oquY9KqeN5VPGeo5nlX8ZxhvOftIRczs5JwoJuZlUSqgX6w6AIKUsXzruI5QzXPu4rnDGM87yTH0M3M7OlS7dDNzKyPA93MrCSmOtCrujl1jvP+HUkPSLpX0r9JuqSIOsdp1Dn3HHe9pJBUiultec5b0i9kv+/7Jf3jpGsctxzv7/2SPi3py9l7/Noi6hwnSbdJekTSfUOel6S3Zf9O7pV0xRn9oIiYyi86l+r9b+DZwALwFeCyvmN+Dbg1u30D8L6i657Qef80sDu7/frUzzvPOWfHLQGfA+4A1oque0K/60uBLwPPyO5fWHTdEzjng8Drs9uXAd8suu4xnPdPAlcA9w15/lrgY3R2f7sKuPNMfs40d+inN6eOiCbQ3Zy613XAO7PbHwBeImnQdngpGXneEfHpiKhnd++gs4tUyvL8rgH+FPhL4MlJFreD8pz3rwK3RMRjABHxyIRrHLc85xzA92S3z+XpO6QlJyI+x9a7uF0HvCs67gDOk3TRdn/ONAf6oM2pLx52TES0gO7m1CnLc969bqTzX/aUjTxnSS8E9kXERydZ2A7L87t+LvBcSV+QdIekayZW3c7Ic85vAn5R0lE6+zD8xmRKK9R2/+4HyrXBRUHGtjl1YnKfk6RfBNaAn9rRinbelucsaQZ4K/DaSRU0IXl+13N0hl2upvN/Yp+XdHlEfHeHa9spec75VcA7IuKvJL2Yzm5ol0fE5s6XV5ixZNk0d+hV3Zw6z3kj6aXAG4FXRERjQrXtlFHnvARcDnxG0jfpjDEeKsEHo3nf4x+JiFMR8Q3ga3QCPlV5zvlG4P0AEfEfwDl0LmBVZrn+7keZ5kCv6ubUI887G374WzphnvqYKow454g4ERHLEbEaEat0Pjd4RUQcLqbcscnzHv8wnQ/BkbRMZwjm6xOtcrzynPO3gZcASPoBOoG+PtEqJ+8Q8OpststVwImIeHjbr1L0p78jPhm+FvgvOp+KvzF77M10/pih84v+J+AI8EXg2UXXPKHz/iTwv8A92dehomve6XPuO/YzlGCWS87ftYC3AA8AXwVuKLrmCZzzZcAX6MyAuQf4uaJrHsM5vwd4GDhFpxu/EXgd8Lqe3/Mt2b+Tr57p+9tL/83MSmKah1zMzGwbHOhmZiXhQDczKwkHuplZSTjQzcxKwoFuZlYSDnQzs5L4P7frtpOU0faIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_h = 1* sgm(1000 * x_1 -300) + -1* sgm(1000 * x_1 - 500) + \\\n",
    "3 * sgm(1000 * x_1 -500) -1* sgm(1000 * x_1 - 700)\n",
    "plt.plot(x_1, y_h)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAWtklEQVR4nO3df4zcdZ3H8eerP2gL/QV2ldpuWRG4iEQENgjxgpx6l8oZ+odwKYk/MHiNnJwaTYxogsrd/eHlTu4UIlcPIniewqHRSkoMnnCgOdAFSvlRNT38wRSkC0w7LTvb7rbv+2O+ux2mM53vtvPrM/t6JBO+M/PZ2feX2X3tp5/5fD8fRQRmZpa+Od0uwMzMWsOBbmbWJxzoZmZ9woFuZtYnHOhmZn1iXre+8YoVK2JoaKhb397MLEmPPPLIixExUO+5rgX60NAQIyMj3fr2ZmZJkvT7Rs95yMXMrE840M3M+oQD3cysTzjQzcz6hAPdzKxP5A50SXMlPSbp7jrPLZB0h6Ttkh6WNNTKIs3MrLmZ9NA/AWxr8NxVQDEiTgNuAL58rIWZmdnM5JqHLmk18JfAPwCfqtNkHfDF7Pgu4EZJCq/Na5asO0eepVAsg3+NW2546CQuOqPutUHHJO+FRf8CfAZY0uD5VcCzABExKWk38BrgxepGkjYAGwDWrFlzNPWaWQfsHpvgM3dtBUDqcjF96KPveGN3Al3Se4GdEfGIpIsbNavz2GF/1iNiI7ARYHh42H/2zXrU7vIEAP90+dlcdt7qLldjeeUZQ387cKmk3wHfBd4p6T9q2hSAQQBJ84BlwMstrNPMOqg0Xgn0pQu7tjqIHYWmgR4R10bE6ogYAtYDP42I99c02wR8KDu+LGvjHrhZokpZD33povldrsRm4qj//Eq6HhiJiE3ALcC3JG2n0jNf36L6zKwLDvXQHegpmVGgR8T9wP3Z8XVVj48Dl7eyMDPrnlJ5EoClizzkkhJfKWpmh5nuoXvIJSkOdDM7TKk8gQSLj3MPPSUOdDM7TGl8kiUL5jFnjiehp8SBbmaHKZUnPNySIAe6mR2mND7pGS4JcqCb2WFK4xOe4ZIgB7qZHaZUnnAPPUEOdDM7zJ7xSZY40JPjQDezw1Q+FPWQS2oc6Gb2KgcOBnv2+UPRFDnQzexV9o5PXfbvQE+NA93MXsVL56bLgW5mr7LbS+cmy4FuZq/ipXPT5UA3s1fx0rnpcqCb2au4h56upoEuaaGkX0h6XNJTkr5Up82VkkYlbcluH2lPuWbWbns8yyVZef5NtQ94Z0TslTQf+JmkeyLioZp2d0TENa0v0cw6aWot9CULPOSSmqbvWLbZ897s7vzs5g2gzfpUaXyCxV4LPUm5xtAlzZW0BdgJ3BsRD9dp9j5JWyXdJWmwwetskDQiaWR0dPQYyjazdimVfZVoqnIFekQciIi3AquB8yWdVdPkR8BQRLwF+AlwW4PX2RgRwxExPDAwcCx1m1mblMYnWOKLipI0o1kuEbELuB9YW/P4SxGxL7v7DeC8llRnZh3n3YrSlWeWy4Ck5dnxIuDdwK9q2qysunspsK2VRZpZ53i3onTl+XfVSuA2SXOp/AG4MyLulnQ9MBIRm4CPS7oUmAReBq5sV8Fm1l6l8gRvWrmk22XYUcgzy2UrcE6dx6+rOr4WuLa1pZlZN5TGvVtRqnylqJlNO3gw2Ltv0mPoiXKgm9m0PfsmifDSualyoJvZtJKXzk2aA93MpnlhrrQ50M1s2qGFuTzkkiIHuplNmx5ycQ89SQ50M5tWynroyzyGniQHuplNm+qhey2XNDnQzWza1Ieii70WepIc6GY2rVSeZPGCecyb62hIkd81M5tWuezfvfNUOdDNbJqXzk2bA93MpnlhrrQ50M1sWqk86YuKEuZAN7Np7qGnzYFuZtM8hp62PFvQLZT0C0mPS3pK0pfqtFkg6Q5J2yU9LGmoHcWaWftMr4XuWS7JytND3we8MyLOBt4KrJV0QU2bq4BiRJwG3AB8ubVlmlm7vbJ/koPhpXNT1jTQo2Jvdnd+douaZuuA27Lju4B3SVLLqjSztptax8WX/acr1xi6pLmStgA7gXsj4uGaJquAZwEiYhLYDbymzutskDQiaWR0dPTYKjezlvJKi+nLFegRcSAi3gqsBs6XdFZNk3q98dpePBGxMSKGI2J4YGBg5tWaWdt4t6L0zWiWS0TsAu4H1tY8VQAGASTNA5YBL7egPjPrkKkhF/fQ05VnlsuApOXZ8SLg3cCvapptAj6UHV8G/DQiDuuhm1nvOtRD9xh6qvK8cyuB2yTNpfIH4M6IuFvS9cBIRGwCbgG+JWk7lZ75+rZVbGZt4f1E09c00CNiK3BOncevqzoeBy5vbWlm1kmlsme5pM5XipoZUOmhn3DcXK+FnjC/c2YG+LL/fuBANzPAC3P1Awe6mQFeOrcfONDNDHAPvR840M0MyALdY+hJc6CbGQB7xic9ZTFxDnQzIyIqs1w85JI0B7qZ8cr+A9la6O6hp8yBbmZeOrdPONDN7NA6Lv5QNGkOdDObXsfFPfS0OdDNzEvn9gkHupl56dw+4UA3M28/1yfy7Fg0KOk+SdskPSXpE3XaXCxpt6Qt2e26eq9lZr1pavs5X1iUtjzv3iTw6Yh4VNIS4BFJ90bE0zXtHoyI97a+RDNrt1J5guOPm8t8r4WetKbvXkQ8HxGPZsd7gG3AqnYXZmad44W5+sOM/hxLGqKyHd3DdZ6+UNLjku6R9OYGX79B0oikkdHR0RkXa2bt4aVz+0PuQJe0GPge8MmIKNU8/ShwSkScDXwN+EG914iIjRExHBHDAwMDR1uzmbVYaXyCJe6hJy9XoEuaTyXMvx0R3699PiJKEbE3O94MzJe0oqWVmlnbVIZc3ENPXZ5ZLgJuAbZFxFcatDk5a4ek87PXfamVhZpZ+1SGXNxDT12eP8lvBz4APCFpS/bY54A1ABFxM3AZcLWkSaAMrI+IaEO9ZtYGe/yhaF9oGugR8TNATdrcCNzYqqLMrHMigtK4PxTtB550ajbLje0/wIGD4R56H3Cgm81yXjq3fzjQzWY5L53bPxzoZrPcoR66x9BT50A3m+W8/Vz/cKCbzXIeQ+8fDnSzWe7QGLqHXFLnQDeb5aaGXLyWS/oc6GazXGl8goXz53DcPMdB6vwOms1ypfKkPxDtEw50s1muND7hD0T7hAPdbJbz0rn9w4FuNst56dz+4UA3m+W8n2j/cKCbzXKl8oQv++8TDnSzWSwi2DPuWS79Is8WdIOS7pO0TdJTkj5Rp40kfVXSdklbJZ3bnnLNrJXKEweYPBgeQ+8Tef6dNQl8OiIelbQEeETSvRHxdFWb9wCnZ7e3AV/P/mtmPcxL5/aXPFvQPQ88nx3vkbQNWAVUB/o64PZsH9GHJC2XtDL7WrOkPfaHIv/+4G852Ifb5O7dlwW6x9D7wozeRUlDwDnAwzVPrQKerbpfyB57VaBL2gBsAFizZs3MKjXrkjtHCvz4qT9y6sAJ3S6lpZRtFXz24HLOXr28y9VYK+QOdEmLge8Bn4yIUu3Tdb7ksO5MRGwENgIMDw/3X3fH+lKhOMabX7+UH17zp90uxeyIcs1ykTSfSph/OyK+X6dJARisur8aeO7YyzPrvh3FMqtPPL7bZZg1lWeWi4BbgG0R8ZUGzTYBH8xmu1wA7Pb4ufWDiGDHrjKrTlzU7VLMmsoz5PJ24APAE5K2ZI99DlgDEBE3A5uBS4DtwBjw4daXatZ5o3v3sW/yIKsd6JaAPLNcfkb9MfLqNgF8rFVFmfWKQrEM4EC3JPhKUbMjOBToHkO33udANzuCQnEMgFXL3UO33udANzuCQrHMSSccxwkLfOGN9T4HutkRFIpl984tGQ50syPYURzzB6KWDAe6WQMRQaFYdqBbMhzoZg28uHd/NgfdM1wsDQ50swamZri4h26pcKCbNeA56JYaB7pZA1OB7nVcLBUOdLMGCsUxlh8/n8Weg26JcKCbNbBjl2e4WFoc6GYNFIplVi/3+Lmlw4FuVkdlDrovKrK0ONDN6njplf2MT3gddEuLA92sDk9ZtBTl2YLuVkk7JT3Z4PmLJe2WtCW7Xdf6Ms06a/qiopPcQ7d05JmP9U3gRuD2I7R5MCLe25KKzHrAjqk56F5p0RLStIceEQ8AL3egFrOeUSiWWbpwHksWzu92KWa5tWoM/UJJj0u6R9KbGzWStEHSiKSR0dHRFn1rs9YrFMcYPMnj55aWVgT6o8ApEXE28DXgB40aRsTGiBiOiOGBgYEWfGuz9vCyuZaiYw70iChFxN7seDMwX9KKY67MrEsOrYPuHrql5ZgDXdLJkpQdn5+95kvH+rpm3fLyK/spTxxwD92S03SWi6TvABcDKyQVgC8A8wEi4mbgMuBqSZNAGVgfEdG2is3arOAZLpaopoEeEVc0ef5GKtMazfqCLyqyVPlKUbMaUxcVeR10S40D3azGjl2VOejLFnkOuqXFgW5WwzNcLFUOdLMaXjbXUuVAN6viOeiWMge6WZXi2ARj+w/4A1FLkgPdrMr0srkOdEuQA92syqE56A50S48D3azKoR66x9AtPQ50syqFYpklnoNuiXKgm1XZ4RkuljAHulkVr4NuKXOgm2Uqc9DHvMqiJcuBbpbZNTbBK/u9Drqly4FulvGyuZa6poEu6VZJOyU92eB5SfqqpO2Stko6t/VlmrWfLyqy1OXpoX8TWHuE598DnJ7dNgBfP/ayzDpvqoc+6B66JSrPjkUPSBo6QpN1wO3ZtnMPSVouaWVEPN+iGq2HPDO6lxdK+7pdRssEh3ZLfPQPRZYsmMfSRU1/Lcx6Uit+clcBz1bdL2SPHRbokjZQ6cWzZs2aFnxr66TxiQOs/dcH2T95sNultM3Zg8vJ9jw3S04rAr3eT3/dTaIjYiOwEWB4eNgbSSemUCyzf/IgH3/naVz4xhXdLqdlqvP71IETuleI2TFqRaAXgMGq+6uB51rwutZjpj40vOiMAYaHTupyNWZWqxXTFjcBH8xmu1wA7Pb4eX/ytD6z3ta0hy7pO8DFwApJBeALwHyAiLgZ2AxcAmwHxoAPt6tY665Cscz8ueK1SxZ0uxQzqyPPLJcrmjwfwMdaVpH1rKnL4ufM8YeGZr3IV4pabt5r06y3OdAtt0Kx7IWrzHqYA91yGZ84wIt79/myeLMe5kC3XHbsyma4nORAN+tVDnTLxVMWzXqfA91y8UqEZr3PgW65HJqDvrDbpZhZAw50y6VQLPP65YuY6znoZj3LgW65eK9Ns97nQLdcdhTLHj8363EOdGtqfOIAO/fs8wwXsx7nQLemnpuag+4eullPc6BbU56DbpYGB7o1dSjQ3UM362UOdGuqUBxj3hzxuqWeg27Wyxzo1tSOXWVWLl/oOehmPS5XoEtaK+nXkrZL+myd56+UNCppS3b7SOtLtW4pFMusXu7xc7Nel2cLurnATcCfU9kQ+peSNkXE0zVN74iIa9pQo3VZoTjGRacPdLsMM2siTw/9fGB7RDwTEfuB7wLr2luW9Yp9kwd4oeQ56GYpyBPoq4Bnq+4XssdqvU/SVkl3SRqs90KSNkgakTQyOjp6FOVapz23axyAVZ7hYtbz8gR6vU/Coub+j4ChiHgL8BPgtnovFBEbI2I4IoYHBvxP+BRMLZs76EA363l5Ar0AVPe4VwPPVTeIiJciYl929xvAea0pz7ptag66e+hmvS9PoP8SOF3SGyQdB6wHNlU3kLSy6u6lwLbWlWjdtKNYZu4ccbLnoJv1vKazXCJiUtI1wI+BucCtEfGUpOuBkYjYBHxc0qXAJPAycGUba7YOKhTHWLlsIfPm+pIFs17XNNABImIzsLnmseuqjq8Frm1tadYLCl421ywZ7nbZEVUC3VMWzVLgQLeG9k0e4IU94+6hmyXCgW4NPb9rnAgvm2uWCge6NTQ9ZdF7iZolwYFuDe3YVbmoyEMuZmlwoFtDhWwO+splnoNulgIHujVUKJY5eannoJulwr+p1lChOObhFrOEONCtIc9BN0uLA93q2j95kD+WPAfdLCUOdKvrj7src9C9yqJZOhzoVtfUOujuoZulw4FudU1dVDToMXSzZDjQra5CcYw5gpM9B90sGQ50q6tQLLNy2SLmew66WTL822p1FYplfyBqlphcgS5praRfS9ou6bN1nl8g6Y7s+YclDbW6UOssX1Rklp6mgS5pLnAT8B7gTOAKSWfWNLsKKEbEacANwJdbXah1zsSBbA66V1k0S0qeLejOB7ZHxDMAkr4LrAOermqzDvhidnwXcKMkRUS0sFYA/uc3o/z93U83b2gzNvVmHTgYHPQ66GbJyRPoq4Bnq+4XgLc1apNtKr0beA3wYnUjSRuADQBr1qw5qoIXL5jH6a9bfFRfa80JAXDO4HIu/pOBLldjZjORJ9BV57HanneeNkTERmAjwPDw8FH13s875UTOO+W8o/lSM7O+ludD0QIwWHV/NfBcozaS5gHLgJdbUaCZmeWTJ9B/CZwu6Q2SjgPWA5tq2mwCPpQdXwb8tB3j52Zm1ljTIZdsTPwa4MfAXODWiHhK0vXASERsAm4BviVpO5We+fp2Fm1mZofLM4ZORGwGNtc8dl3V8ThweWtLMzOzmfCVomZmfcKBbmbWJxzoZmZ9woFuZtYn1K3ZhZJGgd8f5ZevoOYq1FliNp73bDxnmJ3nPRvPGWZ+3qdERN3LuLsW6MdC0khEDHe7jk6bjec9G88ZZud5z8Zzhtaet4dczMz6hAPdzKxPpBroG7tdQJfMxvOejecMs/O8Z+M5QwvPO8kxdDMzO1yqPXQzM6vhQDcz6xM9HeizdXPqHOf9KUlPS9oq6b8lndKNOlup2TlXtbtMUkjqi+ltec5b0l9l7/dTkv6z0zW2Wo6f7zWS7pP0WPYzfkk36mwlSbdK2inpyQbPS9JXs/8nWyWde1TfKCJ68kZlqd7/A04FjgMeB86safM3wM3Z8Xrgjm7X3aHz/jPg+Oz46tTPO885Z+2WAA8ADwHD3a67Q+/16cBjwInZ/dd2u+4OnPNG4Ors+Ezgd92uuwXnfRFwLvBkg+cvAe6hsvvbBcDDR/N9ermHPr05dUTsB6Y2p662DrgtO74LeJeketvhpaTpeUfEfRExlt19iMouUinL814D/B3wj8B4J4trozzn/dfATRFRBIiInR2usdXynHMAS7PjZRy+Q1pyIuIBjryL2zrg9qh4CFguaeVMv08vB3q9zalXNWoTEZPA1ObUKctz3tWuovKXPWVNz1nSOcBgRNzdycLaLM97fQZwhqSfS3pI0tqOVdceec75i8D7JRWo7MPwt50pratm+ntfV64NLrqkZZtTJyb3OUl6PzAMvKOtFbXfEc9Z0hzgBuDKThXUIXne63lUhl0upvIvsQclnRURu9pcW7vkOecrgG9GxD9LupDKbmhnRcTB9pfXNS3Jsl7uoc/WzanznDeS3g18Hrg0IvZ1qLZ2aXbOS4CzgPsl/Y7KGOOmPvhgNO/P+A8jYiIifgv8mkrApyrPOV8F3AkQEf8LLKSygFU/y/V730wvB/ps3Zy66Xlnww//RiXMUx9ThSbnHBG7I2JFRAxFxBCVzw0ujYiR7pTbMnl+xn9A5UNwJK2gMgTzTEerbK085/wH4F0Akt5EJdBHO1pl520CPpjNdrkA2B0Rz8/4Vbr96W+TT4YvAX5D5VPxz2ePXU/llxkqb/R/AduBXwCndrvmDp33T4AXgC3ZbVO3a273Ode0vZ8+mOWS870W8BXgaeAJYH23a+7AOZ8J/JzKDJgtwF90u+YWnPN3gOeBCSq98auAjwIfrXqfb8r+nzxxtD/fvvTfzKxP9PKQi5mZzYAD3cysTzjQzcz6hAPdzKxPONDNzPqEA93MrE840M3M+sT/A8JfvvaiUvIAAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_h = 1* sgm(1000 * x_1 -300) + -1* sgm(1000 * x_1 - 500) + \\\n",
    "2 * sgm(1000 * x_1 -500) - 1* sgm(1000 * x_1 - 700) \\\n",
    "+ 3 * sgm(1000 * x_1 -700)\n",
    "plt.plot(x_1, y_h)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/levan/anaconda3/envs/edu/lib/python3.7/site-packages/ipykernel_launcher.py:2: RuntimeWarning: overflow encountered in exp\n",
      "  \n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAYi0lEQVR4nO3dbZBc5Xnm8f8lzUgaodEL1jjI88LEASfGlAlmitjllE2wdwuzLvQhkJWrHJsUWZWdsHYqqUqZpIo4bPaDN4mdOFBhlYUy9mZtWJJ1FArKsddmjVOL7AELGaE4pRDbtMBmgJZGYnpGM9KdD31m1Iy61Uczp/v06b5+VVP0y5nu+9Az1zx6zn2eo4jAzMyKb03eBZiZWTYc6GZmXcKBbmbWJRzoZmZdwoFuZtYl+vJ64+3bt8f4+Hheb29mVkhPPPHESxExVO+53AJ9fHycycnJvN7ezKyQJP2w0XOecjEz6xIOdDOzLuFANzPrEg50M7Mu4UA3M+sSqQNd0lpJ35X0UJ3n1ku6X9JhSfskjWdZpJmZNXc+I/SPA4caPHcLUI6IS4DPAJ9abWFmZnZ+UvWhSxoB/gPwX4HfrrPJTuCTye0HgTslKbw2r1lhPTD5HKVyBVr4azxy4UZ+ZWK0Za/fa9KeWPRnwO8Cgw2eHwaeA4iIBUnHgNcBL9VuJGk3sBtgbGxsJfWaWRscm5nndx88AIDUmvdY/Dvxnp97Pa/btL41b9Jjmga6pPcDL0bEE5KuabRZncfO+rMeEXuAPQATExMevZt1qGOVeQD+5KYruPGqkZa8xz8c/DG7v/AEpXLFgZ6RNHPo7wRukPQD4EvAtZL+57JtSsAogKQ+YAvwSoZ1mlkbTc9WA33zhtatDjKybSMAR45WWvYevaZpoEfEbRExEhHjwC7g6xHxwWWb7QU+nNy+MdnGI3CzgppORuibB/pb9h7D2wYAKJVnWvYevWbFf34l3QFMRsRe4B7gC5IOUx2Z78qoPjPLwZkReusCfctAP5s39FUPvFomzivQI+JR4NHk9u01j88CN2VZmJnlZ7qyAMDmgdYuyDqybaMDPUM+U9TMzrI0Qm/hlAvAyLYBT7lkyIFuZmeZrswjwaZ17Rmh+5BbNhzoZnaW6dkFBtf3sWZNi5rQE8PbBpg5eYqjM/MtfZ9e4UA3s7NMV+ZbPt0C1SkXwPPoGXGgm9lZpmcXWtrhsmjErYuZcqCb2VmmZ+db3uECZ04u8gg9Gw50MzvLdGW+LSP0LQP9DG7o8wg9Iw50MzvL8dkFBtsQ6OBe9Cw50M3sLNWDoq2fcoHFXnQHehYc6Gb2GqdOB8fn2nNQFGB46wBHjroXPQsOdDN7jROzi6f9t2vKZYATcwtLS/bayjnQzew12rF0bi13umTHgW5mr3GsDUvn1nIvenYc6Gb2Gu1YOrfWqEfomXGgm9lrtGvp3EWbB/oYXO910bPgQDez12j3CF0Sw15GNxNNA13SBknflvSUpIOS/rDONjdLmpK0P/n69daUa2atdrzNXS7gXvSspPk31RxwbUSckNQPfEvSIxHx+LLt7o+IW7Mv0czaaXEt9MH17ZlygWqny75nXyEikFq7ZG83S3OR6IiIE8nd/uTLZwCYdanp2Xk2tWEt9Foj2wY47l70VUs1hy5praT9wIvAVyNiX53NflnSAUkPShpt8Dq7JU1KmpyamlpF2WbWKtOV9p0lusjromcjVaBHxKmI+HlgBLha0uXLNvl7YDwi3gp8DbivwevsiYiJiJgYGhpaTd1m1iLTs/MMtumkokU+uSgb59XlEhFHgUeB65Y9/nJEzCV3/wq4KpPqzKzt2nW1olo+uSgbabpchiRtTW4PAO8F/mnZNjtq7t4AHMqySDNrn3ZdrajWloF+Lli31iP0VUrz76odwH2S1lL9A/BARDwk6Q5gMiL2Ah+TdAOwALwC3Nyqgs2staYr87x5x2Bb31MSI9s2cuSoA301mgZ6RBwArqzz+O01t28Dbsu2NDPLw/Rse65WtJx70VfPZ4qa2ZLTp4MTcwttn0OHxUD3HPpqONDNbMnxuQUi2rd0bq2RbRs5Pute9NVwoJvZkuk2L51by50uq+dAN7Ml7V6Yq5Z70VfPgW5mS84szNX+KZfhZIR+xIG+Yg50M1uyNOWSwwh928Z+NroXfVUc6Ga2ZDoZoW/JYQ692ovuTpfVcKCb2ZLFEXq713JZNLJto0foq+BAN7MliwdFN7VxLfRaHqGvjgPdzJZMVxbYtL6PvrX5RMPItgGm3Yu+Yg50M1tSPe0/n9E5nGlddKfLyjjQzWxJHkvn1hremrQuepGuFXGgm9mSvBbmWuSzRVfHgW5mS6YrC7mcVLTowgvWMdDvXvSVcqCb2ZK8R+juRV8dB7qZLcl7Dh28LvpqpLkE3QZJ35b0lKSDkv6wzjbrJd0v6bCkfZLGW1GsmbXO0lroOXa5gE8uWo00n9wccG1EnJDUD3xL0iMR8XjNNrcA5Yi4RNIu4FPAf2xBvWbWIq+eXOB05LN0bq3hbQMcq8zzw5dfZeO6fP+4tMrAurUtOXkrzSXoAjiR3O1PvmLZZjuBTya3HwTulKTke82sABbXccnrtP9FF19Y7UV/9x8/mmsdrfSRd/8Mn3jfz2X+uqk+ueQC0U8AlwB3RcS+ZZsMA88BRMSCpGPA64CXlr3ObmA3wNjY2OoqN7NM5bnSYq1r3/x6/vjGtzK7cDrXOlrpLW/Y3JLXTRXoEXEK+HlJW4H/I+nyiHi6ZhPV+7Y6r7MH2AMwMTHh0btZB8nzakW11vet5aaJ0VxrKKrz6nKJiKPAo8B1y54qAaMAkvqALcArGdRnZm2yOOWS9wjdVi5Nl8tQMjJH0gDwXuCflm22F/hwcvtG4OuePzcrljMj9O48ENkL0nxyO4D7knn0NcADEfGQpDuAyYjYC9wDfEHSYaoj810tq9jMWiLP64laNtJ0uRwArqzz+O01t2eBm7ItzczaabrSGV0utnI+U9TMgOoI/YJ1a3NbC91Wz5+cmQGdcdq/rY4D3cyA/BfmstVzoJsZkP/SubZ6DnQzAzxC7wYOdDMDkkD3HHqhOdDNDIDjswtuWSw4B7qZERHVLhdPuRSaA93MePXkqWQtdI/Qi8yBbmYds3SurY4D3czOrOPig6KF5kA3s6V1XDxCLzYHupl56dwu4UA3My+d2yUc6GbWMZefs9VJc8WiUUnfkHRI0kFJH6+zzTWSjknan3zdXu+1zKwzLV5+zicWFVuaT28B+J2IeFLSIPCEpK9GxDPLtnssIt6ffYlm1mrTlXk2rltLv9dCL7Smn15EvBARTya3jwOHgOFWF2Zm7eOFubrDef05ljRO9XJ0++o8/Q5JT0l6RNJbGnz/bkmTkianpqbOu1gzaw0vndsdUge6pE3A3wC/FRHTy55+Erg4Iq4A/gL4cr3XiIg9ETERERNDQ0MrrdnMMjY9O8+gR+iFlyrQJfVTDfO/joi/Xf58RExHxInk9sNAv6TtmVZqZi1TnXLxCL3o0nS5CLgHOBQRn26wzUXJdki6Onndl7Ms1Mxapzrl4hF60aX5k/xO4FeB70nanzz2e8AYQETcDdwIfFTSAlABdkVEtKBeM2uB4z4o2hWaBnpEfAtQk23uBO7Mqigza5+IYHrWB0W7gZtOzXrczMlTnDodHqF3AQe6WY/z0rndw4Fu1uO8dG73cKCb9bgzI3TPoRedA92sx/nyc93DgW7W4zyH3j0c6GY97swcuqdcis6BbtbjFqdcvJZL8TnQzXrc9Ow8G/rXsK7PcVB0/gTNetx0ZcEHRLuEA92sx03PzvuAaJdwoJv1OC+d2z0c6GY9zkvndg8HulmP8/VEu4cD3azHTVfmfdp/l3Cgm/WwiOD4rLtcukWaS9CNSvqGpEOSDkr6eJ1tJOmzkg5LOiDpba0p18yyVJk/xcLp8Bx6l0jz76wF4Hci4klJg8ATkr4aEc/UbPM+4NLk6xeAv0z+a2YdzEvndpc0l6B7AXghuX1c0iFgGKgN9J3A55PriD4uaaukHcn3mhXad39U5n889q+c7sLL5J6YSwLdc+hd4bw+RUnjwJXAvmVPDQPP1dwvJY+9JtAl7QZ2A4yNjZ1fpWY5eWCyxFcO/pg3Dl2QdymZUnKp4CtGt3LFyNacq7EspA50SZuAvwF+KyKmlz9d51vOGs5ExB5gD8DExET3DXesK5XKM7zlDZv5u1t/Me9SzM4pVZeLpH6qYf7XEfG3dTYpAaM190eA51dfnln+jpQrjGzbmHcZZk2l6XIRcA9wKCI+3WCzvcCHkm6XtwPHPH9u3SAiOHK0wvC2gbxLMWsqzZTLO4FfBb4naX/y2O8BYwARcTfwMHA9cBiYAX4t+1LN2m/qxBxzC6cZcaBbAaTpcvkW9efIa7cJ4DezKsqsU5TKFQAHuhWCzxQ1O4czge45dOt8DnSzcyiVZwAY3uoRunU+B7rZOZTKFS68YB0XrPeJN9b5HOhm51AqVzw6t8JwoJudw5HyjA+IWmE40M0aiAhK5YoD3QrDgW7WwEsnTiY96O5wsWJwoJs1sNjh4hG6FYUD3awB96Bb0TjQzRpYDHSv42JF4UA3a6BUnmHrxn42uQfdCsKBbtbAkaPucLFicaCbNVAqVxjZ6vlzKw4Hulkd1R50n1RkxeJAN6vj5VdPMjvvddCtWBzoZnW4ZdGKKM0l6O6V9KKkpxs8f42kY5L2J1+3Z1+mWXstnVR0oUfoVhxp+rE+B9wJfP4c2zwWEe/PpCKzDnBksQfdKy1agTQdoUfEN4FX2lCLWccolSts3tDH4Ib+vEsxSy2rOfR3SHpK0iOS3tJoI0m7JU1Kmpyamsrorc2yVyrPMHqh58+tWLII9CeBiyPiCuAvgC832jAi9kTERERMDA0NZfDWZq3hZXOtiFYd6BExHREnktsPA/2Stq+6MrOcnFkH3SN0K5ZVB7qkiyQpuX118povr/Z1zfLyyqsnqcyf8gjdCqdpl4ukLwLXANsllYA/APoBIuJu4Ebgo5IWgAqwKyKiZRWbtVjJHS5WUE0DPSI+0OT5O6m2NZp1BZ9UZEXlM0XNllk8qcjroFvRONDNljlytNqDvmXAPehWLA50s2Xc4WJF5UA3W8bL5lpROdDNargH3YrMgW5Wozwzz8zJUz4gaoXkQDersbRsrgPdCsiBblbjTA+6A92Kx4FuVuPMCN1z6FY8DnSzGqVyhUH3oFtBOdDNahxxh4sVmAPdrIbXQbcic6CbJao96DNeZdEKy4Fuljg6M8+rJ70OuhWXA90s4WVzreiaBrqkeyW9KOnpBs9L0mclHZZ0QNLbsi/TrPV8UpEVXZoR+ueA687x/PuAS5Ov3cBfrr4ss/ZbHKGPeoRuBZXmikXflDR+jk12Ap9PLjv3uKStknZExAsZ1Wgd5NmpE/xkei7vMjITnLla4pM/KjO4vo/NA01/Lcw6UhY/ucPAczX3S8ljZwW6pN1UR/GMjY1l8NbWTrPzp7juzx/j5MLpvEtpmStGt5Jc89yscLII9Ho//XUvEh0Re4A9ABMTE76QdMGUyhVOLpzmY9dewjt+Znve5WSmNr/fOHRBfoWYrVIWgV4CRmvujwDPZ/C61mEWDxq+601DTIxfmHM1ZrZcFm2Le4EPJd0ubweOef68O7mtz6yzNR2hS/oicA2wXVIJ+AOgHyAi7gYeBq4HDgMzwK+1qljLV6lcoX+teP3g+rxLMbM60nS5fKDJ8wH8ZmYVWcdaPC1+zRofNDTrRD5T1FLztTbNOpsD3VIrlSteuMqsgznQLZXZ+VO8dGLOp8WbdTAHuqVy5GjS4XKhA92sUznQLRW3LJp1Pge6peKVCM06nwPdUjnTg74h71LMrAEHuqVSKld4w9YB1roH3axjOdAtFV9r06zzOdAtlSPliufPzTqcA92amp0/xYvH59zhYtbhHOjW1POLPegeoZt1NAe6NeUedLNicKBbU2cC3SN0s07mQLemSuUZ+taIn9rsHnSzTuZAt6aOHK2wY+sG96CbdbhUgS7pOknfl3RY0ifqPH+zpClJ+5OvX8++VMtLqVxhZKvnz806XZpL0K0F7gL+HdULQn9H0t6IeGbZpvdHxK0tqNFyVirP8K5Lh/Iuw8yaSDNCvxo4HBHPRsRJ4EvAztaWZZ1ibuEUP5l2D7pZEaQJ9GHguZr7peSx5X5Z0gFJD0oarfdCknZLmpQ0OTU1tYJyrd2ePzoLwLA7XMw6XppAr3ckLJbd/3tgPCLeCnwNuK/eC0XEnoiYiIiJoSH/E74IFpfNHXWgm3W8NIFeAmpH3CPA87UbRMTLETGX3P0r4KpsyrO8Lfage4Ru1vnSBPp3gEsl/bSkdcAuYG/tBpJ21Ny9ATiUXYmWpyPlCmvXiIvcg27W8Zp2uUTEgqRbga8Aa4F7I+KgpDuAyYjYC3xM0g3AAvAKcHMLa7Y2KpVn2LFlA31rfcqCWadrGugAEfEw8PCyx26vuX0bcFu2pVknKHnZXLPC8LDLzqka6G5ZNCsCB7o1NLdwip8cn/UI3awgHOjW0AtHZ4nwsrlmReFAt4aWWhZ9LVGzQnCgW0NHjlZPKvKUi1kxONCtoVLSg75ji3vQzYrAgW4NlcoVLtrsHnSzovBvqjVUKs94usWsQBzo1pB70M2KxYFudZ1cOM2Pp92DblYkDnSr68fHqj3oXmXRrDgc6FbX4jroHqGbFYcD3epaPKlo1HPoZoXhQLe6SuUZ1ggucg+6WWE40K2uUrnCji0D9LsH3aww/NtqdZXKFR8QNSuYVIEu6TpJ35d0WNIn6jy/XtL9yfP7JI1nXai1l08qMiuepoEuaS1wF/A+4DLgA5IuW7bZLUA5Ii4BPgN8KutCrX3mTyU96F5l0axQ0lyC7mrgcEQ8CyDpS8BO4JmabXYCn0xuPwjcKUkRERnWCsD/++cp/uihZ5pvaOdt8cM6dTo47XXQzQonTaAPA8/V3C8Bv9Bom+Si0seA1wEv1W4kaTewG2BsbGxFBW9a38elP7VpRd9rzQkBcOXoVq752aGcqzGz85Em0FXnseUj7zTbEBF7gD0AExMTKxq9X3XxNq66+KqVfKuZWVdLc1C0BIzW3B8Bnm+0jaQ+YAvwShYFmplZOmkC/TvApZJ+WtI6YBewd9k2e4EPJ7dvBL7eivlzMzNrrOmUSzInfivwFWAtcG9EHJR0BzAZEXuBe4AvSDpMdWS+q5VFm5nZ2dLMoRMRDwMPL3vs9prbs8BN2ZZmZmbnw2eKmpl1CQe6mVmXcKCbmXUJB7qZWZdQXt2FkqaAH67w27ez7CzUHtGL+92L+wy9ud+9uM9w/vt9cUTUPY07t0BfDUmTETGRdx3t1ov73Yv7DL253724z5DtfnvKxcysSzjQzcy6RFEDfU/eBeSkF/e7F/cZenO/e3GfIcP9LuQcupmZna2oI3QzM1vGgW5m1iU6OtB79eLUKfb7tyU9I+mApP8r6eI86sxSs32u2e5GSSGpK9rb0uy3pF9JPu+Dkv5Xu2vMWoqf7zFJ35D03eRn/Po86sySpHslvSjp6QbPS9Jnk/8nByS9bUVvFBEd+UV1qd5/Ad4IrAOeAi5bts1vAHcnt3cB9+ddd5v2+5eAjcntjxZ9v9Psc7LdIPBN4HFgIu+62/RZXwp8F9iW3H993nW3YZ/3AB9Nbl8G/CDvujPY73cBbwOebvD89cAjVK/+9nZg30rep5NH6EsXp46Ik8Dixalr7QTuS24/CLxHUr3L4RVJ0/2OiG9ExExy93GqV5EqsjSfNcB/Af4bMNvO4loozX7/J+CuiCgDRMSLba4xa2n2OYDNye0tnH2FtMKJiG9y7qu47QQ+H1WPA1sl7Tjf9+nkQK93cerhRttExAKweHHqIkuz37VuofqXvcia7rOkK4HRiHionYW1WJrP+k3AmyT9o6THJV3XtupaI80+fxL4oKQS1esw/Of2lJar8/29ryvVBS5yktnFqQsm9T5J+iAwAby7pRW13jn3WdIa4DPAze0qqE3SfNZ9VKddrqH6L7HHJF0eEUdbXFurpNnnDwCfi4g/lfQOqldDuzwiTre+vNxkkmWdPELv1YtTp9lvJL0X+H3ghoiYa1NtrdJsnweBy4FHJf2A6hzj3i44MJr2Z/zvImI+Iv4V+D7VgC+qNPt8C/AAQET8f2AD1QWsulmq3/tmOjnQe/Xi1E33O5l++O9Uw7zoc6rQZJ8j4lhEbI+I8YgYp3rc4IaImMyn3Myk+Rn/MtWD4EjaTnUK5tm2VpmtNPv8I+A9AJLeTDXQp9paZfvtBT6UdLu8HTgWES+c96vkffS3yZHh64F/pnpU/PeTx+6g+ssM1Q/6fwOHgW8Db8y75jbt99eAnwD7k6+9edfc6n1etu2jdEGXS8rPWsCngWeA7wG78q65Dft8GfCPVDtg9gP/Pu+aM9jnLwIvAPNUR+O3AB8BPlLzOd+V/D/53kp/vn3qv5lZl+jkKRczMzsPDnQzsy7hQDcz6xIOdDOzLuFANzPrEg50M7Mu4UA3M+sS/wYf0HtzYsAzggAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_h = 1* sgm(1000 * x_1 -300) + -1* sgm(1000 * x_1 - 500) + \\\n",
    "2 * sgm(1000 * x_1 -500) - 1* sgm(1000 * x_1 - 700) \\\n",
    "+ 3 * sgm(1000 * x_1 -700) - 1* sgm(1000 * x_1 - 800)\n",
    "plt.plot(x_1, y_h)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "More approximation need wider layer\n",
    "<img src=\"images/add1/ua_2.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Multi-dimensional examples\n",
    "<img src=\"images/add1/ua_3.jpg\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/tl_7.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transfer learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/tl_8.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Self-supervised learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Labeling data is expensive:\n",
    "- Labeling images for classification\n",
    "- Tend to be biased and has an errors\n",
    "- ImageNet is one of the\n",
    "- Labeling images for detection and segmentation is at least twice expensive and biased"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Self supervised learning is ML learning when labels are simple generated by the modification of original data, without human-in-the loop:\n",
    "- Generative models:\n",
    " - Auto-encoders when labels are the original images\n",
    "- Discriminative models:\n",
    " - When labels are the meta information about original data modification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Self-supervised representation learning is a self-supervised learning, when model is trained for pretext task and for downstream task"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Representation learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Deep models for feature extraction.\n",
    "<br>\n",
    "Train deep model on modified data (pretext task)\n",
    "<br>\n",
    "Use it without last layers as feature extractors and train other model on top of it (downstream task)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Predicting neighbouring context"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/nc_1.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Word embeddings with dimensionality reduction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/we_1.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Word embeddings (Word2Vec)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/wv_2.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/wv_3.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/wv_4.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Noise contrastive estimation, instead of softmax on the huge amount of negative samples, we create binary classifier, if pair of vectors are from the same class (yes, no)\n",
    "<br>\n",
    "In our case the same class means then, pair of vectors are from the same image, with different augmentation (or augmentation and source images), or part of the same image\n",
    "<br>\n",
    "All other images or patches can be considered as images from the different classes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Contrastive learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Imagine two normalized vectors $v_1 = \\frac{z_1}{||z_1||}$ and $v_2 = \\frac{z_2}{||z_2||}$\n",
    "<br>\n",
    "\n",
    "Lets compute scalar product $v_1 \\cdot v_2 = ||v_1|| \\cdot ||v_2|| \\cdot \\cos{(v_1, v_2)}$\n",
    "<br>\n",
    "\n",
    "Because they are normalized, we can imagine that $||v_1|| \\cdot ||v_2|| \\approx 1$ and therefore only $cos(v_1, v_2)$ matters\n",
    "<br>\n",
    "\n",
    "Recall that $\\cos{90^{\\circ}} = 0$ and $\\cos{0^{\\circ}} = 1$\n",
    "<br>\n",
    "\n",
    "So higher cosine means that angle is sharper and therefore (recall that vectors are normalized) vectors are closer to each other:\n",
    "\n",
    "$$\n",
    "v_1 \\cdot v_2 \\text{ is higher } \\implies v_1 \\text{is closer to } v_2\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/sim_1.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Geometric and algorithmic perspective"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/nce_det_1.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/nce_det_2.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/nce_det_3.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Mutual-information perspective"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Mutual information maximization between anchor / source and positive examples:\n",
    "$$\n",
    "I(x; c) = \\sum_{x, c}{p(x, c)\\log{\\frac{p(x| c)}{p(x)}}}\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "The mutual information:\n",
    "$$\n",
    "I(x; c) = \\sum_{x, c}{\\log{\\frac{p(x, c)}{p(x) \\cdot p(c)}}}\n",
    "$$\n",
    "<br>\n",
    "\n",
    "The joint probability:\n",
    "$$\n",
    "p(x, c) = p(x|c)p(c)\n",
    "$$\n",
    "<br>\n",
    "So we have the above formula"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\"We do not predict future observations $x_{t+k}$ directly with a generative model $p_k(x_{t+k}|c_t)$\"\n",
    "<br>\n",
    "\n",
    "\"Instead we model a density ratio which preserves the mutual information between $x_{t+k}$ and $c_t$\"\n",
    "<br>\n",
    "\n",
    "Instead of predicting future according the past, increase mutual information for future and past in comparison with past and some, random \"negative\" future\n",
    "\n",
    "$$\n",
    "f_k(x_{t+k}, c_t) \\propto \\frac{p(x_{t+k}|c_t)}{p(x_{t+k})}\n",
    "$$\n",
    "<br>\n",
    "\n",
    "This means that we need to make $f_k(x_{t+k}, c_t)$ \"larger\" in order to increase mutual information and make other $f_k(x_{s}, c_t)$ smaller for negative samples $x_s$ and $c_t$ in order to decrease the mutual information between them\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "The $f_k(x_{t+k}, c_t) = v_{t+k} \\cdot v_t$ can be vector multiplication, as we saw above, which makes vectors closer with higher mutual information and far with less mutual information\n",
    "<br>\n",
    "\n",
    "We can also have more complicated models such as a linear combination:\n",
    "$$\n",
    "f_k(x_{t+k}, c_t) = \\exp(z^T_{t+k} \\cdot W_k \\cdot c_t)\n",
    "$$\n",
    "<br>\n",
    "\n",
    "Or even a neural network instead of $z^T_{t+k} \\cdot W_k \\cdot c_t$ for combination\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Cross-entropy perspective"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "\"Cross-entropy is a measure of the difference between two probability distributions for a given random variable or set of events\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "The cross entropy of the distribution \n",
    "$q$\n",
    " relative to a distribution \n",
    "$p$\n",
    " over a given set is defined as follows:\n",
    "\n",
    "$$\n",
    "H(p,q)=-{E} _{p}[\\log q]\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Or for the discrete cases:\n",
    "$$\n",
    "H(p,q)=- \\sum_{x \\in \\mathcal{X}}p(x) \\cdot \\log q(x)\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "On our case $p(x) = 1$ when samples are from the same class\n",
    "<br>\n",
    "So we had to minimize:\n",
    "$$\n",
    "H(p,q)=- \\log q(x)\n",
    "$$\n",
    "<br>\n",
    "for the positive $x$\n",
    "<br>\n",
    "Or in maximize the:\n",
    "$$\n",
    "q(x)\n",
    "$$\n",
    "Note that here $\\mathcal{X} = \\{(x_p, x) \\in \\{x_p\\} \\times X\\}$ and $X = \\{a_k(x) : x \\in X_o, a \\in A\\}$ where $A$ is the set of the augmentations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Or in maximize the:\n",
    "$$\n",
    "q(x) = \\frac{f(x_p)}{\\sum_{x \\in \\mathcal{X}}{f(x)}}\n",
    "$$\n",
    "where $x_p$ is our positive example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "But $f(x_p) = f(x_a, x_p)$ and $f(x) = f(x_p, x)$ all the other examples:\n",
    "$$\n",
    "H(p, q) = \\frac{f(x_a, x_p)}{\\sum_{x \\in X}{f(x_a, x)}}\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Instead of all the negatives $\\sum_{x \\in X}$ we can take \"big enough\" amount of negative samples $X_0 \\subset X$ and $x_a, x_p \\in X_0$:\n",
    "$$\n",
    "H(p, q) = \\frac{f(x_a, x_p)}{\\sum_{x \\in X_0}{f(x_a, x)}}\n",
    "$$\n",
    "<br>\n",
    "$$\n",
    "f(x_a, x) = m(x_a)^T \\cdot m(x) = z_p^T \\cdot z\n",
    "$$\n",
    "or\n",
    "$$\n",
    "f(x_a, x) = e^{m(x_a)^T \\cdot m(x)} = e^{z_p^T \\cdot z}\n",
    "$$\n",
    "<br>\n",
    "\n",
    "or\n",
    "$\\{x_a, x_p\\}$ vs $\\{x_a\\} \\times X = \\{(x_a,x)|x \\in X\\}$\n",
    "<br>\n",
    "\n",
    "Or from some \"big enough\" subset of $X_0 \\subset X$\n",
    "<br>\n",
    "\n",
    "$\\{x_a, x_p\\}$ vs $\\{x_a\\} \\times X_0 = \\{(x_a,x)|x \\in X_0\\}$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Instance discrimination"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/id_1.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/id_2.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Momentum contrast (MoCo)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/moco1_1.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/moco1_2.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Initialize model $f_q$ and make a copy - $f_k$, and fixed size queue $L$\n",
    "<br>\n",
    "than take image $x$ and two augmentation functions $a_q$ and $a_k$ (dynamically for each iteration)\n",
    "<br>\n",
    "Augment the source image:\n",
    "$$\n",
    "x_q = a_q(x)\n",
    "$$\n",
    "<br>\n",
    "and\n",
    "$$\n",
    "x_k = a_k(x)\n",
    "$$\n",
    "Calculate representation vectors\n",
    "$$\n",
    "q = f_q(x_q)\n",
    "$$\n",
    "<br>\n",
    "$$\n",
    "k = f_k(x_k)\n",
    "$$\n",
    "<br>\n",
    "Calculate the cost:\n",
    "$$\n",
    "c = \\frac{f(q, k)}{\\sum_{l \\in L}f(q, l)}\n",
    "$$\n",
    "<br>\n",
    "Backpropagate only $f_q$ (so the weights $W_q$ are updated)\n",
    "<br>\n",
    "Update $f_k$ models weighs $W_k$ with exponential moving average:\n",
    "$$\n",
    "W_k = \\alpha \\cdot W_k + (1 - \\alpha) \\cdot W_q\n",
    "$$\n",
    "<br>\n",
    "and save representation $k$ in the queue $L = (l_1, l_2, \\dots, l_n)$ (remove oldest and thus less influenced by the $f_q$ models weighs vectors)\n",
    "$$\n",
    "k \\to l_1, \\\\\n",
    "l_1 \\to l_2 \\\\\n",
    "\\cdots \\\\\n",
    "l_{n-1} \\to l_n \\\\\n",
    "$$\n",
    "and $l_n$ goes out\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "So in queue\n",
    "$$\n",
    "L = (l_1, l_2, \\dots, l_n)\n",
    "$$\n",
    "are representation vectors generated by the $f_k$\n",
    "<br>\n",
    "\n",
    "Lower the index $i$ is, $l_i$ is updated with $f_k = f(x, W_k)$ with closer weights with $f_q = f_q(x, W_q)$ which was updated by the backpropagation\n",
    "<br>\n",
    "\n",
    "The less influenced vectors from the last updated $f_q = f_q(x, W_q)$ is $l_n$ which is removed and queue is updated after each iteration\n",
    "<br>\n",
    "\n",
    "In this manner algorithm maintains the \"better\" representation vectors for negative sampling\n",
    "<br>\n",
    "\n",
    "In my opinion this works plus because $k = f_k(x_k)$ in positive sampling is also calculated with $f_k = f_k(x, W_k)$ which weights was updated in previous iteration with exponentially moving (closest) average of $f_q = f_q(x, W_q)$ model\n",
    "<br>\n",
    "\n",
    "Closest to backpropagated weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Now translate everything in batches:\n",
    "<br>\n",
    "Just consider each call as batch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/moco1_3.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/moco1_4.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/moco1_5.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/add1/moco1_6.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Self-supervised learning for language models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/ssl_1/mask_lm_1.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/ssl_1/ce_1.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/ssl_1/ce_5.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/ssl_1/mask_lm_2.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/ssl_1/nw_lm_1.jpg\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/ssl_1/nw_lm_2.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Next word probability according to previous words:\n",
    "$$\n",
    "P(w_k | w_1, w_2, \\dots, w_{k-1})\n",
    "$$\n",
    "<br>\n",
    "\n",
    "and for masked language model:\n",
    "$$\n",
    "P(w_j | w_1, w_2, \\dots, w_{j-1}, w_{j+1}, \\dots w_{k})\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Self-supervised learning on graphs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/ssl_1/graph_ms_1.gif\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/ssl_1/graph_ms_2.gif\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/ssl_1/graph_ms_3.png\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src=\"images/ssl_1/graph_ms_4.gif\" height=\"800\" width=\"800\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Conclusion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Deep learning model's representation part is part which reduces dimension and converts input row / word / text/ image / node / graph into the vector which preserves representational information about input:\n",
    "\n",
    "$$\n",
    "h: X \\to \\mathbb{R}^d\n",
    "$$\n",
    "<br>\n",
    "\n",
    "This vectors then might be used for other training objectives\n",
    "<br>\n",
    "\n",
    "Train model with auto generated targets pretext task such that, $h$ will accumulate enough information for downstream tasks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Most effective pretext tasks:\n",
    "- Contrastive learning for computer vision\n",
    "- Unmask + next word prediction for NLP\n",
    "- Contrastive learning with MI maximization for graphs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "We preserve weights for $h$ model and train other model on top of it\n",
    "<br>\n",
    "\n",
    "Or fine tune entire model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Worth to mention:\n",
    "- Contrastive clustering\n",
    "- Mutual information maximization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Questions\n",
    "\n",
    "<img src=\"images/add1/questions_1.jpg\" height=\"600\" width=\"600\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Thank you"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}