lec_11_v1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "96b15482",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17a94b55",
   "metadata": {},
   "source": [
    "#### Glorot/Xavier init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "35890063",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ -1.6343,  -9.7700, -21.7768,  11.3288,  18.2016],\n",
       "        [-11.8279,   8.9871,  -4.2368,  10.7221,   3.8209],\n",
       "        [ 10.1499,   9.0273,  -5.5877,   7.5599,   4.3356],\n",
       "        [  0.9312,  14.5441,   7.3266, -10.6763,   8.4439],\n",
       "        [  4.9628,   2.8174,  -3.2859, -20.9180, -15.2168]])"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xr = torch.randn(200, 100)\n",
    "x = xr.clone()\n",
    "for i in range(1): \n",
    "    x = x @ torch.randn(100,100)\n",
    "x[0:5,0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "1315962c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.0753), tensor(9.9558))"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.mean(), x.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "00a3840c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0.]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xr = torch.randn(200, 100)\n",
    "x = xr.clone()\n",
    "for i in range(50): \n",
    "    x = x @ torch.randn(100,100)*0.01\n",
    "x[0:5,0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "4b017565",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.randn??"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11865fa3",
   "metadata": {},
   "source": [
    "#### sigmoid function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "6234eb87",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "aeb2c8c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def act_sig(x):\n",
    "    return (1/(1 + np.exp(-x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "bab197dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.linspace(-5, 5, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "9431110c",
   "metadata": {},
   "outputs": [],
   "source": [
    "y = act_sig(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "bbedefc9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x16271c650>]"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAf/klEQVR4nO3deXRV5b3/8fc380wgCVMIoyAgMwGcWm2tXlBUrLYqWCva8tPW/qy1rdpr9d6qrcNtb71Vy496sVrrWC2lSkFtna4Dk4wBAjEMCVMmMs85z++PRG6KQQ5wkn2Gz2utrGSfvZN8jmZ91sOzh8ecc4iISOiL8jqAiIgEhgpdRCRMqNBFRMKECl1EJEyo0EVEwkSMV784MzPTDR061KtfLyISktauXVvmnMvqap9nhT506FDWrFnj1a8XEQlJZrb7aPs05SIiEiZU6CIiYUKFLiISJlToIiJh4piFbmaLzazEzDYfZb+Z2X+ZWYGZbTSzKYGPKSIix+LPCP33wMzP2T8LGNnxsQD47cnHEhGR43XMQnfOvQtUfM4hlwJPu3YfAelmNiBQAUVExD+BuA49GyjqtF3c8dr+Iw80swW0j+IZPHhwAH61iEhwcM7R1OqjurGF2sZWaptaD3+ub26jrrmV+qb2z1OH9OYLI7u8N+ikBKLQrYvXunzIunNuEbAIIDc3Vw9iF5Gg5PM5DtU3U1bbTHldE+W1zRyqb6airpnK+hYO1bd/rmpo/6huaKGmsZXmNp9fP/+mc0cEbaEXAzmdtgcB+wLwc0VEAq6xpY29lQ3sq2xgf1Uj+ysbOVDdSEl1IwdrGimpbqK8rpk2X9djzrSEGNKT4uidFEtaYiyDeifSK7H969SEGFITYklLiCE5LoaUhBhS4mNIjo8hOS6apPgYEmOjiY7qahx88gJR6EuBm83seWAGUOWc+8x0i4hIT6lqaGFnWR2FpbXsKq9nT3kduyvqKapooKy26TPHZyTH0S8tgX5p8YwdkEZWajxZKfFkpMSTkRJHZko8fZLjSE+MJSY6eK/2Pmahm9lzwLlAppkVA/cAsQDOuYXAMuBCoACoB+Z3V1gRkc6qG1vIP1DDtv3VbD9Yy46SGgpKaimrbT58TJTBgF6JDMlI4rzRfRnUO5FBfRIZ0CuRgb0S6dcrnviYaA/fReAcs9Cdc1cfY78DvhuwRCIiXahqaGFjcSUbi6vYvLeKTXurKD7UcHh/akIMI/um8OXRfTmlbwrDMlMYlpnM4D5JxMUE76g6kDx72qKIyNE459hdXs+qnRWs3lXBuqJKCkpqD+8fkpHExJx05s4YzJj+aYwekEr/tATMumduOlSo0EUkKOyrbOB/Csr4oKCMDz4pp6Smfa67d1IsUwb3Zs6kgUzK6c34Qb3olRjrcdrgpEIXEU+0tPlYvauCt/NLeWtbCTs6RuCZKXGcMSKT04f3YfrQPpzSNyXiR97+UqGLSI9pbGnj7fxSXs87wN+3lVDV0EJstDFjWAZXTsvh7JGZnNovVQV+glToItKtWtp8/M+OMpZu2MfreQeoa26jV2Is543pywVj+3P2yExS4lVFgaD/iiLSLfIP1PDSmiKWrN9LWW0zaQkxzJ4wkIsnDmTG8D7EBvH13KFKhS4iAdPY0sZrG/fzh492s76okpgo47wxfbliag7njMqKmMsHvaJCF5GTdqCqkac+3MXzq/ZwqL6F4VnJ3HXRGC6bnE1GSrzX8SKGCl1ETti2A9UsereQv27YR5vPccHY/lx7xhDOGJGhE5seUKGLyHHbvLeK3/xjByvyDpIUF828GUO44exh5PRJ8jpaRFOhi4jf8g/U8PCKfN7cepDUhBhuOW8k888aSnpSnNfRBBW6iPih+FA9v3pjO39et5eUuBh+cP4orjtrKGkJumMzmKjQReSo6ptbefytT1j0XiEA3/7CcG46ZwS9kzUiD0YqdBH5DOccSzfs4xfLtnGgupE5kwbyo5mjyU5P9DqafA4Vuoj8k93lddy1ZDPv7ShjwqBePDZvMlOH9PE6lvhBhS4iALS2+Vj0XiGPvLmDuOgo7p0zjnnTBxPVTculSeCp0EWEgpIabntxAxuKq5g1rj//dslp9EtL8DqWHCcVukgE8/kci9/fyUMr8kmOi+bxeVO4cPwAr2PJCVKhi0SokppGbntxA+/tKOMrY/rxi6+OJytVt+mHMhW6SAR6d3spP3hxPTWNrfz8svFcPT1Ht+qHARW6SATx+RyP/H0Hj/x9B6P6pfDHb53Oqf1TvY4lAaJCF4kQVQ0t3PrCev6xrYTLpwzivjnjSIyL9jqWBJAKXSQC7DhYw7eeXsPeQw3ce+lpXHP6EE2xhCEVukiYe29HKd955mPiY6N5fsHp5A7VTULhSoUuEsae+Wg39yzNY2TfFP77umm6dT/MqdBFwpBzjgeX57PwnU8499QsfnP1ZFL1ZMSwp0IXCTOtbT7ueGUTf1pbzNwZg/nZJacRowWZI4IKXSSMNDS3cfOzH/P3bSXcct5Ivv+VkTr5GUFU6CJhoq6plet/v5pVuyq4d844vnH6EK8jSQ9ToYuEgerGFuY/uZr1RZX8+spJXDop2+tI4gEVukiIq6pv4donV5G3t4pHr57MLD1cK2L5dabEzGaaWb6ZFZjZHV3s72VmfzWzDWaWZ2bzAx9VRI5U3djCtYtXsnVfNb+9ZqrKPMIds9DNLBp4DJgFjAWuNrOxRxz2XWCLc24icC7wSzPTooMi3aiuqZX5T64mb181j8+bwvlj+3kdSTzmzwh9OlDgnCt0zjUDzwOXHnGMA1Kt/XR6ClABtAY0qYgc1tjSxreeWsO6PYf4r6sn8xWVueBfoWcDRZ22izte6+xRYAywD9gE3OKc8x35g8xsgZmtMbM1paWlJxhZJLK1tPm46Zm1fLSznF99fZIWpJDD/Cn0ri5idUds/wuwHhgITAIeNbO0z3yTc4ucc7nOudysrKzjjCoiPp/j9j9t5K38Uu6fM545k3U1i/wvfwq9GMjptD2I9pF4Z/OBV1y7AmAnMDowEUXkUw8s38Yr6/Zy2/mjmDtjsNdxJMj4U+irgZFmNqzjROdVwNIjjtkDnAdgZv2AU4HCQAYViXS/e7eQRe8W8s0zhnDzl0/xOo4EoWNeh+6cazWzm4EVQDSw2DmXZ2Y3duxfCNwL/N7MNtE+RXO7c66sG3OLRJRlm/Zz/7KtXDR+APdcfJpu55cu+XVjkXNuGbDsiNcWdvp6H3BBYKOJCMDHew5x6wvrmTqkN7/8+kSiolTm0jU9gk0kiBVV1PPtp9bQLy2BRd+YSkKsloyTo1OhiwSpmsYWrv/9alp9jifnTyMjJd7rSBLk9CwXkSDk8zlufWE9hWV1/OH66YzISvE6koQAjdBFgtCv3tjOm1tL+OlFYzjzlEyv40iIUKGLBJlXN+7j0bcKuDI3h2+eOdTrOBJCVOgiQST/QA0/emkjU4f05mdzdHmiHB8VukiQqGls4aZn1pIcH8Nv500hPkZXtMjx0UlRkSDgnONHL21kd0U9z35rBn3TEryOJCFII3SRIPC79wpZnneAO2aOZsbwDK/jSIhSoYt4bO3uCh5cns+scf351heGeR1HQpgKXcRDh+qa+d6z68hOT+TBKyboJKicFM2hi3jEOceP/rSB0tomXr7pTNISYr2OJCFOI3QRjyx+fxdvbi3hzlljmDAo3es4EgZU6CIe2Ly3igf+tpWvjOnH/LOGeh1HwoQKXaSHNTS38X+fX0ef5Dge1ry5BJDm0EV62H2vbaGwtI4/fmsGvZPjvI4jYUQjdJEe9HreAf64cg8Lvjics/TQLQkwFbpIDympaeT2lzdy2sA0brtglNdxJAyp0EV6gHOOO1/eRH1zG49cNUnPaZFuoUIX6QEvrini79tKuH3maE7pm+p1HAlTKnSRblZUUc/P/rqFM4ZncJ2eby7dSIUu0o18PsdtL20gyoz/+PpEoqJ0iaJ0HxW6SDd68oNdrNpZwd0XjyU7PdHrOBLmVOgi3WRnWR0Pr9jGeaP7csXUQV7HkQigQhfpBj6f48d/2kBcdBQ//+p43Q0qPUKFLtINnvpwF6t3HeLui0+jn1Yfkh6iQhcJsN3ldTy4fBtfOjWLy6dkex1HIogKXSSAnHPc8fImYqM01SI9T4UuEkAvrC7iw8Jy7rxwDAN66aoW6VkqdJEAOVjdyP3LtnL68D5cNS3H6zgSgVToIgHgnOOuJZtpbvXxwFcn6AYi8YRfhW5mM80s38wKzOyOoxxzrpmtN7M8M3snsDFFgtvfNh/gjS0Hue2CUQzNTPY6jkSoYy5wYWbRwGPA+UAxsNrMljrntnQ6Jh14HJjpnNtjZn27Ka9I0KlqaOGepXmMy07j+rOGeR1HIpg/I/TpQIFzrtA51ww8D1x6xDFzgVecc3sAnHMlgY0pErweXL6N8tomHvjqBGKiNYsp3vHnry8bKOq0XdzxWmejgN5m9raZrTWza7v6QWa2wMzWmNma0tLSE0ssEkRW76rg2ZV7uOHsYYzL7uV1HIlw/hR6V2d33BHbMcBU4CLgX4CfmtlnlmRxzi1yzuU653KzsrKOO6xIMGlqbePOVzaRnZ7IredrBSLxnj+LRBcDna/BGgTs6+KYMudcHVBnZu8CE4HtAUkpEoT+3zuFFJTU8uT8aSTFab118Z4/I/TVwEgzG2ZmccBVwNIjjvkL8AUzizGzJGAGsDWwUUWCx86yOh59q4DZEwbwpVN1DYAEh2MOK5xzrWZ2M7ACiAYWO+fyzOzGjv0LnXNbzWw5sBHwAU845zZ3Z3ARr7Rfc76J+Ogo7p491us4Iof59e9E59wyYNkRry08Yvth4OHARRMJTn9Zv4/3C8q5d844+upJihJEdI2VyHGorG/mvte2MCknnXnTB3sdR+Sf6EyOyHF4cHk+h+pbePr68bq9X4KORugiflq7+xDPrdrD/DOHMnZgmtdxRD5DhS7ih9Y2H3ct2Uz/tAS+r2vOJUip0EX88PsPdrF1fzX3XDyWlHjNVEpwUqGLHMP+qgb+843tnHtqFjPH9fc6jshRqdBFjuHeV7fQ6nP87JJxWlJOgpoKXeRzvJ1fwrJNB7j5S6cwOCPJ6zgin0uFLnIUjS1t3LM0j2GZySw4Z7jXcUSOSWd3RI5i4TufsLu8nmdumEF8TLTXcUSOSSN0kS7sKqvj8bc/4eKJAzl7ZKbXcUT8okIXOYJzjnuW5hEXHcVdF43xOo6I31ToIkdYkXeAd7aXcuv5o+inh29JCFGhi3RS19TKv/91C6P7p/LNM4Z4HUfkuOikqEgn//WPHeyvauTRuZO14LOEHP3FinTYfrCG/35vJ1/PHcTUIX28jiNy3FToIrSfCP3pks0kx8dw+8zRXscROSEqdBFgyfq9rNxZwe0zR5OREu91HJETokKXiFdV38L9r21lYk46V03L8TqOyAlToUvE+4/X86moa+b+OeO0CpGENBW6RLSNxZU8s3I3154xlHHZvbyOI3JSVOgSsdp8jruWbCYjOZ4fXKBViCT0qdAlYj27cjcbi6v46ewxpCXEeh1H5KSp0CUildQ08tCKfM46JYNLJg70Oo5IQKjQJSL9/LWtNLX4uPdSrUIk4UOFLhHng4Iylqzfx43njmB4VorXcUQCRoUuEaWptY27/rKZIRlJfOfcEV7HEQkoPZxLIsrCtwspLK3j6eunkxCrVYgkvGiELhGjsLSWx94u4OKJA/niqCyv44gEnApdIoJz7decx8dE8dPZWoVIwpMKXSLCkvV7+eCTcn48czR9U7UKkYQnvwrdzGaaWb6ZFZjZHZ9z3DQzazOzKwIXUeTkHKpr5r5XtzIpJ5150wd7HUek2xzzpKiZRQOPAecDxcBqM1vqnNvSxXEPAiu6I6jIifr5sq1UNrTwh8vG6+FbEtb8GaFPBwqcc4XOuWbgeeDSLo77HvAyUBLAfCIn5YOCMl5aW8yCLw5n7MA0r+OIdCt/Cj0bKOq0Xdzx2mFmlg1cBiz8vB9kZgvMbI2ZrSktLT3erCLHpbGljZ/8eRNDMpK45byRXscR6Xb+FHpX/0Z1R2z/GrjdOdf2eT/IObfIOZfrnMvNytJlY9K9Hv1HAbvK67l/znhdcy4RwZ8bi4qBzsu4DAL2HXFMLvB8xzMxMoELzazVObckECFFjtfW/dUsfOcTvjo5m7NHZnodR6RH+FPoq4GRZjYM2AtcBcztfIBzbtinX5vZ74FXVebildY2H7e/vJFeibH8dPZYr+OI9JhjFrpzrtXMbqb96pVoYLFzLs/MbuzY/7nz5iI97cn3d7GxuIrfXD2Z3slxXscR6TF+PcvFObcMWHbEa10WuXPuupOPJXJidpXV8cs38vnKmH7MnjDA6zgiPUp3ikrYcM5x5yubiI2K4r45es65RB4VuoSNP67cw4eF5dx54Rj699Lt/RJ5VOgSFooq6vnFsq2cfUomV0/POfY3iIQhFbqEPOccd7yyEYAHLh+vqRaJWCp0CXnPrSri/YJyfnLRGAb1TvI6johnVOgS0ooq6vn5sq2cOSKDuXqSokQ4FbqELJ/P8cOXNgDw4OUTNNUiEU+FLiFr8fs7WbmzgrsvHktOH021iKjQJSTtOFjDQyvabyD62tRBXscRCQoqdAk5LW0+bn1xPSnxMfziq7qqReRTft36LxJMfv3mdjbvrWbhNVPISo33Oo5I0NAIXULKysJyHn/7E76eO4iZ4/SsFpHOVOgSMqrqW7j1hfUM6ZPEPRef5nUckaCjKRcJCc45frJkEyU1Tbx805kkx+tPV+RIGqFLSHhpbTGvbdzPreePYmJOutdxRIKSCl2C3o6DNdz9l82cMTyDG88Z4XUckaClQpeg1tDcxs3PriM5LoZHrppEdJQuURQ5Gk1ESlD72at55B+s4anrp9M3Tc84F/k8GqFL0Fqybi/PrSripnNHcM6oLK/jiAQ9FboEpfwDNdz5yiamDe3ND84f5XUckZCgQpegU93Ywo3PrCUlIYbH5k4hNlp/piL+0By6BBXnHD96aQN7Kup57tuna95c5Dho6CNB5bfvfMKKvIPcOWs004f18TqOSEhRoUvQ+PvWgzy8Ip/ZEwZww9nDvI4jEnJU6BIUCkpquOX59YwdkMbDV0zUI3FFToAKXTxXVd/Ct59eS0JsFIuuzSUxLtrrSCIhSSdFxVMtbT6+8+xaig/V8+y3Tyc7PdHrSCIhS4UunnHOcdefN/N+QTkPXzGBaUN1ElTkZGjKRTzz23c+4YU1RXzvy6fwtdwcr+OIhDwVunji1Y37eGh5PpdMHKg7QUUCRIUuPe79gjJufWE904f24aErJuiKFpEA8avQzWymmeWbWYGZ3dHF/nlmtrHj4wMzmxj4qBIONhVXseDpNQzPTOF31+aSEKsrWkQC5ZiFbmbRwGPALGAscLWZjT3isJ3AOc65CcC9wKJAB5XQt7OsjuueXEV6UhxP3zCdXkmxXkcSCSv+jNCnAwXOuULnXDPwPHBp5wOccx845w51bH4EDApsTAl1xYfqueaJlfic4+kbptNPz2gRCTh/Cj0bKOq0Xdzx2tHcAPytqx1mtsDM1pjZmtLSUv9TSkg7UNXIvCdWUt3Ywh9umMGIrBSvI4mEJX8KvaszVq7LA82+RHuh397VfufcIudcrnMuNytLCxZEgrLaJuY98RFlNU08df10xmX38jqSSNjy58aiYqDzRcKDgH1HHmRmE4AngFnOufLAxJNQVlrTXuZ7Kxt4av50pgzu7XUkkbDmzwh9NTDSzIaZWRxwFbC08wFmNhh4BfiGc2574GNKqDlQ1ciViz6kqKKBxd+cxozhGV5HEgl7xxyhO+dazexmYAUQDSx2zuWZ2Y0d+xcCdwMZwOMd1xS3Oudyuy+2BLO9lQ3M/d3/TrPoueYiPcOc63I6vNvl5ua6NWvWePK7pfsUlNTyzcWrqG5s4anrNc0iEmhmtvZoA2Y9nEsCZn1RJfOfXEV0lPHct0/XCVCRHqZCl4B4d3spNz6zloyUOP5w/QyGZiZ7HUkk4qjQ5aQ9t2oPdy3ZzKh+qTw1f5oWdhbxiApdTlibz/Hg8m0sereQc0Zl8ejcyaQm6HZ+Ea+o0OWE1DS28IMXN/DGloNce8YQ7p49lphoPbxTxEsqdDluBSW1/J8/rGFXeT3/dvFYrjtrmNeRRAQVuhyn5ZsP8MOXNhAfE8UzN8zgjBG6YUgkWKjQxS9NrW088LdtPPn+LibmpLPwmikM6KUFnUWCiQpdjqmwtJbvPbeOvH3VzD9rKHfMGk18jBamEAk2KnQ5Kuccz60q4r7XthAXE8Xvrs3l/LH9vI4lIkehQpcuHahq5PaXN/LO9lLOOiWDh6+YyMB0TbGIBDMVuvwTn8/x4poifr5sKy1tjp9dehrXzBhCVJQWchYJdip0OaygpIafvLKZVbsqmD6sDw9ePoFhuoVfJGSo0IXaplYe/UcBi/9nJ4lx0Tx0+QS+ljuIjkchi0iIUKFHMJ/P8cq6vTy4fBulNU1cPmUQd144msyUeK+jicgJUKFHIOccb28v5aHl+WzdX83EnHQWfWMqk/XscpGQpkKPMKt3VfDL1/P5qLCCnD6J/PrKSVwycaBOeoqEARV6hPiosJxH3tzBh4XlZKbE8W8Xj2XujCHExeiBWiLhQoUextp8jje2HGDRu4V8vKeSrNR47rpoDPNmDCExTnd6ioQbFXoYqmpo4eW1xTz14S52l9czuE8S/37JaVw5LYeEWBW5SLhSoYcJ5xwbi6t4btUelqzfS2OLjymD07lj5mguOK0/0ZojFwl7KvQQV1rTxF/W7+XFNUVsP1hLYmw0l03OZt6MIVqkWSTCqNBDUFV9CyvyDrB0wz4++KQMn4NJOencf9k4Zk8YSK9ELQMnEolU6CHiYHUjr285yOt5B/jwk3JafY7BfZL4zrmncOmkgYzsl+p1RBHxmAo9SLW0+dhQVMnb+aW8lV9C3r5qAIZlJnPDF4Yxa9wAJg7qpdvzReQwFXqQaG3zsXV/DSt3lvPBJ+WsLCynrrmN6Chj6uDe/HjmqZw3uh+j+qWoxEWkSyp0j1TWN7O+qJJ1eyr5eM8h1u2ppLapFWgfhV82JZuzRmRy5ohMeiVpTlxEjk2F3s2cc5TWNLH1QA1b9lWzeW8Vm/ZWsaeiHoAog1H9UpkzeSDTh2UwfWgf+vdK8Di1iIQiFXqA+HyO/dWN7Cyto7Cslh0Ha9l+sIYdJbVU1DUfPm5Q70TGZ/fiymk5TB6czoRB6aTE63+DiJw8NYmfnHNUN7Syt7KBvZUNFB+qp6iigT0V9eypqGNPRT2NLb7Dx6cmxDCqXyoXjO3H6P6pjB6Qxuj+qaQnxXn4LkQknEV8oft8jsqGFsprmyirbaa0tomymiZKapooqW7kYE0j+6sa2V/ZSENL2z99b0JsFEP6JDMkI5kvjsxieFYKwzKTGZ6VTN/UeJ28FJEe5Vehm9lM4BEgGnjCOffAEfutY/+FQD1wnXPu4wBn7ZJzjqZWH3VNrdQ1tVHT1EJtYys1ja3UNLVQ3dBKdUMLVQ0tVDa0UFnfQmV9M4fqm6msb+FQfTM+99mfGxtt9E1NoF9aPKf2S+XcUX0ZmJ7AgF6JDOqdSHbvRDKS41TaIhI0jlnoZhYNPAacDxQDq81sqXNuS6fDZgEjOz5mAL/t+Bxwb+WXcN+rW6hvbuv4aKWlrYtGPkJSXDTpibGkJcbSOymOUzumPzKS4+jT8ZGZEk/f1HgyU+JJT4pVWYtISPFnhD4dKHDOFQKY2fPApUDnQr8UeNo554CPzCzdzAY45/YHOnCvxFhGD0gjOS6apLgYkuKiSY6PISU+5vDn1IT2z2mJsaQlxJCaEKvnfotI2POn0LOBok7bxXx29N3VMdnAPxW6mS0AFgAMHjz4eLMCMGVwb6bM1VJpIiJH8mfY2tW8w5FzHP4cg3NukXMu1zmXm5WV5U8+ERHxkz+FXgzkdNoeBOw7gWNERKQb+VPoq4GRZjbMzOKAq4ClRxyzFLjW2p0OVHXH/LmIiBzdMefQnXOtZnYzsIL2yxYXO+fyzOzGjv0LgWW0X7JYQPtli/O7L7KIiHTFr+vQnXPLaC/tzq8t7PS1A74b2GgiInI8dC2fiEiYUKGLiIQJFbqISJiw9ulvD36xWSmw25NffnIygTKvQ/QwvefwF2nvF0L3PQ9xznV5I49nhR6qzGyNcy7X6xw9Se85/EXa+4XwfM+achERCRMqdBGRMKFCP36LvA7gAb3n8Bdp7xfC8D1rDl1EJExohC4iEiZU6CIiYUKFfhLM7Idm5sws0+ss3cnMHjazbWa20cz+bGbpXmfqLmY208zyzazAzO7wOk93M7McM3vLzLaaWZ6Z3eJ1pp5iZtFmts7MXvU6S6Co0E+QmeXQvs7qHq+z9IA3gHHOuQnAduBOj/N0i07r584CxgJXm9lYb1N1u1bgNufcGOB04LsR8J4/dQuw1esQgaRCP3H/CfyYLlZmCjfOudedc60dmx/RvoBJODq8fq5zrhn4dP3csOWc2++c+7jj6xraCy7b21Tdz8wGARcBT3idJZBU6CfAzC4B9jrnNnidxQPXA3/zOkQ3OdrauBHBzIYCk4GVHkfpCb+mfUDm8zhHQPn1PPRIZGZvAv272PWvwE+AC3o2Uff6vPfrnPtLxzH/Svs/0f/Yk9l6kF9r44YjM0sBXga+75yr9jpPdzKz2UCJc26tmZ3rcZyAUqEfhXPuK129bmbjgWHABjOD9umHj81sunPuQA9GDKijvd9Pmdk3gdnAeS58b16IyLVxzSyW9jL/o3PuFa/z9ICzgEvM7EIgAUgzs2ecc9d4nOuk6caik2Rmu4Bc51woPrXNL2Y2E/gVcI5zrtTrPN3FzGJoP+l7HrCX9vV05zrn8jwN1o2sfVTyFFDhnPu+x3F6XMcI/YfOudkeRwkIzaGLPx4FUoE3zGy9mS081jeEoo4Tv5+un7sVeDGcy7zDWcA3gC93/L9d3zFylRCkEbqISJjQCF1EJEyo0EVEwoQKXUQkTKjQRUTChApdRCRMqNBFRMKECl1EJEz8f6GEJvuT5QrFAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "c772e0aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.5000)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "6f49bb5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def act_softmax(x):\n",
    "    t = x.exp().sum()\n",
    "    print(t)\n",
    "    return (x.exp() - x.max())/t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "afcd1abd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(1544.6826)\n"
     ]
    }
   ],
   "source": [
    "y = act_softmax(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "cda3cb36",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x162a3e610>]"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAb70lEQVR4nO3de3Bc5Znn8e+j1tWSbfki3yRfsYNtDMGOIhxIsgkkGdtk4llmJwskIYGdcaUGZpNUJhmS7E5qK6mprcpuQkhYWAJkhgwLuQeTMXFISFKBCQYbGV+wsYWFLcmSdbGs+627n/2j29AIyWrb6j7drd+nqqv7nPc9fZ5TUD8dv+c9p83dERGR3JUXdAEiIpJaCnoRkRynoBcRyXEKehGRHKegFxHJcflBFzCWuXPn+rJly4IuQ0Qka+zZs6fd3SvGasvIoF+2bBm7d+8OugwRkaxhZsfHa9PQjYhIjlPQi4jkOAW9iEiOU9CLiOQ4Bb2ISI5T0IuI5DgFvYhIjlPQi4hkgKdePsX//cOrpOLR8Qp6EZEM8MRLJ3n4T8cxs0n/bgW9iEgGqGvtZdX8spR8t4JeRCRgkajzalsvKysU9CIiOampc4ChcFRn9CIiuepoaw8AK+cp6EVEclJday8AKyump+T7FfQiIgE72tpLxfQiZk4rSMn3K+hFRAJW19rLqhQN24CCXkQkUO7Oq629KRufBwW9iEigTnUP0TMUVtCLiOSq1y/EKuhFRHJTqqdWgoJeRCRQda29zCwpoKKsKGX7UNCLiASoLn4hNhUPMztLQS8iEqBUT60EBb2ISGBO9w3T0Tec0vF5UNCLiAQmHTNuQEEvIhIYBb2ISI472tpDSUGIRTNLUrofBb2ISECOnorNuMnLS92MG1DQi4gE5nBLD6sXpObRxIkU9CIiAWjrGaK9d4jVC2ekfF8KehGRALzSEnv0wRqd0YuI5KbDLd0AXJopQW9mm8zsFTOrM7M7x2g3M7s73r7PzDYktH3OzA6a2QEze9TMiifzAEREstGh5h7mTS9iTgqfcXPWhEFvZiHgHmAzsBa4yczWjuq2GVgVf20D7o1vWwn8V6Da3dcBIeDGSateRCRLHW7pTsv4PCR3Rl8D1Ln7MXcfBh4Dto7qsxV42GOeA8rNbGG8LR8oMbN8YBpwcpJqFxHJSuFIlKOtvWmZcQPJBX0l0JCw3BhfN2Efd28C/hdwAmgGutz912PtxMy2mdluM9vd1taWbP0iIlnntY4+hsPRjAr6sWbyezJ9zGwWsbP95cAioNTMPj7WTtz9fnevdvfqioqKJMoSEclOh5pjM25WL8icoZtGYHHCchVvHX4Zr88HgHp3b3P3EeBnwNUXXq6ISPY73NJNfp5xybzStOwvmaB/AVhlZsvNrJDYxdTto/psB26Jz77ZSGyIppnYkM1GM5tmsafqXwccmsT6RUSyzuHmHi6pKKMoP5SW/eVP1MHdw2Z2B7CT2KyZh9z9oJl9Ot5+H7AD2ALUAf3ArfG2XWb2E+BFIAzUAven4kBERLLF4ZYe3rF0Vtr2N2HQA7j7DmJhnrjuvoTPDtw+zrZfBb56ETWKiOSMroERms4M8LGNS9K2T90ZKyKSRm88+iA9F2JBQS8iklZnH32wemF6plaCgl5EJK0Ot/Qws6SABTPS9zQYBb2ISBq9fLKbSxdMJzYRMT0U9CIiaRKORDnU3M3llTPTul8FvYhImtS19TIUjrKuMn0XYkFBLyKSNvsbuwB0Ri8ikqsONHUxrTDE8rllad2vgl5EJE0OnOzmskUzCOWl70IsKOhFRNIiEnVePtnNZYvSO2wDCnoRkbR4ta2XgZFI2sfnQUEvIpIWB5riF2KrFPQiIjlpf1MXxQV5XFKR3guxoKAXEUmLA01drF2Y/guxoKAXEUm5aNQ5eDL9d8SepaAXEUmxY+199A9HWKegFxHJTUFeiAUFvYhIyh1o6qIoP4+VAVyIBQW9iEjK7W/qYs3CGeSHgolcBb2ISAqFI1H2N3VxRUDDNqCgFxFJqSOneukfjrBhyazAalDQi4ikUG1DJ4CCXkQkV9WeOMOc0kIWzy4JrAYFvYhICtWe6GT9kvK0/kbsaAp6EZEU6eof4dW2PtYHOGwDCnoRkZTZ23gGgPWLywOtQ0EvIpIitSc6MYMrFPQiIrmp9sQZLp0/nbKi/EDrUNCLiKRANOrsbTjD+iXlQZeioBcRSYX6jj66BkZYvzjYC7GgoBcRSYkXj8dvlFpaHmwhKOhFRFKituEM04vzWTE3mCdWJkoq6M1sk5m9YmZ1ZnbnGO1mZnfH2/eZ2YaEtnIz+4mZHTazQ2b2rsk8ABGRTFR74gxXLi4nL4CfDhxtwqA3sxBwD7AZWAvcZGZrR3XbDKyKv7YB9ya0fRv4lbuvBt4OHJqEukVEMlb34AiHW7oDfb5NomTO6GuAOnc/5u7DwGPA1lF9tgIPe8xzQLmZLTSzGcB7gQcB3H3Y3c9MXvkiIplnz2uduMNVK2YHXQqQXNBXAg0Jy43xdcn0WQG0Ad83s1oze8DMSsfaiZltM7PdZra7ra0t6QMQEck0u+pPUxCyjJhxA8kF/VgDTJ5kn3xgA3Cvu68H+oC3jPEDuPv97l7t7tUVFRVJlCUikpmer+/giqpySgpDQZcCJBf0jcDihOUq4GSSfRqBRnffFV//E2LBLyKSkwaGI+xr7KJmeWYM20ByQf8CsMrMlptZIXAjsH1Un+3ALfHZNxuBLndvdvcWoMHMLo33uw54ebKKFxHJNLUnOglHPaOCfsIHMLh72MzuAHYCIeAhdz9oZp+Ot98H7AC2AHVAP3Brwlf8HfBI/I/EsVFtIiI55bn60+QZVC/NjPF5SCLoAdx9B7EwT1x3X8JnB24fZ9u9QPWFlygikj2er+/gskUzmV5cEHQpr9OdsSIik2QoHKH2xJmMGrYBBb2IyKTZ39jFUDiqoBcRyVW76k8D8M5lCnoRkZz0fP1p3ja/jNmlhUGX8iYKehGRSTASibLneGfGnc2Dgl5EZFLsazxD71CYa1bODbqUt1DQi4hMgj8ebccMrr5kTtClvIWCXkRkEjxztJ3LK2dSPi2zxudBQS8ictF6BkeobTjDuzNw2AYU9CIiF23XsdNEos67VynoRURy0jN17RQX5PGODHq+TSIFvYjIRXqmrp2a5XMoys+M58+PpqAXEbkIzV0D1LX28p4MHZ8HBb2IyEV55mg7QEbOnz9LQS8ichGeqWtnblkhqxdMD7qUcSnoRUQukLvzbF0716ycS17eWD+dnRkU9CIiF+jgyW7ae4d5z6qKoEs5JwW9iMgF+u2hVszgfZcq6EVEctLTh09x5eJy5pYVBV3KOSnoRUQuQGvPIC81dnHd6nlBlzIhBb2IyAX4/eE2AK5dPT/gSiamoBcRuQC/PXyKhTOLWbMwc6dVnqWgFxE5T0PhCH882s61q+dhlrnTKs9S0IuInKddx07TPxzhujWZPz4PCnoRkfP29OFWigvyuPqSzH3sQSIFvYjIeXB3fnv4FNdcMpfigsx8WuVoCnoRkfNw5FQvDacHuDZLhm1AQS8icl527G/GDD60dkHQpSRNQS8ich6ePNBMzbLZVEzP7LthEynoRUSSVNfaw5FTvWy5fGHQpZwXBb2ISJKe3N8CwKZ12TNsAwp6EZGk7TjQQvXSWcyfURx0KeclqaA3s01m9oqZ1ZnZnWO0m5ndHW/fZ2YbRrWHzKzWzH45WYWLiKRTfXsfh5q72ZxlwzaQRNCbWQi4B9gMrAVuMrO1o7ptBlbFX9uAe0e1fwY4dNHViogE5MkDzUD2DdtAcmf0NUCdux9z92HgMWDrqD5bgYc95jmg3MwWAphZFXA98MAk1i0iklY79jdz5eJyKstLgi7lvCUT9JVAQ8JyY3xdsn3uAr4IRM+1EzPbZma7zWx3W1tbEmWJiKTHiY5+DjR1s+Xy7Dubh+SCfqxHs3kyfczsw0Cru++ZaCfufr+7V7t7dUVFZv8sl4hMLb/Y2wTA9VcsCriSC5NM0DcCixOWq4CTSfa5BviImb1GbMjnWjP71wuuVkQkzdydn9c2sXHF7KwctoHkgv4FYJWZLTezQuBGYPuoPtuBW+KzbzYCXe7e7O5fcvcqd18W3+5pd//4ZB6AiEgq7W04Q317Hzesrwq6lAuWP1EHdw+b2R3ATiAEPOTuB83s0/H2+4AdwBagDugHbk1dySIi6fPz2iaK8vPYlKXj85BE0AO4+w5iYZ647r6Ezw7cPsF3/B74/XlXKCISkOFwlCdeOskH1s5nRnFB0OVcMN0ZKyIyjj8caaOzf4Qb1o+eaJhdFPQiIuP4eW0jc0oLee/bsnsmoIJeRGQMXQMj/OZQK3/+9kUUhLI7KrO7ehGRFNm+t4nhcJQbNmT3sA0o6EVE3sLdeWTXCS5bNIPLK2cGXc5FU9CLiIxS23CGwy093FSzBLOxbvzPLgp6EZFRHt11gmmFIbZemZ2PPBhNQS8ikqBrYIQn9p1k65WLmJ7Fc+cTKehFRBL8oraJwZEoN9csDbqUSaOgFxGJc3f+364TXF45k8ursv8i7FkKehGRuBdPdPLKqdhF2FyioBcRiXvo2deYXpyfMxdhz1LQi4gADaf7eXJ/MzfXLKG0KKnnPWYNBb2ICPAv//4aZsYnr14WdCmTTkEvIlNez+AIj73QwPWXL2RRlv6K1Lko6EVkyvvR7kZ6h8L89XuWB11KSijoRWRKC0eifP/ZemqWzeaKqvKgy0kJBb2ITGm/OthCY+cAt707N8/mQUEvIlNYNOp89+k6Vswt5YNr5wddTsoo6EVkyvrNoVMcbunhjmtXEsrL/qdUjkdBLyJTkrtz99NHWTpnGh95e27dIDWagl5EpqTfvdLKgaZubn//SvKz/KcCJ5LbRyciMgZ359u/raNqVgn/cX32/1TgRBT0IjLl/OFIGy81nOH296/M+h/+TkbuH6GISIJo1PnGzleoLC/hLzdUBV1OWijoRWRKeWLfSQ6e7OYLf3YphflTIwKnxlGKiABD4Qjf2PkKly2akfMzbRIp6EVkyvjBn47T2DnAlzavIS+H582PpqAXkSmha2CE7/6ujvesmsu7V80Nupy0UtCLyJTwf35XR9fACHduXh10KWmnoBeRnFfX2sODz9TzV++o4rJFufOj38lS0ItITnN3/vHxg0wrDPEPm6be2TwkGfRmtsnMXjGzOjO7c4x2M7O74+37zGxDfP1iM/udmR0ys4Nm9pnJPgARkXP55b5m/v3VDr6waTVzyoqCLicQEwa9mYWAe4DNwFrgJjNbO6rbZmBV/LUNuDe+Pgx83t3XABuB28fYVkQkJXqHwnz9315mXeUMbq5ZEnQ5gUnmjL4GqHP3Y+4+DDwGbB3VZyvwsMc8B5Sb2UJ3b3b3FwHcvQc4BOT+gyVEJCN866kjnOoe4mtb1+X0Y4gnkkzQVwINCcuNvDWsJ+xjZsuA9cCusXZiZtvMbLeZ7W5ra0uiLBGR8e05fpqHnq3nY1ctYf2SWUGXE6hkgn6sP4N+Pn3MrAz4KfBZd+8eayfufr+7V7t7dUVFRRJliYiMbXAkwhd+vI9FM0v40pY1QZcTuPwk+jQCixOWq4CTyfYxswJiIf+Iu//swksVEUnON586wrH2Ph7566soK0om5nJbMmf0LwCrzGy5mRUCNwLbR/XZDtwSn32zEehy92YzM+BB4JC7f3NSKxcRGcOe451874/HuPmqJVyzcmrdATueCf/UuXvYzO4AdgIh4CF3P2hmn4633wfsALYAdUA/cGt882uATwD7zWxvfN2X3X3HpB6FiAjQMzjC53+0NzZkMwXvgB1PUv+miQfzjlHr7kv47MDtY2z3DGOP34uITCp357/94gAnTvfz2LZ3Mb24IOiSMobujBWRnPDjPY08vvckn/vA26hZPjvocjKKgl5Esl5daw9fffwgV18yh799/8qgy8k4CnoRyWq9Q2H+9pEXmVYY4q7/fOWUvjFqPJp3JCJZKxp1PvfDvbza1scPbqth3ozioEvKSDqjF5GsdddvjvDUy6f479ev4WpNpRyXgl5EstKO/c3c/XQdH62u4pNXLwu6nIymoBeRrLPn+Gk+98O9bFhSztf+Yh2xezNlPAp6EckqR0/1cNs/72ZReQnfu6WaovxQ0CVlPAW9iGSN5q4BbnnoeQrz83j4tpop+0Mi50tBLyJZob13iFsefJ6ewTDf/9Q7WTx7WtAlZQ0FvYhkvI7eIT72vV00dPZz/y3vYF3l1PuB74uhoBeRjNbRO8TN39vFax19PPjJd3L1JZpGeb4U9CKSsVq7B/nYA7GQf+hT79Rjhy+Q7owVkYxU397HJx7cxem+YYX8RVLQi0jG2dd4hlu//wIOPLZtI1dUlQddUlZT0ItIRtl5sIXP/XAvs0sLefi2GlZUlAVdUtZT0ItIRnB3vvN0Hd986ghvX1zO9z7xDj2kbJIo6EUkcL1DYb74k5fYsb+FG9ZX8k83XE5xge54nSwKehEJ1IGmLv7u0VqOd/Tx5S2r+Zv3rNCzayaZgl5EAuHu/OC543z9l4eYXVrIo3+zkatWzAm6rJykoBeRtGvuGuDOn+7nD0faeP+lFfzvj17J7NLCoMvKWQp6EUkbd+fHexr52hMvE446/+Mjl/GJjUvJ08//pZSCXkTS4uipHv7x8YP86VgHNctn843/dAVL55QGXdaUoKAXkZTqHQrznd8e5cFn6iktyufrf7GOm2uW6Cw+jRT0IpISI5Eojz1/grt+c5SOvmE+Wl3FP2xarWfIB0BBLyKTKhJ1/m1/M9966gj17X3ULJ/Ng1vWcOXi8qBLm7IU9CIyKSJR54mXTvKdp4/yalsfq+aV8cAt1Vy3Zp7mxQdMQS8iF6V3KMyPdzfw0LP1NJwe4NL507nn5g1sXrdA4/AZQkEvIhfk1bZeHt11gh/ubqBnMEz10ll8ZctaPrR2vgI+wyjoRSRpA8MRfv1yC48+f4Lnjp0mP8/YtG4B/+Xdy1m/ZFbQ5ck4FPQick7hSJQ/HevgF7Un+dWBZvqGI1TNKuELf3Ypf1VdxbzpesJkplPQi8hbDI5E+NOxDnYeaGHnwRY6+0coK8rn+isWcsOGKmqWzdbwTBZJKujNbBPwbSAEPODu/3NUu8XbtwD9wKfc/cVkthWRzNDY2c8zR9t5+nArz9S10z8cobQwxAfWzmfzuoW879IKPTo4S00Y9GYWAu4BPgg0Ai+Y2XZ3fzmh22ZgVfx1FXAvcFWS24pIAFq7B9lVf5pd9R08W9dBfXsfAItmFvOXG6q4ds083rVijsI9ByRzRl8D1Ln7MQAzewzYCiSG9VbgYXd34DkzKzezhcCyJLYVkRQbDkc5cqqH2oYz1B7v5MUTnbzW0Q/AtMIQVy2fzcc3LuW9q+aycl6Z5r3nmGSCvhJoSFhuJHbWPlGfyiS3BcDMtgHbAJYsWZJEWSIylu7BEY609HCopYdDzd0caOricHMPw5EoAHPLili/pJyPXbWUmuWzuWzRDPJDeQFXLamUTNCP9afdk+yTzLaxle73A/cDVFdXj9lHRGKiUae5e5D6tj7qO/o41tZLXWvs1dw1+Hq/GcX5rKucya3XLGNd5UyuXFxO1awSnbFPMckEfSOwOGG5CjiZZJ/CJLYVkVGGw1FOdQ/S0j3IyTMDNJ0ZoKkz9n6io5/GzoHXz9ABSgpCrJxXxsYVc1g5r4w1C6ezesEMFs4sVqhLUkH/ArDKzJYDTcCNwM2j+mwH7oiPwV8FdLl7s5m1JbGtSM5zd/qGI3T2DdPZP8zpvtiro3eY9t4h2nuHaesdorV7kNaeIU73Db/lO8qnFVBZXsLqhdP54GXzWTJ7GsvnlrJibhnzZxQp0GVcEwa9u4fN7A5gJ7Epkg+5+0Ez+3S8/T5gB7GplXXEplfeeq5tU3IkIikQjkQZGIkwMByhfzhC33CYgeEIfcMR+obC9A2F6Y2/9wyF6Rk8+xqhayD26o6/j0TGHpEsDOUxp6yQiulFVM2axoals5g/vZiFM4tZMDP2vqi8hNIi3fYiF8ZiE2UyS3V1te/evTvoMuQc3J2ov/Eejf9/FE1cHz27/Oa+EXei0dj6SPSN7SPRhFe8Tzgae4947HMkEnsPR6NEos5IxIlEo4xEnHAkSji+biQSJRyJMhz/PBKJMhyOMnz2Pf55aCT2PjgSYSgcex8ciTI0EmEwHBk3nMdSGMpjenE+M0oKYu/FBcwsKWBGSQHl0wqYNa2A8pJCZpUWMru0gNmlRcwuLWRGcb7OxuWimdked68eqy2nThE+/J0/MjgSnbBfMn/cxu0xTkPi6sTvf/P6xP4+9vpzlOb+xlajv+vssr+pLbbeE2o62/76dyW0v96WsB3x5ai/uU+2KAzlUZifR0HIKIh/LszPozCUR1F+HkX5IUoKQsyaVkBRfii2riBEcUEexQUhphWEKCkMUVQQorQwxLTCECWF+ZQVhZhWmE9pYT5lxfmUFoUoytd8c8lMORX0KyvKkj8DS+IEarwu45192Zv6jLfexlzPm/q/+ftHf9fZ5cR+Zon9LKFPvO0t6+yN7eJtr/e1+DcntOUl9jN703Io743vCpm9/p15eUbobP+8N7YJ5UGeGXlmhPLeaAvFl8+uS1zOj68ryMt7Y10ovhwyCvKM/FDe6+sKQrE+OlMWybGgv+vG9UGXICKScXSXhIhIjlPQi4jkOAW9iEiOU9CLiOQ4Bb2ISI5T0IuI5DgFvYhIjlPQi4jkuIx81k38qZfHg67jPM0F2oMuIs10zFODjjk7LHX3irEaMjLos5GZ7R7vgUK5Ssc8NeiYs5+GbkREcpyCXkQkxynoJ8/9QRcQAB3z1KBjznIaoxcRyXE6oxcRyXEKehGRHKegTwEz+3szczObG3QtqWZm3zCzw2a2z8x+bmblQdeUCma2ycxeMbM6M7sz6HpSzcwWm9nvzOyQmR00s88EXVO6mFnIzGrN7JdB1zJZFPSTzMwWAx8ETgRdS5o8Baxz9yuAI8CXAq5n0plZCLgH2AysBW4ys7XBVpVyYeDz7r4G2AjcPgWO+azPAIeCLmIyKegn37eAL3KO3xfPJe7+a3cPxxefA6qCrCdFaoA6dz/m7sPAY8DWgGtKKXdvdvcX4597iAVfZbBVpZ6ZVQHXAw8EXctkUtBPIjP7CNDk7i8FXUtAbgOeDLqIFKgEGhKWG5kCoXeWmS0D1gO7Ai4lHe4idqIWDbiOSZVTPw6eDmb2G2DBGE1fAb4MfCi9FaXeuY7Z3R+P9/kKsX/uP5LO2tLExlg3Jf7FZmZlwE+Bz7p7d9D1pJKZfRhodfc9Zva+gMuZVAr68+TuHxhrvZldDiwHXjIziA1hvGhmNe7eksYSJ914x3yWmX0S+DBwnefmjRmNwOKE5SrgZEC1pI2ZFRAL+Ufc/WdB15MG1wAfMbMtQDEww8z+1d0/HnBdF003TKWImb0GVLt7tj0B77yY2Sbgm8B/cPe2oOtJBTPLJ3ah+TqgCXgBuNndDwZaWApZ7GzlX4DT7v7ZgMtJu/gZ/d+7+4cDLmVSaIxeLtZ3genAU2a218zuC7qgyRa/2HwHsJPYRckf5XLIx10DfAK4Nv7fdW/8TFeykM7oRURynM7oRURynIJeRCTHKehFRHKcgl5EJMcp6EVEcpyCXkQkxynoRURy3P8HWSo8F/KnugcAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3f53624",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "b00d854f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.4893,  0.0763, -0.5147,  0.4051,  0.4859],\n",
       "        [-0.9563, -2.0963,  2.0460, -0.2998, -0.5597],\n",
       "        [ 0.7562,  1.0429, -1.1559,  1.3175,  0.8683],\n",
       "        [ 2.6986, -0.0074, -1.6993,  0.5111, -0.2448],\n",
       "        [-0.3942, -0.4390,  0.7185, -0.5542, -0.8304]])"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xr = torch.randn(200, 100)\n",
    "x = xr.clone()\n",
    "for i in range(50): \n",
    "    x = x @ torch.randn(100,100)*0.1\n",
    "x[0:5,0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "55be1186",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(-0.0073), tensor(0.9751))"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.mean(), x.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "8dbb26bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.tensor([1, 2, 3, 4, 5]).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "79a65062",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(3.), tensor(1.5811))"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.mean(), x.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "2b9d172d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.01706540159881115, 98.8023494720459)"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean,sqr = 0.,0.\n",
    "for i in range(100):\n",
    "    x = torch.randn(100)\n",
    "    a = torch.randn(512, 100)\n",
    "    y = a @ x\n",
    "    mean += y.mean().item()\n",
    "    sqr  += y.pow(2).mean().item()\n",
    "mean/100,sqr/100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "a20f5dbb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.01873561680316925, 98.50323196411132)"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean,sqr = 0.,0.\n",
    "for i in range(100):\n",
    "    x = torch.randn(100)\n",
    "    a = torch.randn(512, 100)\n",
    "    y = a@x\n",
    "    mean += y.mean().item()\n",
    "    sqr += y.pow(2).mean().item()\n",
    "mean/100, sqr/100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "60d54778",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.0029166091486780487, 0.9641124381211379)"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean,sqr = 0.,0.\n",
    "for i in range(10000):\n",
    "    x = torch.randn(1)\n",
    "    a = torch.randn(1)\n",
    "    y = a*x\n",
    "    mean += y.item()\n",
    "    sqr  += y.pow(2).item()\n",
    "mean/10000,sqr/10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "20321776",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.006341835848118399, 0.9670273849413336)"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean,sqr = 0.,0.\n",
    "for i in range(10000):\n",
    "    x = torch.randn(1)\n",
    "    a = torch.randn(1)\n",
    "    y = a*x\n",
    "    mean += y.item()\n",
    "    sqr += y.pow(2).item()\n",
    "\n",
    "mean/10000,sqr/10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "ebc5baee",
   "metadata": {},
   "outputs": [],
   "source": [
    "t = torch.tensor([1., 2, 4, 18])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "63f615e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(6.2500)"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = t.mean(); m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "47109d7a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.)"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(t-m).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "3bd3f543",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(47.1875)"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(t-m).pow(2).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "2dd68411",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(5.8750)"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(t-m).abs().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "2dd16825",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(6.8693)"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(t-m).pow(2).mean().sqrt()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "id": "861abb16",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(47.1875), tensor(47.1875))"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(t-m).pow(2).mean(), t.pow(2).mean() - m.pow(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "eef41738",
   "metadata": {},
   "outputs": [],
   "source": [
    "u = t*2\n",
    "u *= torch.rand_like(t)/10+0.95"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "cc654fd9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 2.0936,  4.1910,  8.3615, 37.6709])"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "afa3e0d6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 1.,  2.,  4., 18.])"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "e24366a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x162805110>"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAP6ElEQVR4nO3df6jd9X3H8edrMaVZK4vOq4s/WFaRsK7QKBdxcyuuttXKqDroqIwuMCEtVKjQybSFzv7XLrWFjeGIKM2K6+pojFLsrEiLFFq7q0YTiZltsZ1Jlty2pCoLncb3/rjfW66353jPvTk/7sc8H3A43/P5fr73+84nX1733M/3e74nVYUkqT2/MekCJEkrY4BLUqMMcElqlAEuSY0ywCWpUaeMc2dnnHFGbdy4cZy7lKTmPfbYYz+tqqnF7WMN8I0bNzIzMzPOXUpS85L8uFe7UyiS1CgDXJIaZYBLUqMMcElqlAEuSY0a61UoknSy2fXEAbY9uJ+DR49x9vp13HTFJq658Jyh/GwDXJJGZNcTB7hl5x6OvXwcgANHj3HLzj0AQwlxp1AkaUS2Pbj/V+E979jLx9n24P6h/HwDXJJG5ODRY8tqXy4DXJJG5Oz165bVvlwGuCSNyE1XbGLd2jWvaVu3dg03XbFpKD/fk5iSNCLzJyq9CkWSGnTNhecMLbAXcwpFkhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGLRngSd6c5PtJnkzydJLPdO23JjmQZHf3uGr05UqS5g3yQZ5fAu+uqpeSrAW+k+Qb3bovVtXnR1eeJKmfJQO8qgp4qXu5tnvUKIuSJC1toDnwJGuS7AaOAA9V1aPdqhuSPJXkriSn9dl2a5KZJDOzs7PDqVqSNFiAV9XxqtoMnAtcnOQdwO3A+cBm4BBwW59tt1fVdFVNT01NDaVoSdIyr0KpqqPAt4Erq+pwF+yvAncAFw+/PElSP4NchTKVZH23vA54D/BMkg0Lul0L7B1JhZKknga5CmUDsCPJGuYC/56q+nqSLyfZzNwJzeeAj4ysSknSrxnkKpSngAt7tH94JBVJkgbiJzElqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRg3yrfRvTvL9JE8meTrJZ7r205M8lOTZ7vm00ZcrSZo3yDvwXwLvrqp3ApuBK5NcAtwMPFxVFwAPd68lSWOyZIDXnJe6l2u7RwFXAzu69h3ANaMoUJLU20Bz4EnWJNkNHAEeqqpHgbOq6hBA93xmn223JplJMjM7OzuksiVJAwV4VR2vqs3AucDFSd4x6A6qantVTVfV9NTU1ArLlCQttqyrUKrqKPBt4ErgcJINAN3zkWEXJ0nqb5CrUKaSrO+W1wHvAZ4B7ge2dN22APeNqEZJUg+nDNBnA7AjyRrmAv+eqvp6ku8C9yS5HvgJ8MER1ilJWmTJAK+qp4ALe7T/DLh8FEVJkpbmJzElqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDVqkC81Pi/Jt5LsS/J0ko937bcmOZBkd/e4avTlSpLmDfKlxq8An6iqx5OcCjyW5KFu3Rer6vOjK0+S1M8gX2p8CDjULb+YZB9wzqgLkyS9vmXNgSfZyNw31D/aNd2Q5KkkdyU5bdjFSZL6GzjAk7wV+BpwY1W9ANwOnA9sZu4d+m19ttuaZCbJzOzs7IlXLEkCBgzwJGuZC++7q2onQFUdrqrjVfUqcAdwca9tq2p7VU1X1fTU1NSw6pakk94gV6EEuBPYV1VfWNC+YUG3a4G9wy9PktTPIFehXAp8GNiTZHfX9knguiSbgQKeAz4ygvokSX0MchXKd4D0WPXA8MuRJA3KT2JKUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjRrkW+nPS/KtJPuSPJ3k41376UkeSvJs93za6MuVJM0b5B34K8Anqur3gUuAjyV5O3Az8HBVXQA83L2WJI3JkgFeVYeq6vFu+UVgH3AOcDWwo+u2A7hmRDVKknpY1hx4ko3AhcCjwFlVdQjmQh44s882W5PMJJmZnZ09wXIlSfMGDvAkbwW+BtxYVS8Mul1Vba+q6aqanpqaWkmNkqQeBgrwJGuZC++7q2pn13w4yYZu/QbgyGhKlCT1MshVKAHuBPZV1RcWrLof2NItbwHuG355kqR+Thmgz6XAh4E9SXZ3bZ8EPgvck+R64CfAB0dSoSSppyUDvKq+A6TP6suHW44kaVB+ElOSGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElq1CDfSn9XkiNJ9i5ouzXJgSS7u8dVoy1TkrTYIO/AvwRc2aP9i1W1uXs8MNyyJElLWTLAq+oR4OdjqEWStAwnMgd+Q5KnuimW0/p1SrI1yUySmdnZ2RPYnSRpoZUG+O3A+cBm4BBwW7+OVbW9qqaranpqamqFu5MkLbaiAK+qw1V1vKpeBe4ALh5uWZKkpawowJNsWPDyWmBvv76SpNE4ZakOSb4CXAackeR54O+Ay5JsBgp4DvjI6EqUJPWyZIBX1XU9mu8cQS2SpGXwk5iS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSo5YM8CR3JTmSZO+CttOTPJTk2e75tNGWKUlabJB34F8CrlzUdjPwcFVdADzcvZYkjdGSAV5VjwA/X9R8NbCjW94BXDPcsiRJS1npHPhZVXUIoHs+s1/HJFuTzCSZmZ2dXeHuJEmLjfwkZlVtr6rpqpqempoa9e4k6aSx0gA/nGQDQPd8ZHglSZIGsdIAvx/Y0i1vAe4bTjmSpEENchnhV4DvApuSPJ/keuCzwHuTPAu8t3stSRqjU5bqUFXX9Vl1+ZBrkSQtw5IBrtHY9cQBtj24n4NHj3H2+nXcdMUmrrnwnEmXJakhBvgE7HriALfs3MOxl48DcODoMW7ZuQfAEJc0MO+FMgHbHtz/q/Ced+zl42x7cP+EKpLUIgN8Ag4ePbasdknqxQCfgLPXr1tWuyT1YoBPwE1XbGLd2jWvaVu3dg03XbFpQhVJapEnMSdg/kSlV6FIOhEG+IRcc+E5BrakE+IUiiQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1KgTuhdKkueAF4HjwCtVNT2MoiRJSxvGzaz+tKp+OoSfI0laBqdQJKlRJxrgBXwzyWNJtvbqkGRrkpkkM7Ozsye4O0nSvBOdQrm0qg4mORN4KMkzVfXIwg5VtR3YDjA9PV0nuL8V2/XEAb9AQdIbygm9A6+qg93zEeBe4OJhFDVsu544wC0793Dg6DEKOHD0GLfs3MOuJw5MujRJWrEVB3iStyQ5dX4ZeB+wd1iFDdO2B/dz7OXjr2k79vJxtj24f0IVSdKJO5EplLOAe5PM/5x/rar/GEpVQ3bw6LFltUtSC1Yc4FX1I+CdQ6xlZM5ev44DPcL67PXrJlCNJA3HSXEZ4U1XbGLd2jWvaVu3dg03XbFpQhVJ0ok7Kb6Vfv5qE69CkfRGclIEOMyFuIEt6Y3kpJhCkaQ3IgNckhplgEtSowxwSWrUqj+J6T1MJKm3VR3g8/cwmf8Y/Pw9TABDXNJJb1VPoXgPE0nqb1UHuPcwkaT+VnWA97tXifcwkaRVHuDew0SS+lvVJzG9h4kk9beqAxy8h4kk9bOqp1AkSf0Z4JLUKANckhplgEtSowxwSWpUqmp8O0tmgR+PbYcrdwbw00kXsUzWPHqt1QvWPC6jrvl3q2pqceNYA7wVSWaqanrSdSyHNY9ea/WCNY/LpGp2CkWSGmWAS1KjDPDetk+6gBWw5tFrrV6w5nGZSM3OgUtSo3wHLkmNMsAlqVEnbYAnOS/Jt5LsS/J0ko/36HNZkl8k2d09Pj2JWhfV9FySPV09Mz3WJ8k/JPlBkqeSXDSJOrtaNi0Yu91JXkhy46I+Ex/jJHclOZJk74K205M8lOTZ7vm0PttemWR/N943T7jmbUme6f7f702yvs+2r3sMjbnmW5McWPD/f1WfbVfTOH91Qb3PJdndZ9vRj3NVnZQPYANwUbd8KvBfwNsX9bkM+Pqka11U03PAGa+z/irgG0CAS4BHJ11zV9ca4H+Y+0DCqhpj4F3ARcDeBW1/D9zcLd8MfK7Pv+mHwNuANwFPLj6Gxlzz+4BTuuXP9ap5kGNozDXfCvzNAMfOqhnnRetvAz49qXE+ad+BV9Whqnq8W34R2Ae8EW48fjXwLzXne8D6JBsmXRRwOfDDqlp1n8StqkeAny9qvhrY0S3vAK7psenFwA+q6kdV9X/Av3XbjVyvmqvqm1X1Svfye8C546hlUH3GeRCrapznJQnwF8BXxlFLLydtgC+UZCNwIfBoj9V/mOTJJN9I8gfjraynAr6Z5LEkW3usPwf47wWvn2d1/GL6EP0P9NU2xgBnVdUhmPtlD5zZo89qHWuAv2buL7FeljqGxu2Gbtrnrj5TVat1nP8EOFxVz/ZZP/JxPukDPMlbga8BN1bVC4tWP87cn/zvBP4R2DXm8nq5tKouAt4PfCzJuxatT49tJnqtaJI3AR8A/r3H6tU4xoNadWMNkORTwCvA3X26LHUMjdPtwPnAZuAQc1MSi63KcQau4/XffY98nE/qAE+ylrnwvruqdi5eX1UvVNVL3fIDwNokZ4y5zMU1HeyejwD3Mvfn5ULPA+cteH0ucHA81fX1fuDxqjq8eMVqHOPO4fmpp+75SI8+q26sk2wB/gz4y+omYhcb4Bgam6o6XFXHq+pV4I4+tazGcT4F+HPgq/36jGOcT9oA7+av7gT2VdUX+vT5na4fSS5mbrx+Nr4qf62etyQ5dX6ZuZNWexd1ux/4q+5qlEuAX8xPBUxQ33cqq22MF7gf2NItbwHu69HnP4ELkvxe91fGh7rtJiLJlcDfAh+oqv/t02eQY2hsFp2fubZPLatqnDvvAZ6pqud7rRzbOI/jTO5qfAB/zNyfYU8Bu7vHVcBHgY92fW4AnmburPf3gD+acM1v62p5sqvrU137wpoD/BNzZ+33ANMTrvk3mQvk31rQtqrGmLlfLoeAl5l7t3c98NvAw8Cz3fPpXd+zgQcWbHsVc1cw/XD+/2OCNf+Aubni+eP5nxfX3O8YmmDNX+6O06eYC+UNq32cu/YvzR/DC/qOfZz9KL0kNeqknUKRpNYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalR/w9binmHtJOjkgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(t, u)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "cdd666de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 57.6745,  37.7749,  10.6150, 288.9516])"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prod = (t-t.mean())*(u-u.mean());prod"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "870a55a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(98.7540)"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prod.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "a88f6715",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x1627c89d0>"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAARh0lEQVR4nO3dbYyc13ne8f/VJQXQiWvGIZNYJBsxqUpXAWxLnbJJmxcXjkNKSEI5DVrKQe3mBQKLsEg+lDCFAEYAf3KJFkUTpQSbCk4LI0wK0zST0KWDIImLpgq4tChRlLz2mnGsJV1rbZd27RKQSN/9sENhNJrZnZWGM7uH/x+w2HnOuWfm1uHg2mfPzLNKVSFJWv/+xrQbkCSNh4EuSY0w0CWpEQa6JDXCQJekRmyY1hNv2bKl7rrrrmk9vSStS+fOnftyVW0dNDe1QL/rrruYnZ2d1tNL0rqU5K+HzbnlIkmNMNAlqREGuiQ1wkCXpEYY6JLUiKl9ymWaTj5xmSNn5rhy9Rp3bt7EoT27ePDebdNuS5Jek9su0E8+cZlHTlzg2os3ALh89RqPnLgAYKhLWtduuy2XI2fmXgrzm669eIMjZ+am1JEkjcdtF+hXrl5b1bgkrRe3XaDfuXnTqsYlab0YKdCT7E0yl2Q+yeEB829I8gdJnkxyMcnPj7/V8Ti0ZxebNs68bGzTxhkO7dk1pY4kaTxWfFM0yQzwKPBOYAE4m+RUVT3TU/bLwDNV9VNJtgJzST5cVS/ckq5fg5tvfPopF0mtGeVTLruB+aq6BJDkOLAP6A30Al6fJMC3A18Fro+517F58N5tBrik5oyy5bINeK7neKE71us3gb8LXAEuAL9SVd/qf6AkDyeZTTK7uLj4KluWJA0ySqBnwFj1He8BzgN3Am8DfjPJ33zFnaqOVVWnqjpbtw78c76SpFdplEBfAHb0HG9n6Uy8188DJ2rJPPBXwJvH06IkaRSjBPpZ4O4kO5PcAewHTvXVfAF4B0CS7wZ2AZfG2agkaXkrvilaVdeTHATOADPAY1V1McmB7vxR4APAh5JcYGmL5n1V9eVb2Lckqc9If8ulqk4Dp/vGjvbcvgL8xHhbkyStxm13pagktcpAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhoxUqAn2ZtkLsl8ksMD5g8lOd/9ejrJjSRvHH+7kqRhVgz0JDPAo8D9wD3AQ0nu6a2pqiNV9baqehvwCPDnVfXVW9CvJGmIUc7QdwPzVXWpql4AjgP7lql/CPjdcTQnSRrdKIG+DXiu53ihO/YKSV4H7AU+MmT+4SSzSWYXFxdX26skaRmjBHoGjNWQ2p8C/uew7ZaqOlZVnarqbN26ddQeJUkjGCXQF4AdPcfbgStDavfjdoskTcUogX4WuDvJziR3sBTap/qLkrwB+DHgY+NtUZI0ig0rFVTV9SQHgTPADPBYVV1McqA7f7Rb+i7gE1X1zVvWrSRpqFQN2w6/tTqdTs3Ozk7luSVpvUpyrqo6g+a8UlSSGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqxIr/k+i15OQTlzlyZo4rV69x5+ZNHNqziwfv3TbttiRpTVg3gX7yics8cuIC1168AcDlq9d45MQFAENdkhhxyyXJ3iRzSeaTHB5S8/Yk55NcTPLn420TjpyZeynMb7r24g2OnJkb91NJ0rq04hl6khngUeCdwAJwNsmpqnqmp2Yz8FvA3qr6QpLvGnejV65eW9W4JN1uRjlD3w3MV9WlqnoBOA7s66t5N3Ciqr4AUFXPj7dNuHPzplWNS9LtZpRA3wY813O80B3r9XeA70jyZ0nOJXnPoAdK8nCS2SSzi4uLq2r00J5dbNo487KxTRtnOLRn16oeR5JaNcqbohkwVgMe5+8B7wA2Af8ryeNV9ZmX3anqGHAMoNPp9D/Gsm6+8emnXCRpsFECfQHY0XO8HbgyoObLVfVN4JtJPgm8FfgMY/TgvdsMcEkaYpQtl7PA3Ul2JrkD2A+c6qv5GPAjSTYkeR3wD4Bnx9uqJGk5K56hV9X1JAeBM8AM8FhVXUxyoDt/tKqeTfLfgaeAbwG/XVVP38rGJUkvl6pVbWWPTafTqdnZ2ak8tyStV0nOVVVn0Jx/y0WSGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpESMFepK9SeaSzCc5PGD+7Um+luR89+v9429VkrScDSsVJJkBHgXeCSwAZ5Ocqqpn+kr/R1X95C3oUZI0glHO0HcD81V1qapeAI4D+25tW5Kk1Rol0LcBz/UcL3TH+v1QkieTfDzJDwx6oCQPJ5lNMru4uPgq2pUkDTNKoGfAWPUdfwr43qp6K/AbwMlBD1RVx6qqU1WdrVu3rqpRSdLyRgn0BWBHz/F24EpvQVV9vaq+0b19GtiYZMvYupQkrWiUQD8L3J1kZ5I7gP3Aqd6CJN+TJN3bu7uP+5VxNytJGm7FT7lU1fUkB4EzwAzwWFVdTHKgO38U+FngXya5DlwD9ldV/7aMJOkWyrRyt9Pp1Ozs7FSeW5LWqyTnqqozaM4rRSWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1IiRAj3J3iRzSeaTHF6m7u8nuZHkZ8fXoiRpFCsGepIZ4FHgfuAe4KEk9wyp+yBwZtxNSpJWNsoZ+m5gvqouVdULwHFg34C6fwV8BHh+jP1JkkY0SqBvA57rOV7ojr0kyTbgXcDR8bUmSVqNUQI9A8aq7/jfA++rqhvLPlDycJLZJLOLi4sjtihJGsWGEWoWgB09x9uBK301HeB4EoAtwANJrlfVyd6iqjoGHAPodDr9PxQkSa/BKIF+Frg7yU7gMrAfeHdvQVXtvHk7yYeAP+wPc0nSrbVioFfV9SQHWfr0ygzwWFVdTHKgO+++uSStAaOcoVNVp4HTfWMDg7yq/sVrb0uStFpeKSpJjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUiJECPcneJHNJ5pMcHjC/L8lTSc4nmU3yw+NvVZK0nA0rFSSZAR4F3gksAGeTnKqqZ3rK/gQ4VVWV5C3A7wNvvhUNS5IGG+UMfTcwX1WXquoF4Diwr7egqr5RVdU9/DagkCRN1CiBvg14rud4oTv2MkneleTTwB8BvzDogZI83N2SmV1cXHw1/UqShhgl0DNg7BVn4FX10ap6M/Ag8IFBD1RVx6qqU1WdrVu3rqpRSdLyRgn0BWBHz/F24Mqw4qr6JPD9Sba8xt4kSaswSqCfBe5OsjPJHcB+4FRvQZK/nSTd2/cBdwBfGXezkqThVvyUS1VdT3IQOAPMAI9V1cUkB7rzR4F/ArwnyYvANeCf9bxJKkmagEwrdzudTs3Ozk7luSVpvUpyrqo6g+a8UlSSGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpESMFepK9SeaSzCc5PGD+55I81f36iyRvHX+rkqTlrBjoSWaAR4H7gXuAh5Lc01f2V8CPVdVbgA8Ax8bdqCRpeaOcoe8G5qvqUlW9ABwH9vUWVNVfVNX/6R4+Dmwfb5uSpJWMEujbgOd6jhe6Y8P8IvDxQRNJHk4ym2R2cXFx9C4lSSsaJdAzYKwGFib/mKVAf9+g+ao6VlWdqups3bp19C4lSSvaMELNArCj53g7cKW/KMlbgN8G7q+qr4ynvdvDyScuc+TMHFeuXuPOzZs4tGcXD9673C9BkvRKowT6WeDuJDuBy8B+4N29BUn+FnAC+OdV9Zmxd9mwk09c5pETF7j24g0ALl+9xiMnLgAY6pJWZcUtl6q6DhwEzgDPAr9fVReTHEhyoFv2fuA7gd9Kcj7J7C3ruDFHzsy9FOY3XXvxBkfOzE2pI0nr1Shn6FTVaeB039jRntu/BPzSeFu7PVy5em1V45I0jFeKTtmdmzetalyShjHQp+zQnl1s2jjzsrFNG2c4tGfXlDqStF6NtOWiW+fmG59+ykXSa2WgrwEP3rvNAJf0mrnlIkmNMNAlqRFuuUjShNzqq8INdEmagElcFe6WiyRNwCSuCjfQJWkCJnFVuIEuSRMwiavCDXRJmoBJXBXum6KSNAGTuCrcQJekCbnVV4W75SJJjTDQJakRBrokNcJAl6RGGOiS1IhU1XSeOFkE/noqT746W4AvT7uJVbLnyVhvPa+3fsGeB/neqto6aGJqgb5eJJmtqs60+1gNe56M9dbzeusX7Hm13HKRpEYY6JLUCAN9Zcem3cCrYM+Tsd56Xm/9gj2vinvoktQIz9AlqREGuiQ1wkAHkuxI8qdJnk1yMcmvDKh5e5KvJTnf/Xr/NHrt6+nzSS50+5kdMJ8k/yHJfJKnktw3jT57+tnVs37nk3w9ya/21Ux1nZM8luT5JE/3jL0xyR8n+Wz3+3cMue/eJHPd9T485Z6PJPl099/9o0k2D7nvsq+hCff860ku9/zbPzDkvmtpnX+vp9/PJzk/5L6TWeequu2/gDcB93Vvvx74DHBPX83bgT+cdq99PX0e2LLM/APAx4EAPwj85bR77ultBvjfLF0ksWbWGfhR4D7g6Z6xfwMc7t4+DHxwyH/P54DvA+4Anux/DU24558ANnRvf3BQz6O8hibc868D/3qE182aWee++X8LvH+a6+wZOlBVX6yqT3Vv/1/gWeDW/dHiydkH/Jda8jiwOcmbpt1U1zuAz1XVmrpauKo+CXy1b3gf8Dvd278DPDjgrruB+aq6VFUvAMe797vlBvVcVZ+oquvdw8eB7ZPoZVRD1nkUa2qdb0oS4J8CvzuJXoYx0PskuQu4F/jLAdM/lOTJJB9P8gOT7WygAj6R5FyShwfMbwOe6zleYO38oNrP8Bf/Wlvn766qL8LSD3/guwbUrOW1/gWWflMbZKXX0KQd7G4TPTZka2utrvOPAF+qqs8OmZ/IOhvoPZJ8O/AR4Fer6ut9059iaXvgrcBvACcn3N4g/6iq7gPuB345yY/2zWfAfab+OdUkdwA/Dfy3AdNrcZ1HsVbX+teA68CHh5Ss9BqapP8IfD/wNuCLLG1h9FuT6ww8xPJn5xNZZwO9K8lGlsL8w1V1on++qr5eVd/o3j4NbEyyZcJt9vd0pfv9eeCjLP062msB2NFzvB24MpnulnU/8Kmq+lL/xFpcZ+BLN7equt+fH1Cz5tY6yXuBnwR+rrobuf1GeA1NTFV9qapuVNW3gP80pJe1uM4bgJ8Bfm9YzaTW2UDnpf2v/ww8W1X/bkjN93TrSLKbpbX7yuS6fEU/35bk9Tdvs/Qm2NN9ZaeA93Q/7fKDwNdubh1M2dCzmbW2zl2ngPd2b78X+NiAmrPA3Ul2dn8D2d+931Qk2Qu8D/jpqvp/Q2pGeQ1NTN/7O+8a0suaWueuHwc+XVULgyYnus6TeHd4rX8BP8zSr21PAee7Xw8AB4AD3ZqDwEWW3lV/HPiHU+75+7q9PNnt69e64709B3iUpU8FXAA6a2CtX8dSQL+hZ2zNrDNLP2i+CLzI0tngLwLfCfwJ8Nnu9zd2a+8ETvfc9wGWPiH1uZv/HlPseZ6lveabr+ej/T0Pew1Nsef/2n2dPsVSSL9pra9zd/xDN1+/PbVTWWcv/ZekRrjlIkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSI/4/ffQxYCdHx+MAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "v = torch.rand_like(t)\n",
    "plt.scatter(t, v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "7f1c3b0f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-1.3247)"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "((t-t.mean())*(v-v.mean())).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "588be398",
   "metadata": {},
   "outputs": [],
   "source": [
    "cov = (t*v).mean() - (t.mean()*v.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "69034bd6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-1.3247)"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cov"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "da4fb9cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-0.5116)"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cov/(t.std()*v.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "0193e07e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(9.9558)"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 766,
   "id": "ea28de47",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([200, 100])\n",
      "mean:-0.01,std:1.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.1\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.01\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:-0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n",
      "torch.Size([200, 100])\n",
      "mean:0.0,std:0.0\n",
      "--------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0.]])"
      ]
     },
     "execution_count": 766,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.randn(200, 100)\n",
    "print(x.shape)\n",
    "print(f'mean:{round(x.mean().item(),2)},std:{round(x.std().item(), 2)}')\n",
    "print('--------------')\n",
    "for i in range(50): \n",
    "    x = x @ (torch.randn(100,100)*(1/100))\n",
    "    print(x.shape)\n",
    "    print(f'mean:{round(x.mean().item(),2)},std:{round(x.std().item(), 2)}')\n",
    "    print('--------------')\n",
    "#     break\n",
    "x[0:5,0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 798,
   "id": "76d4d138",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean:-0.04,std:1.01\n",
      "mean:0.0,std:0.07\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0462, -0.0123, -0.0799, -0.0465,  0.1042]])"
      ]
     },
     "execution_count": 798,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.randn(1, 1000)\n",
    "print(f'mean:{round(x.mean().item(),2)},std:{round(x.std().item(), 2)}')\n",
    "for i in range(500*10):\n",
    "    w = torch.randn(1000,1000) *(1/np.sqrt(1000))\n",
    "#     print(f'wmean:{round(w.mean().item(),2)},wstd:{round(w.std().item(), 2)}') \n",
    "    \n",
    "    x = x@w\n",
    "#     print(f'mean:{round(x.mean().item(),2)},std:{round(x.std().item(), 2)}') \n",
    "#     break\n",
    "print(f'mean:{round(x.mean().item(),2)},std:{round(x.std().item(), 2)}')\n",
    "\n",
    "x[0:5,0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 802,
   "id": "4fbc942c",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.randn(100) + 0\n",
    "y = torch.randn(100) * (1/np.sqrt(100)) + 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 803,
   "id": "ba5f0720",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.0315), tensor(1.0038))"
      ]
     },
     "execution_count": 803,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.mean(),x.var()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 804,
   "id": "8984a6f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(-0.0038), tensor(0.0121))"
      ]
     },
     "execution_count": 804,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.mean(),y.var()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 816,
   "id": "fa4d46f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(-0.0012), tensor(0.9913))"
      ]
     },
     "execution_count": 816,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from math import sqrt\n",
    "\n",
    "x = torch.randn(200, 100)\n",
    "y = torch.randn(200)\n",
    "w1 = torch.randn(100,50)/sqrt(100)\n",
    "b1 = torch.zeros(50)\n",
    "w2 = torch.randn(50, 1)/sqrt(50)\n",
    "b2 = torch.zeros(1)\n",
    "\n",
    "def lin(x, w, b):\n",
    "    return x@w+b\n",
    "\n",
    "def relu(x):\n",
    "    return x.clamp_min(0.)\n",
    "\n",
    "l1 = lin(x, w1, b1)\n",
    "l1.mean(),l1.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 821,
   "id": "7848999c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.3940), tensor(0.5806))"
      ]
     },
     "execution_count": 821,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l2 = relu(l1)\n",
    "l2.mean(),l2.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 824,
   "id": "2d6bf13c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0000e+00, 0.0000e+00, 0.0000e+00, 1.1744e-09, 0.0000e+00],\n",
       "        [0.0000e+00, 0.0000e+00, 0.0000e+00, 1.0455e-09, 0.0000e+00],\n",
       "        [0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00],\n",
       "        [0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00],\n",
       "        [0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00]])"
      ]
     },
     "execution_count": 824,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.randn(200, 100)\n",
    "for i in range(50):\n",
    "    x = relu(x@(torch.randn(100, 100)*0.1))\n",
    "x[0:5,0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 826,
   "id": "e0881d71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0000, 0.6540, 0.0000, 0.0000, 0.0000],\n",
       "        [0.0000, 1.0987, 0.0000, 0.0000, 0.0000],\n",
       "        [0.0000, 0.4435, 0.0000, 0.0000, 0.0000],\n",
       "        [0.0000, 0.8919, 0.0000, 0.0000, 0.0000],\n",
       "        [0.0000, 0.9530, 0.0000, 0.0000, 0.0000]])"
      ]
     },
     "execution_count": 826,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.randn(200, 100)\n",
    "for i in range(50):\n",
    "    x = relu(x@(torch.randn(100,100)*sqrt(2/100)))\n",
    "x[0:5,0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 827,
   "id": "e0b0a713",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.1740), tensor(0.2954))"
      ]
     },
     "execution_count": 827,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.mean(),x.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f06e195",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python_main",
   "language": "python",
   "name": "python_main"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}