Inicializacion.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "yr2uMIEbftg5"
      },
      "source": [
        "import torch"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5pJRVGj3f8u7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0a664b9b-2197-4a8d-d0d3-0606ad78e47b"
      },
      "source": [
        "#Sea x una entrada de dimensión 512 con distribución N(0,1)\n",
        "# Media: 0\n",
        "# Varianza: 1\n",
        "\n",
        "x = torch.randn(512) # esto es Xo, el resultado es X1 = Xo w1 + b1\n",
        "\n",
        "#Simular la pasada forward de la entrada con 100 capas lineales\n",
        "#Las capas tienen 512 neuronas cada una\n",
        "\n",
        "for i in range(100):\n",
        "    w = torch.randn(512,512)\n",
        "    x = w @ x # matrix multiplication dot(a, b)\n",
        "x.mean(), x.std()\n",
        "\n",
        "# 100 capas de dim 512, por ende w1 tiene dim 512x512 y b1 tiene dim 512x512, y así hasta w100 y b100\n",
        "\n",
        "#Media y desviación del resultado explotan -> nan (productos con N(0,1))"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(nan), tensor(nan))"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aiLqD4PegAjx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "66728d2b-913c-40ff-c503-a5a129602308"
      },
      "source": [
        "# En qué capa sucede la explosión?\n",
        "\n",
        "x = torch.randn(512)\n",
        "\n",
        "for i in range(100):\n",
        "    a = torch.randn(512,512)\n",
        "    x = a @ x\n",
        "    if torch.isnan(x.std()): break\n",
        "\n",
        "print(i)\n",
        "\n",
        "#Entrada es pequeña, la única razón para la explosión es que los pesos son muy grandes"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "27\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BE14vqoigD_R",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "795815cb-a674-4561-ea48-0779cf15c40d"
      },
      "source": [
        "#Podemos vernos tentados a reducir los pesos para evitar la explosión\n",
        "# Escalamos los pesos por algún factor\n",
        "\n",
        "x = torch.randn(512)\n",
        "\n",
        "for i in range(100):\n",
        "    a = torch.randn(512,512) * 0.01\n",
        "    x = a @ x\n",
        "x.mean(), x.std()\n",
        "\n",
        "# Ahora la media y la desviación se fueron a cero"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(0.), tensor(0.))"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LUC7c8FDgKjh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e88e1f77-d007-48b3-8a52-0df3315c3f86"
      },
      "source": [
        "# ¿Cuál es el promedio y desviación estándar de multiplicar un vector de 512 dimensiones y una matriz 512x512?\n",
        "# Ambos en N(0,1)\n",
        "\n",
        "#Ejecutamos 10000 multiplicaciones, y promediamos los resultados\n",
        "\n",
        "import math\n",
        "\n",
        "mean, var = 0.,0.\n",
        "for i in range(10000):\n",
        "    x = torch.randn(512)\n",
        "    a = torch.randn(512,512)\n",
        "    y = a @ x\n",
        "    mean += y.mean().item()\n",
        "    var += y.pow(2).mean().item()\n",
        "mean/10000, math.sqrt(var/10000)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(-0.013727493983134628, 22.643891870461175)"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jFH1y85WgVU4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "04f78f7c-d815-4e13-a6bc-50b5da3a16ca"
      },
      "source": [
        "#La desviación es muy similar a la raiz cuadrada de la dimension del vector de entrada\n",
        "\n",
        "math.sqrt(512) # la desviacion es proporcional a la raiz cuadrada de la dimension del vector de entrada"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "22.627416997969522"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t-GC3Tw8goRb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5588985f-a5de-4956-d76e-6f17ba6ae42b"
      },
      "source": [
        "#El producto de dos números en distribucipón N(0,1) es siempre un número en la misma distribución\n",
        "\n",
        "mean, var = 0.,0.\n",
        "for i in range(10000):\n",
        "    x = torch.randn(1)\n",
        "    a = torch.randn(1)\n",
        "    y = a*x\n",
        "    mean += y.item()\n",
        "    var += y.pow(2).item()\n",
        "mean/10000, math.sqrt(var/10000)\n"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(-0.013396202174478936, 1.0040449287803521)"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BKc_2u8agv7H",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "471b554a-d39b-443d-b708-c3cb4aaf804a"
      },
      "source": [
        "#La varianza promedio debe estar en el orden de 1/512\n",
        "mean, var = 0.,0.\n",
        "for i in range(10000):\n",
        "    x = torch.randn(1)\n",
        "    a = torch.randn(1)*math.sqrt(1./512)\n",
        "    y = a*x\n",
        "    mean += y.item()\n",
        "    var += y.pow(2).item()\n",
        "mean/10000, var/10000"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(-0.00015903993283615138, 0.0019520228149370736)"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PWCbW4LbC7dW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f2e6ec53-5a29-41bb-f7da-1ef514742a82"
      },
      "source": [
        "1/512"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.001953125"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lu6by5wRgzPa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "318eb5d7-318c-48a5-9665-d09266279d4e"
      },
      "source": [
        "#Así que deberíamos usar sqrt(1/512) para escalar los pesos\n",
        "\n",
        "mean, var = 0.,0.\n",
        "for i in range(10000):\n",
        "    x = torch.randn(512)\n",
        "    a = torch.randn(512,512) * math.sqrt(1./512) # antes era 0.01\n",
        "    y = a @ x\n",
        "    mean += y.mean().item()\n",
        "    var += y.pow(2).mean().item()\n",
        "mean/10000, math.sqrt(var/10000)  # se mantienen acotado, ya no son nan o 0.0"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(-0.0004002824828145094, 1.0005287594472172)"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jgzWvKxyhA6P",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6cb18239-32b0-4956-fc81-c9a4dd027fc9"
      },
      "source": [
        "#Probemos en nuestra red neuronal simulada\n",
        "\n",
        "x = torch.randn(512)\n",
        "\n",
        "for i in range(100):\n",
        "    a = torch.randn(512,512) * math.sqrt(1./512)\n",
        "    x = a @ x\n",
        "x.mean(), x.std()\n",
        "\n",
        "#Las salidas no explotan ni se desvanecen"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(0.0508), tensor(0.5467))"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yOI_874HhQuu"
      },
      "source": [
        "#Hasta ahora no hemos utilizado funciones de activación.\n",
        "#Veamos que pasa si aplicamos una función de activación\n",
        "#TANH a nuestro modelo basico de red neuronal\n",
        "\n",
        "def tanh(x): return torch.tanh(x)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e60Jv62FEEpM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "401c435c-121b-4113-cbdd-bf75988126e7"
      },
      "source": [
        "x = torch.randn(512)\n",
        "\n",
        "for i in range(100):\n",
        "    a = torch.randn(512,512) * math.sqrt(1./512)\n",
        "    x = tanh(a @ x)\n",
        "x.mean(), x.std()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(0.0004), tensor(0.0535))"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zt53e08kEH7o",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f4107f55-f32f-45a2-ebfc-495814a6b6f0"
      },
      "source": [
        "x = torch.randn(512)\n",
        "\n",
        "for i in range(100):\n",
        "    a = torch.Tensor(512,512).uniform_(-1, 1) * math.sqrt(1./512)\n",
        "    x = tanh(a @ x)\n",
        "x.mean(), x.std()"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(2.1927e-26), tensor(9.6142e-25))"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "unBp6GCeEoTP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9af8ff5e-b5d9-47e9-a797-efaea9790daa"
      },
      "source": [
        "#Glorot y Bengio propusieron una nueva inicialización\n",
        "def xavier(m,h):\n",
        "    return torch.Tensor(m, h).uniform_(-1, 1)*math.sqrt(6./(m+h))\n",
        "\n",
        "x = torch.randn(512)\n",
        "\n",
        "for i in range(100):\n",
        "    a = xavier(512, 512)\n",
        "    x = tanh(a @ x)\n",
        "x.mean(), x.std()"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(0.0018), tensor(0.0651))"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tttsh6KTFWBH"
      },
      "source": [
        "#Pero que pasa cuando la función de activación es RELU?\n",
        "def relu(x): return x.clamp_min(0.)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pAziFtMWGYJB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5663982e-82ea-4b57-ebf8-c310ebcf1fef"
      },
      "source": [
        "mean, var = 0.,0.\n",
        "for i in range(10000):\n",
        "    x = torch.randn(512)\n",
        "    a = torch.randn(512,512)\n",
        "    y = relu(a @ x)\n",
        "    mean += y.mean().item()\n",
        "    var += y.pow(2).mean().item()\n",
        "mean/10000, math.sqrt(var/10000)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(9.040104362583161, 16.02591488882887)"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2sk9AFqPGau0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e6d7ad5b-f7ae-4b1d-991f-9889ac25f582"
      },
      "source": [
        "#Desviación estándar es cernaca a sqrt(512)/sqrt(2)\n",
        "math.sqrt(512/2)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "16.0"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WU_S41AaHlEo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "82d2c47f-4cde-41d6-9db4-72a4f926f554"
      },
      "source": [
        "mean, var = 0.,0.\n",
        "for i in range(10000):\n",
        "    x = torch.randn(512)\n",
        "    a = torch.randn(512,512)*math.sqrt(2/512)\n",
        "    y = relu(a @ x)\n",
        "    mean += y.mean().item()\n",
        "    var += y.pow(2).mean().item()\n",
        "mean/10000, math.sqrt(var/10000)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0.564910521799326, 1.0013181406520248)"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BXWs2xEaHwP-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0ae3a16f-3b29-41f9-84b4-c168854dd91f"
      },
      "source": [
        "def kaiming(m,h):\n",
        "  return torch.randn(m,h)*math.sqrt(2./m)\n",
        "\n",
        "x = torch.randn(512)\n",
        "\n",
        "for i in range(100):\n",
        "  a = kaiming(512, 512)\n",
        "  x = relu(a @ x)\n",
        "\n",
        "x.mean(), x.std()"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(0.3962), tensor(0.5374))"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lUdXELOeIsNl",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "df537509-1ef5-47ef-ca8c-fa14c59de959"
      },
      "source": [
        "#Xavier con RELU?\n",
        "\n",
        "x = torch.randn(512)\n",
        "\n",
        "for i in range(100):\n",
        "  a = xavier(512, 512)\n",
        "  x = relu(a @ x)\n",
        "\n",
        "x.mean(), x.std()"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(3.4718e-16), tensor(4.9262e-16))"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M3OCG_47JACG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "22af0c18-788a-4018-9c27-a67f9078ebe1"
      },
      "source": [
        "!python -V"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Python 3.10.12\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!date"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SzdEChqv_dCb",
        "outputId": "a9fd6cde-9343-4fba-e862-6886551d93f7"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Thu Aug 31 07:08:42 PM UTC 2023\n"
          ]
        }
      ]
    }
  ]
}