Our_first_neural_network_linear_regression_workset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "H04So6C3IFoJ"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/mgfrantz/CodingNomads-Intro-To-Deep-Learning/blob/master/Fundamentals/Our%20first%20neural%20network%20-%20linear%20regression.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "UVZlK6AjphyK"
   },
   "outputs": [],
   "source": [
    "# Imports\n",
    "import torch\n",
    "from torch import nn, optim\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Zhrag2TVqUut"
   },
   "outputs": [],
   "source": [
    "def seed_all(seed=42):\n",
    "    \"\"\"\n",
    "    Sets the numpy and torch random seed.\n",
    "    \"\"\"\n",
    "    np.random.seed(seed)\n",
    "    torch.random.seed = seed\n",
    "\n",
    "seed_all()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JiTOh6dtWS0i"
   },
   "source": [
    "# Single-Variable Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "3i5JbcrJp8Ng"
   },
   "outputs": [],
   "source": [
    "# Create some X data\n",
    "X = np.random.uniform(0, 10, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "uWXuFmJPqglE"
   },
   "outputs": [],
   "source": [
    "# Define the slope (m), bias (b), and some noise we want to add to X to make y\n",
    "m = 3\n",
    "b = 1.8\n",
    "noise = np.random.normal(scale=3, size=100) # add this so we don't have a perfect line relationship"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "68gEAbSPxqhM"
   },
   "source": [
    "### Exercise 2.1: Create your `y` variable\n",
    "\n",
    "In the cell above, we've created the slope `m`, the intercept `b`, and a noise factor `noise`. \n",
    "In the cell below, use multiplication and addition to create your `y` variable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "fScgFtR8qvMP"
   },
   "outputs": [],
   "source": [
    "# Your code here\n",
    "y = ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6R0i1APRq0SW"
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(10,10))\n",
    "ax.set_xlabel('X')\n",
    "ax.set_ylabel('y')\n",
    "ax.scatter(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "havhKnLZBih0"
   },
   "source": [
    "### Exercise 2.2: Create a function that returns MSE\n",
    "\n",
    "Look at some of the functions in `torch`.\n",
    "Based on what you've learned in the lesson, create a function that returns the mean squared error for a `torch.Tensor` of `predictions` and `actuals`.\n",
    "There is a test case below so you can understand what the inputs and outputs should be."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qMoZ3iRPrS-x"
   },
   "outputs": [],
   "source": [
    "# Define MSE\n",
    "def mse(predictions:torch.Tensor, actuals:torch.Tensor) -> torch.Tensor:\n",
    "    # your code here\n",
    "    raise NotImplementedError(\"Implement MSE, then remove this line\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YXVJjYjQYS-r"
   },
   "outputs": [],
   "source": [
    "ys = torch.tensor([1,2,3])\n",
    "yhats = torch.tensor([1.1, 2.1, 3.1])\n",
    "assert isinstance(mse(ys, yhats), torch.Tensor), \"The output of mse should be a torch.Tensor!\"\n",
    "assert torch.allclose(mse(ys, yhats), torch.tensor(0.01)), \"The MSE should be about 0.01\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 236,
     "status": "ok",
     "timestamp": 1635377052150,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "aqjXsI8dr4ba",
    "outputId": "703939f8-b3d0-46f0-d44a-5e5d88228740"
   },
   "outputs": [],
   "source": [
    "mse(ys, yhats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 3,
     "status": "ok",
     "timestamp": 1635377052345,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "lmtAEb1_YOo6",
    "outputId": "990a8e0f-4044-425f-c05c-cca11189b21d"
   },
   "outputs": [],
   "source": [
    "# Now that we've defined MSE, let's just use Torch's.\n",
    "mse_loss = nn.MSELoss()\n",
    "mse_loss(ys, yhats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 4,
     "status": "ok",
     "timestamp": 1635377052690,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "HROGoKWjY0g1",
    "outputId": "24f41024-9889-4d52-f21d-fbe7ebb5fc2f"
   },
   "outputs": [],
   "source": [
    "# We can also use the functional API to calculate MSE\n",
    "F.mse_loss(ys, yhats)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tncRx13VY8Zr"
   },
   "source": [
    "### Exercise 2.3: Based on your knowledge of `scikit-learn`, perform a linear regression to predict `y` from `X`.\n",
    "\n",
    "Fill in the code to fit a linear regression model and find the coefficient(s) and intercept/bias.\n",
    "Are your slope and intercept in the ballpark of the sope and intercept we defined earlier in the notebook?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_i03QSwHZQFj"
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "OQY_W6WaZTf0"
   },
   "outputs": [],
   "source": [
    "# Fit a linear model on our data\n",
    "lr = ... # Instantiate and fit a vanilla linear regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 271,
     "status": "ok",
     "timestamp": 1635377495843,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "Lp5rB1BWZnyE",
    "outputId": "3491e12e-dd9b-4b15-8e45-e9ca18dcb56d"
   },
   "outputs": [],
   "source": [
    "# Display the slope and intercept\n",
    "slope = ... # Your code here\n",
    "intercept = ... # Your code here\n",
    "\n",
    "assert isinstance(slope, np.ndarray)\n",
    "assert isinstance(intercept, float)\n",
    "\n",
    "print(f'The slope is {str(slope)} and the intercept is {intercept}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PANocYcgGMGa"
   },
   "outputs": [],
   "source": [
    "assert isinstance(intercept, float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 342,
     "status": "ok",
     "timestamp": 1635377656821,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "1PAk2Xt3cAEy",
    "outputId": "72fec334-b7a5-4fa4-f33a-0229dc73be40"
   },
   "outputs": [],
   "source": [
    "# Calculate the mean squared error\n",
    "predictions = ... # Your code here\n",
    "lr_mse = mean_squared_error(y, predictions)\n",
    "\n",
    "print(f'The mean squared error from our linear regression is {lr_mse}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 623
    },
    "executionInfo": {
     "elapsed": 221,
     "status": "ok",
     "timestamp": 1624592622612,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "8eIhs0pjZyem",
    "outputId": "c5aac525-3146-4fc0-d5b6-92d54cd91520"
   },
   "outputs": [],
   "source": [
    "# Plot our line of best fit\n",
    "fig, ax = plt.subplots(figsize=(10,10))\n",
    "ax.set_xlabel('X')\n",
    "ax.set_ylabel('y')\n",
    "ax.scatter(X, y)\n",
    "_x = np.arange(0, 10)\n",
    "_y = _x * lr.coef_[0] + lr.intercept_\n",
    "ax.plot(_x, _y, c='red', label=f\"Line of best fit\")\n",
    "ax.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zOJ2WiAyxGyo"
   },
   "outputs": [],
   "source": [
    "# Because we're in torch now, let's just turn X and y into tensors.\n",
    "X = torch.tensor(X)\n",
    "y = torch.tensor(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_Nln-yhPaceX"
   },
   "source": [
    "## Exercise 2.4: Complete the `forward` method\n",
    "\n",
    "Based on your knowledge of single-variable linear models and object-oriented programming, complete the `forward` method in the `LinReg` class.\n",
    "This method should take the input X, multiply it by the `slope` class attribute, and add the `bias` class attribute."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dohP7dnQtAyN"
   },
   "outputs": [],
   "source": [
    "# Build our linear regression model\n",
    "class LinReg(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        # Randomly initialize 2 parameters, one for our slope and one for our bias.\n",
    "        self.slope = nn.Parameter(torch.rand(1))\n",
    "        self.bias = nn.Parameter(torch.rand(1))\n",
    "\n",
    "    def forward(self, X):\n",
    "        return ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "okgeYEnUtfHx"
   },
   "outputs": [],
   "source": [
    "lr = LinReg()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gEodiY28UruL"
   },
   "source": [
    "## Exercise 2.5: Modifying the learning rate and number of epochs\n",
    "\n",
    "Change the number of epochs `N_EPOCHS` and learning rate `LR` variables.\n",
    "What do you observe about the loss over time and  when the learning rate becomes too large or too small?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "g5peJppY0Tsm"
   },
   "outputs": [],
   "source": [
    "N_EPOCHS = 300\n",
    "LR = 1e-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 209,
     "status": "ok",
     "timestamp": 1624592622806,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "WGi-JIk-0ZP7",
    "outputId": "9d4b7f53-2c90-4ea3-a43e-fd78857b7cda"
   },
   "outputs": [],
   "source": [
    "slopes = []\n",
    "biases = []\n",
    "losses = []\n",
    "_alphas = []\n",
    "for i in range(N_EPOCHS):\n",
    "    # Make some inferences\n",
    "    yhat = lr(X)\n",
    "    # Measure how bad those guesses were\n",
    "    loss = F.mse_loss(yhat, y)\n",
    "    if i%(N_EPOCHS/10)==0:\n",
    "        print(f\"Epoch {i} Train Loss: {loss:.04f}\")\n",
    "    # Calculate the gradient of all the parameters with respect to the loss\n",
    "    loss.backward()\n",
    "    # Apply the SGD update rule\n",
    "    lr.slope.data.sub_(lr.slope.grad * LR)\n",
    "    lr.bias.data.sub_(lr.bias.grad * LR)\n",
    "    # Zero out the gradients for the next round\n",
    "    lr.slope.grad.zero_()\n",
    "    lr.bias.grad.zero_()\n",
    "\n",
    "    # Record the parameters and losses so we can plot them out later\n",
    "    slopes.append(float(lr.slope.data.detach().numpy()))\n",
    "    biases.append(float(lr.bias.data.detach().numpy()))\n",
    "    losses.append(float(loss.detach().numpy()))\n",
    "    _alphas.append(i/N_EPOCHS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 10,
     "status": "ok",
     "timestamp": 1624592622807,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "wgyFeEBudKpF",
    "outputId": "7c8947fc-fcdc-47e7-9847-9c32c0706d52"
   },
   "outputs": [],
   "source": [
    "lr.slope, lr.bias"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GWDG9EcIrmxh"
   },
   "source": [
    "Let's plot what we've done so far.\n",
    "The blue dots represent the data, and the red lines represent the functions created by our slopes and biases as the model learns.\n",
    "The more transparent lines are the first iterations in our training loop.\n",
    "Notice that as the lines get more solid, they fit the data better.\n",
    "This illustrates the process of our model learning the data!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 606
    },
    "executionInfo": {
     "elapsed": 1076,
     "status": "ok",
     "timestamp": 1624592623877,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "qrWoCKIV1z4X",
    "outputId": "b9f3bcaf-a235-439f-f0e6-2ff446844c8b"
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(10,10))\n",
    "ax.set_xlabel('X')\n",
    "ax.set_ylabel('y')\n",
    "ax.scatter(X, y)\n",
    "for s, b, a in zip(slopes, biases, _alphas):\n",
    "    _x = np.arange(0, 10)\n",
    "    _y = _x * s + b\n",
    "    ax.plot(_x, _y, alpha=a, c='red', label=f\"Epoch {int(a)}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 621
    },
    "executionInfo": {
     "elapsed": 274,
     "status": "ok",
     "timestamp": 1624592624143,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "7gQXK8bw2NL-",
    "outputId": "ec8e0cb7-172d-487f-ce69-7aa556bbc2d5"
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(14,10))\n",
    "ax.plot(losses)\n",
    "ax.set_xlabel('Epoch')\n",
    "ax.set_ylabel('Loss (MSE)')\n",
    "if (losses[-1] > losses[0]) | np.isnan(losses[-1]):\n",
    "    ax.set_title('Diverging - BAD!')\n",
    "else:\n",
    "    ax.set_title('Converging - goood!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7xwSwwrK2dXr"
   },
   "source": [
    "<!-- split -->\n",
    "\n",
    "# Multi-varable linear regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-VVJBDwN6X-_"
   },
   "outputs": [],
   "source": [
    "# Make yet another fake dataset\n",
    "from sklearn.datasets import make_regression\n",
    "X, y = make_regression(n_samples=1000, n_features=3, n_informative=2, bias=3, noise=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "3XHTBONL8APT"
   },
   "outputs": [],
   "source": [
    "# No more bad habits, we need to split our data.\n",
    "X_train, X_valid, y_train, y_valid = (torch.tensor(i).float() for i in train_test_split(X, y, test_size=0.1, random_state=42))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "DNX_biscdGbX"
   },
   "source": [
    "## Exercise 2.6: Create `weights` and `bias` tensors\n",
    "\n",
    "In the cell below, create a `weights` tensor and a `bias` tensor.\n",
    "For both of these tensors, use the [`torch.rand`](https://pytorch.org/docs/stable/generated/torch.rand.html) function.\n",
    "The `weights` tensor should have as many values as `X_train` has features.\n",
    "The `bias` tensor should just be a single random value.\n",
    "\n",
    "Once you've created these tensors, turn them into parameters using the [`nn.Parameter`](https://pytorch.org/docs/stable/generated/torch.nn.parameter.Parameter.html) class.\n",
    "If you need a reference here, look at how this was done when we created the `LinReg` class in Exercise 2.4."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2X8v_aqs8IMY"
   },
   "outputs": [],
   "source": [
    "# Let's create some temporary weights and biases and test out our matrix operations before we build our model.\n",
    "# Create a weights parameter with 1 beta per column in X\n",
    "weights = ...\n",
    "# Create our bias parameter\n",
    "bias = ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 29,
     "status": "ok",
     "timestamp": 1624592624150,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "ZW3sWkn49A9r",
    "outputId": "f5ebfdc9-7801-4acd-9cc0-39994349c71b"
   },
   "outputs": [],
   "source": [
    "# Test out the operation we want to perform in the forward pass\n",
    "torch.matmul(X_train[:10], weights) + bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 26,
     "status": "ok",
     "timestamp": 1624592624151,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "y8bauLhD7N8V",
    "outputId": "0fb9e04b-6e1b-4285-bd92-e9f2f5911cae"
   },
   "outputs": [],
   "source": [
    "# FYI: @ does the same thing as matmul in this context and is easier\n",
    "X_train[:10]@weights + bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mYRcEdGH8aN3"
   },
   "outputs": [],
   "source": [
    "# Sanity check: different implementations of our forward pass are the same\n",
    "assert (X_train@weights + bias == torch.matmul(X_train, weights) + bias).all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gNzWV8938wjO"
   },
   "outputs": [],
   "source": [
    "# Let's make our model\n",
    "class LinRegMulti(nn.Module):\n",
    "    def __init__(self, n_cols):\n",
    "        super().__init__()\n",
    "        self.n_cols = n_cols\n",
    "\n",
    "        self.weights = nn.Parameter(torch.rand(self.n_cols))\n",
    "        self.bias = nn.Parameter(torch.rand(1))\n",
    "    \n",
    "    def forward(self, X):\n",
    "        return X@self.weights.T + self.bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "wEzQbpnR8hqO"
   },
   "outputs": [],
   "source": [
    "N_EPOCHS = 10000\n",
    "LR = 1e-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MfWaPFjW9dfb"
   },
   "outputs": [],
   "source": [
    "lrm = LinRegMulti(X_train.shape[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "UlRs5GF7-CQh"
   },
   "outputs": [],
   "source": [
    "# Instead of updating each parameter individually, let's make an update rule function.\n",
    "def gd_update_rule(parameters, lr):\n",
    "    parameters.data.sub_(parameters.grad * lr)\n",
    "    parameters.grad.zero_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zzk5omp6-hEX"
   },
   "outputs": [],
   "source": [
    "train_losses = []\n",
    "valid_losses = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2681,
     "status": "ok",
     "timestamp": 1624592626814,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "GczinipS9hRz",
    "outputId": "23dd21db-ad3f-4552-e0e3-0c5f2b271b48"
   },
   "outputs": [],
   "source": [
    "for i in range(N_EPOCHS):\n",
    "    yhat = lrm(X_train)\n",
    "    loss = mse(yhat, y_train)\n",
    "    loss.backward()\n",
    "    for p in lrm.parameters():\n",
    "        gd_update_rule(p, LR)\n",
    "    train_losses.append(loss.detach().numpy())\n",
    "\n",
    "    with torch.no_grad():\n",
    "        yhat = lrm(X_valid)\n",
    "        valid_loss = mse(yhat, y_valid)\n",
    "        valid_losses.append(valid_loss.numpy())\n",
    "\n",
    "    if i%(N_EPOCHS/10) == 0:\n",
    "        print(f\"Epoch {i} Train Loss: {loss:.04f}, Valid Loss: {valid_loss:.04f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 717
    },
    "executionInfo": {
     "elapsed": 463,
     "status": "ok",
     "timestamp": 1624592627271,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "RoVuHW7a9iau",
    "outputId": "bef772dc-bbff-47b2-d451-476b5b63749c"
   },
   "outputs": [],
   "source": [
    "EPOCHS_TO_SHOW = 2000\n",
    "fig, ax = plt.subplots(figsize=(12, 12))\n",
    "ax.plot(train_losses[:EPOCHS_TO_SHOW], label='Train', linewidth=3, alpha=0.5)\n",
    "ax.plot(valid_losses[:EPOCHS_TO_SHOW], ls='--', label='Valid')\n",
    "ax.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 26,
     "status": "ok",
     "timestamp": 1624592627272,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "nVxNByIg-mas",
    "outputId": "33f0b42a-c82f-4f2a-d9d9-b5f58f595db7"
   },
   "outputs": [],
   "source": [
    "lrm.weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 22,
     "status": "ok",
     "timestamp": 1624592627273,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "fYNLiUqy-yxx",
    "outputId": "0931c986-89d5-4824-9868-233eb4fd5cb1"
   },
   "outputs": [],
   "source": [
    "lrm.bias"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Nof2GJMtAhEO"
   },
   "source": [
    "## The `Linear` layer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "59HrUTC5_YfS"
   },
   "outputs": [],
   "source": [
    "class Linear(nn.Module):\n",
    "    def __init__(self, dim_in, dim_out):\n",
    "        super().__init__()\n",
    "        self.weights = nn.Parameter(torch.rand((dim_in, dim_out)))\n",
    "        self.bias = nn.Parameter(torch.rand(dim_out))\n",
    "    \n",
    "    def forward(self, X):\n",
    "\n",
    "        return X@self.weights + self.bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "glH-bIYp8JxC"
   },
   "outputs": [],
   "source": [
    "# Let's compare our Linear class with nn.Linear\n",
    "l1 = Linear(3, 5)\n",
    "l2 = nn.Linear(3, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 19,
     "status": "ok",
     "timestamp": 1624592627275,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "SNAS7jI99yGE",
    "outputId": "3532b433-d66e-402b-8f39-ab7bf0eac38a"
   },
   "outputs": [],
   "source": [
    "l2.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 16,
     "status": "ok",
     "timestamp": 1624592627275,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "QgwbpaGw9wCT",
    "outputId": "afdaf6c1-c2ca-4f34-98d3-a67694fc7914"
   },
   "outputs": [],
   "source": [
    "l1.weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 15,
     "status": "ok",
     "timestamp": 1624592627276,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "ZHTeLzXP_Rzc",
    "outputId": "4b3a80f1-8318-4e50-e6f2-56bc16cfe784"
   },
   "outputs": [],
   "source": [
    "l1.weights.data.copy_(l2.weight.T)\n",
    "l1.bias.data.copy_(l2.bias)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 15,
     "status": "ok",
     "timestamp": 1624592627277,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "LZsxCvsZ8c5R",
    "outputId": "f77be2ad-2785-41cf-a3b0-1ad200923870"
   },
   "outputs": [],
   "source": [
    "l1(X_train[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 178,
     "status": "ok",
     "timestamp": 1624592627441,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "o-l2dsw78oyO",
    "outputId": "9a9b8d07-7c3e-4d8c-81db-804121039379"
   },
   "outputs": [],
   "source": [
    "l2(X_train[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "OJK1Q27O_cdR"
   },
   "outputs": [],
   "source": [
    "assert (l1(X_train[:5]) == l2(X_train[:5])).all()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kRUnA85j8l9i"
   },
   "source": [
    "Sometimes `torch`'s implementations with some optimizations that make operations run faster.\n",
    "We'll see examples of this later in the course.\n",
    "I was just curious to see whether `torch`'s implementation was significantly faster than ours using `%%timeit`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 10273,
     "status": "ok",
     "timestamp": 1624592637710,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "iRV8GSei7rt_",
    "outputId": "45b47dbb-9978-4eb3-a9e4-970fea42332c"
   },
   "outputs": [],
   "source": [
    "%%timeit \n",
    "l1(X_train[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 9823,
     "status": "ok",
     "timestamp": 1624592647518,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "WFOymbqq7wHO",
    "outputId": "89f5fa74-04e1-4e2a-d126-a11c8a5cc3bc"
   },
   "outputs": [],
   "source": [
    "%%timeit\n",
    " l2(X_train[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "aw6rORfmCmZt"
   },
   "source": [
    "## Nonlinearities (activation functions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 516
    },
    "executionInfo": {
     "elapsed": 497,
     "status": "ok",
     "timestamp": 1624592647998,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "gE7TAbIG2tjD",
    "outputId": "1141b5b3-d551-4c94-9c9c-c293770667e1"
   },
   "outputs": [],
   "source": [
    "rng = torch.arange(-5, 5.01, 0.05)\n",
    "fig, ax = plt.subplots(figsize=(14, 8))\n",
    "ax.plot(rng, F.relu(rng), label='ReLU')\n",
    "ax.plot(rng, torch.tanh(rng), label='tanh')\n",
    "ax.plot(rng, torch.sigmoid(rng), label='sigmoid')\n",
    "ax.plot(rng, F.leaky_relu(rng, negative_slope=0.01), ls='--', label='leaky ReLU')\n",
    "ax.set_ylim(-1.1, 1.1)\n",
    "ax.set_title('Common activation functions')\n",
    "ax.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7jGO3lsathAu"
   },
   "source": [
    "## Exercise 2.7: Explore our neural network\n",
    "\n",
    "In this exercise, use the code below to explore different aspects of our nerual netowrk.\n",
    "For each of the following scenarios, inspect the plots of the train/validaiton loss at the end of the notebook and record what happens with each experient you do.\n",
    "Try to...\n",
    "* Switch out `nn.Linear` for the `Linear` class we defined earlier. Do these networks learn differently?\n",
    "* What happens if if we use `MultiLayerRegressor` (the subclassing API) vs. the `multilayer_regressor` function (the sequential API)? Are these equivalent?\n",
    "* What happens as you increase the `hiiden_dim`? \n",
    "* Remove the nonlinearity. Does your network learn as well?\n",
    "* Try adding at least 1 layer in between the input and output layers of the network, with a nonlinearity. **NOTE:** Since we're learning a really simple function, additional layers may not help our loss."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "E1zUoNys_9QL"
   },
   "outputs": [],
   "source": [
    "class MultiLayerRegressor(nn.Module):\n",
    "    def __init__(self, dim_in, hidden_dim):\n",
    "        super().__init__()\n",
    "        # self.first_layer = Linear(dim_in, hidden_dim)\n",
    "        self.first_layer = nn.Linear(dim_in, hidden_dim)\n",
    "        # self.second_layer = Linear(hidden_dim, 1)\n",
    "        self.second_layer = nn.Linear(hidden_dim, 1)\n",
    "    \n",
    "    def forward(self, X):\n",
    "        x = self.first_layer(X)\n",
    "        # x = relu(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.second_layer(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mc0dA6sxIFoe"
   },
   "outputs": [],
   "source": [
    "def multilayer_regressor(in_dim, hidden_dim):\n",
    "    return nn.Sequential(\n",
    "        nn.Linear(in_dim, hidden_dim),\n",
    "        nn.ReLU(),\n",
    "        nn.Linear(hidden_dim, 1)\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FKLZZbRBAaBD"
   },
   "outputs": [],
   "source": [
    "# mlr = MultiLayerRegressor(3, 4)\n",
    "mlr = multilayer_regressor(3, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0oaN64-iAwJ4"
   },
   "outputs": [],
   "source": [
    "train_losses = []\n",
    "valid_losses = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Yt1uzMjZBkWQ"
   },
   "outputs": [],
   "source": [
    "LR = 1e-3\n",
    "N_EPOCHS = 20000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "CPuw46kBFdty"
   },
   "outputs": [],
   "source": [
    "# Notice that instead of iterating through our parameters and applying\n",
    "# an update rule, we're just using torch's built in SGD optimizer.\n",
    "opt = optim.SGD(mlr.parameters(), lr=LR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 10058,
     "status": "ok",
     "timestamp": 1624592658048,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "Bf91-OmaAepE",
    "outputId": "6b2f2d56-eec0-4728-c86a-8ee60bf3fc0f"
   },
   "outputs": [],
   "source": [
    "for i in range(N_EPOCHS):\n",
    "    yhat = mlr(X_train).squeeze()\n",
    "    loss = F.mse_loss(yhat, y_train)\n",
    "    loss.backward()\n",
    "    opt.step()\n",
    "    opt.zero_grad()\n",
    "    train_losses.append(loss.detach().numpy())\n",
    "\n",
    "    with torch.no_grad():\n",
    "        yhat = mlr(X_valid).squeeze()\n",
    "        valid_loss = F.mse_loss(yhat, y_valid)\n",
    "        valid_losses.append(loss.numpy())\n",
    "    \n",
    "    if i%(N_EPOCHS/10) == 0:\n",
    "        print(f\"Epoch {i} Train loss: {loss:.04f}, Valid loss: {valid_loss:.04f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 717
    },
    "executionInfo": {
     "elapsed": 235,
     "status": "ok",
     "timestamp": 1624592658269,
     "user": {
      "displayName": "Mike Frantz",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiGYCwZXGfGg-CE4_33PQcH9SW8fN6MyRkenN8-IA=s64",
      "userId": "08482506601269125384"
     },
     "user_tz": 420
    },
    "id": "1FImQTDQA278",
    "outputId": "491c8ffa-7707-4af9-ea6f-0de1e7cbb651"
   },
   "outputs": [],
   "source": [
    "idx=1000\n",
    "fig, ax = plt.subplots(figsize=(12, 12))\n",
    "ax.plot(train_losses[:idx], label='Train', linewidth=3, alpha=0.5)\n",
    "ax.plot(valid_losses[:idx], ls='--', label='Valid')\n",
    "ax.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "q-WNsgD3UZlj"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Our_first_neural_network_linear_regression_workset.ipynb",
   "provenance": [
    {
     "file_id": "1Ki18vbSpIKP0t820buqRtCcD4X_4xhjr",
     "timestamp": 1635372066549
    }
   ]
  },
  "kernelspec": {
   "display_name": "cndl",
   "language": "python",
   "name": "cndl"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}