Lab04_1.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jq9FmMNvNzdV"
      },
      "source": [
        "# Lab 4-1: Multivariate Linear Regression"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S0oHTlU-Nzdb"
      },
      "source": [
        "Author: Seungjae Lee (이승재)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ob7dFIiHNzdb"
      },
      "source": [
        "<div class=\"alert alert-warning\">\n",
        "    We use elemental PyTorch to implement linear regression here. However, in most actual applications, abstractions such as <code>nn.Module</code> or <code>nn.Linear</code> are used.\n",
        "</div>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DqtSVKlvNzdc"
      },
      "source": [
        "## Theoretical Overview"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "epjy2XAONzdc"
      },
      "source": [
        "$$ H(x_1, x_2, x_3) = x_1w_1 + x_2w_2 + x_3w_3 + b $$"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FA2NgiUwNzdd"
      },
      "source": [
        "$$ cost(W, b) = \\frac{1}{m} \\sum^m_{i=1} \\left( H(x^{(i)}) - y^{(i)} \\right)^2 $$"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ibR7U-sQNzdd"
      },
      "source": [
        " - $H(x)$: 주어진 $x$ 값에 대해 예측을 어떻게 할 것인가\n",
        " - $cost(W, b)$: $H(x)$ 가 $y$ 를 얼마나 잘 예측했는가"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qkxU_lSGNzde"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jvdcBNgVNzdf"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "E-SVYt4gNzdh",
        "outputId": "55953183-994d-4433-cbe9-8c84d2b86021",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<torch._C.Generator at 0x7f1bbd0d5950>"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "# For reproducibility\n",
        "torch.manual_seed(1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eEIv4ZstNzdi"
      },
      "source": [
        "## Naive Data Representation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GhFgkn5nNzdi"
      },
      "source": [
        "We will use fake data for this example."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WikMV94SNzdj"
      },
      "outputs": [],
      "source": [
        "# 데이터\n",
        "x1_train = torch.FloatTensor([[73], [93], [89], [96], [73]])\n",
        "x2_train = torch.FloatTensor([[80], [88], [91], [98], [66]])\n",
        "x3_train = torch.FloatTensor([[75], [93], [90], [100], [70]])\n",
        "y_train = torch.FloatTensor([[152], [185], [180], [196], [142]])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "R9LHBx2VNzdj",
        "outputId": "3393a6e7-ac2f-41c3-9365-b93d27fbf265",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch    0/1000 w1: 0.294 w2: 0.297 w3: 0.297 b: 0.003 Cost: 29661.800781\n",
            "Epoch  100/1000 w1: 0.674 w2: 0.676 w3: 0.676 b: 0.008 Cost: 1.563628\n",
            "Epoch  200/1000 w1: 0.679 w2: 0.677 w3: 0.677 b: 0.008 Cost: 1.497595\n",
            "Epoch  300/1000 w1: 0.684 w2: 0.677 w3: 0.677 b: 0.008 Cost: 1.435044\n",
            "Epoch  400/1000 w1: 0.689 w2: 0.678 w3: 0.678 b: 0.008 Cost: 1.375726\n",
            "Epoch  500/1000 w1: 0.694 w2: 0.678 w3: 0.678 b: 0.009 Cost: 1.319507\n",
            "Epoch  600/1000 w1: 0.699 w2: 0.679 w3: 0.679 b: 0.009 Cost: 1.266222\n",
            "Epoch  700/1000 w1: 0.704 w2: 0.679 w3: 0.679 b: 0.009 Cost: 1.215703\n",
            "Epoch  800/1000 w1: 0.709 w2: 0.679 w3: 0.679 b: 0.009 Cost: 1.167810\n",
            "Epoch  900/1000 w1: 0.713 w2: 0.680 w3: 0.680 b: 0.009 Cost: 1.122429\n",
            "Epoch 1000/1000 w1: 0.718 w2: 0.680 w3: 0.680 b: 0.009 Cost: 1.079390\n"
          ]
        }
      ],
      "source": [
        "# 모델 초기화\n",
        "w1 = torch.zeros(1, requires_grad=True)\n",
        "w2 = torch.zeros(1, requires_grad=True)\n",
        "w3 = torch.zeros(1, requires_grad=True)\n",
        "b =  torch.zeros(1, requires_grad=True)\n",
        "# optimizer 설정\n",
        "optimizer = optim.SGD([w1, w2, w3, b], lr=1e-5)\n",
        "\n",
        "nb_epochs = 1000\n",
        "for epoch in range(nb_epochs + 1):\n",
        "    \n",
        "    # H(x) 계산\n",
        "    hypothesis = w1*x1_train + w2*x2_train + w3*x3_train + b\n",
        "\n",
        "    # cost 계산\n",
        "    cost = torch.mean((hypothesis - y_train)**2)\n",
        "\n",
        "    # cost로 H(x) 개선\n",
        "    optimizer.zero_grad()\n",
        "    cost.backward()\n",
        "    optimizer.step()\n",
        "\n",
        "    # 100번마다 로그 출력\n",
        "    if epoch % 100 == 0:\n",
        "        print('Epoch {:4d}/{} w1: {:.3f} w2: {:.3f} w3: {:.3f} b: {:.3f} Cost: {:.6f}'.format(\n",
        "            epoch, nb_epochs, w1.item(), w3.item(), w3.item(), b.item(), cost.item()\n",
        "        ))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oyQrAOYDNzdk"
      },
      "source": [
        "## Matrix Data Representation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VHhuDWRkNzdk"
      },
      "source": [
        "$$\n",
        "\\begin{pmatrix}\n",
        "x_1 & x_2 & x_3\n",
        "\\end{pmatrix}\n",
        "\\cdot\n",
        "\\begin{pmatrix}\n",
        "w_1 \\\\\n",
        "w_2 \\\\\n",
        "w_3 \\\\\n",
        "\\end{pmatrix}\n",
        "=\n",
        "\\begin{pmatrix}\n",
        "x_1w_1 + x_2w_2 + x_3w_3\n",
        "\\end{pmatrix}\n",
        "$$"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yM4yuJknNzdl"
      },
      "source": [
        "$$ H(X) = XW $$"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Q1. 다중 선형 회귀(Multivariable Linear regression)에서 Matrix를 사용하는 이유는?\n",
        "\n",
        "=>"
      ],
      "metadata": {
        "id": "psdBkw0nKmnd"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "VDgttVcrNzdl"
      },
      "outputs": [],
      "source": [
        "x_train = torch.FloatTensor([[73, 80, 75],\n",
        "                             [93, 88, 93],\n",
        "                             [89, 91, 90],\n",
        "                             [96, 98, 100],\n",
        "                             [73, 66, 70]])\n",
        "y_train = torch.FloatTensor([[152], [185], [180], [196], [142]])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "vnKe27liNzdl",
        "outputId": "f02f8fe5-49e2-44a6-9728-bb095e45785d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([5, 3])\n",
            "torch.Size([5, 1])\n"
          ]
        }
      ],
      "source": [
        "print(x_train.shape)\n",
        "print(y_train.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "7oCb7Q8kNzdm",
        "outputId": "7d8f85b1-4139-40d2-e5bb-ef4d8299496d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch    0/20 hypothesis: tensor([0., 0., 0., 0., 0.]) Cost: 29661.800781\n",
            "Epoch    1/20 hypothesis: tensor([66.2775, 79.6621, 78.4917, 85.4747, 60.7631]) Cost: 9887.587891\n",
            "Epoch    2/20 hypothesis: tensor([103.3836, 124.2622, 122.4363, 133.3284,  94.7828]) Cost: 3689.406982\n",
            "Epoch    3/20 hypothesis: tensor([124.1576, 149.2324, 147.0393, 160.1196, 113.8299]) Cost: 1746.592529\n",
            "Epoch    4/20 hypothesis: tensor([135.7879, 163.2125, 160.8135, 175.1185, 124.4943]) Cost: 1137.608887\n",
            "Epoch    5/20 hypothesis: tensor([142.2990, 171.0397, 168.5251, 183.5154, 130.4655]) Cost: 946.710999\n",
            "Epoch    6/20 hypothesis: tensor([145.9440, 175.4222, 172.8425, 188.2162, 133.8092]) Cost: 886.860291\n",
            "Epoch    7/20 hypothesis: tensor([147.9844, 177.8760, 175.2597, 190.8475, 135.6819]) Cost: 868.086426\n",
            "Epoch    8/20 hypothesis: tensor([149.1265, 179.2500, 176.6129, 192.3203, 136.7310]) Cost: 862.187195\n",
            "Epoch    9/20 hypothesis: tensor([149.7655, 180.0195, 177.3705, 193.1444, 137.3190]) Cost: 860.324219\n",
            "Epoch   10/20 hypothesis: tensor([150.1230, 180.4506, 177.7946, 193.6054, 137.6489]) Cost: 859.726074\n",
            "Epoch   11/20 hypothesis: tensor([150.3229, 180.6922, 178.0319, 193.8631, 137.8342]) Cost: 859.524048\n",
            "Epoch   12/20 hypothesis: tensor([150.4345, 180.8277, 178.1648, 194.0069, 137.9386]) Cost: 859.446899\n",
            "Epoch   13/20 hypothesis: tensor([150.4967, 180.9038, 178.2392, 194.0870, 137.9977]) Cost: 859.408264\n",
            "Epoch   14/20 hypothesis: tensor([150.5312, 180.9467, 178.2807, 194.1315, 138.0314]) Cost: 859.381958\n",
            "Epoch   15/20 hypothesis: tensor([150.5502, 180.9709, 178.3040, 194.1559, 138.0510]) Cost: 859.359863\n",
            "Epoch   16/20 hypothesis: tensor([150.5605, 180.9847, 178.3169, 194.1692, 138.0625]) Cost: 859.338501\n",
            "Epoch   17/20 hypothesis: tensor([150.5660, 180.9927, 178.3241, 194.1763, 138.0697]) Cost: 859.317871\n",
            "Epoch   18/20 hypothesis: tensor([150.5688, 180.9974, 178.3281, 194.1797, 138.0743]) Cost: 859.297119\n",
            "Epoch   19/20 hypothesis: tensor([150.5700, 181.0003, 178.3303, 194.1813, 138.0776]) Cost: 859.276428\n",
            "Epoch   20/20 hypothesis: tensor([150.5704, 181.0022, 178.3315, 194.1817, 138.0801]) Cost: 859.255615\n"
          ]
        }
      ],
      "source": [
        "# 모델 초기화\n",
        "W = torch.zeros(3, requires_grad=True)\n",
        "b = torch.zeros(1, requires_grad=True) \n",
        "# optimizer 설정\n",
        "optimizer = optim.SGD([W, b], lr=1e-5)\n",
        "\n",
        "nb_epochs = 20\n",
        "for epoch in range(nb_epochs + 1):\n",
        "    \n",
        "    # H(x) 계산\n",
        "    hypothesis = x_train.matmul(W) + b # or .mm or @ #x_train의 shape이 달라졌기 때문에 행렬곱셉을 합니다!!\n",
        "\n",
        "    # cost 계산\n",
        "    cost = torch.mean((hypothesis - y_train)**2)\n",
        "\n",
        "    # cost로 H(x) 개선\n",
        "    optimizer.zero_grad()\n",
        "    cost.backward()\n",
        "    optimizer.step()\n",
        "    \n",
        "    # 100번마다 로그 출력\n",
        "    print('Epoch {:4d}/{} hypothesis: {} Cost: {:.6f}'.format(\n",
        "        epoch, nb_epochs, hypothesis.squeeze().detach(), cost.item()\n",
        "    ))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P6BxO4e1Nzdm"
      },
      "source": [
        "## High-level Implementation with `nn.Module`"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gZGUyQGENzdn"
      },
      "source": [
        "Do you remember this model?"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "4mxIgU1LNzdn"
      },
      "outputs": [],
      "source": [
        "class LinearRegressionModel(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.linear = nn.Linear(1, 1)\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.linear(x)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DRT7z8qQNzdn"
      },
      "source": [
        "We just need to change the input dimension from 1 to 3!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "eCgmdZ4dNzdn"
      },
      "outputs": [],
      "source": [
        "class MultivariateLinearRegressionModel(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.linear = nn.Linear(3, 1)\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.linear(x)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4-iRrmwrNzdo",
        "outputId": "07828324-8a99-49f4-953c-a8e847109741"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch    0/20 Cost: 31667.597656\n",
            "Epoch    1/20 Cost: 9926.266602\n",
            "Epoch    2/20 Cost: 3111.513672\n",
            "Epoch    3/20 Cost: 975.451355\n",
            "Epoch    4/20 Cost: 305.908539\n",
            "Epoch    5/20 Cost: 96.042488\n",
            "Epoch    6/20 Cost: 30.260750\n",
            "Epoch    7/20 Cost: 9.641701\n",
            "Epoch    8/20 Cost: 3.178671\n",
            "Epoch    9/20 Cost: 1.152871\n",
            "Epoch   10/20 Cost: 0.517863\n",
            "Epoch   11/20 Cost: 0.318801\n",
            "Epoch   12/20 Cost: 0.256388\n",
            "Epoch   13/20 Cost: 0.236821\n",
            "Epoch   14/20 Cost: 0.230660\n",
            "Epoch   15/20 Cost: 0.228719\n",
            "Epoch   16/20 Cost: 0.228095\n",
            "Epoch   17/20 Cost: 0.227880\n",
            "Epoch   18/20 Cost: 0.227799\n",
            "Epoch   19/20 Cost: 0.227762\n",
            "Epoch   20/20 Cost: 0.227732\n"
          ]
        }
      ],
      "source": [
        "# 데이터\n",
        "x_train = torch.FloatTensor([[73, 80, 75],\n",
        "                             [93, 88, 93],\n",
        "                             [89, 91, 90],\n",
        "                             [96, 98, 100],\n",
        "                             [73, 66, 70]])\n",
        "y_train = torch.FloatTensor([[152], [185], [180], [196], [142]])\n",
        "# 모델 초기화\n",
        "model = MultivariateLinearRegressionModel()\n",
        "# optimizer 설정\n",
        "optimizer = optim.SGD(model.parameters(), lr=1e-5) # #SGD optimizer를 사용하고 learning rate는 1e-5로 적용하세요.\n",
        "\n",
        "nb_epochs = 20\n",
        "for epoch in range(nb_epochs+1):\n",
        "    \n",
        "    # H(x) 계산\n",
        "    prediction = model(x_train)\n",
        "    \n",
        "    # cost 계산\n",
        "    cost = F.mse_loss(prediction, y_train)\n",
        "    \n",
        "    # cost로 H(x) 개선\n",
        "    optimizer.zero_grad()\n",
        "    cost.backward()\n",
        "    optimizer.step()\n",
        "    \n",
        "    # 20번마다 로그 출력\n",
        "    print('Epoch {:4d}/{} Cost: {:.6f}'.format(\n",
        "        epoch, nb_epochs, cost.item()\n",
        "    ))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "SmVuUNuslyXP"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.0"
    },
    "colab": {
      "name": "Lab04_1.ipynb",
      "provenance": [],
      "collapsed_sections": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}