codes/Pytorch_Seq2Seq_One-feature.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### library setttings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.8.1\n",
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "# others\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import argparse\n",
    "import datetime\n",
    "from copy import deepcopy # Add Deepcopy for args\n",
    "import pickle \n",
    "import seaborn as sns \n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "\n",
    "# pytorch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch.optim as optim\n",
    "\n",
    "\n",
    "print(torch.__version__)\n",
    "%matplotlib inline\n",
    "%pylab inline\n",
    "pylab.rcParams['figure.figsize'] = (15, 9)\n",
    "\n",
    "\n",
    "# read file\n",
    "file_path = './data/merged_data.h5'\n",
    "data = pd.read_hdf(file_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = data.sample_id.nunique()\n",
    "\n",
    "# train, val, test = 7 : 2  : 1\n",
    "train_df = data[data.sample_id < int(n*0.7)]\n",
    "val_df =data[(data.sample_id >= int(n*0.7)) & (data.sample_id < int(n*0.9))]\n",
    "test_df = data[data.sample_id >= int(n*0.9)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DatasetGenerater(Dataset):\n",
    "    ''' 설명 생략 '''\n",
    "    \n",
    "    def __init__(self, dataframe, x_frames, y_frames, status_idx = None):\n",
    "\n",
    "        self.grouped_dataframe = dataframe.groupby('sample_id')\n",
    "        self.sample_length = dataframe.sample_id.nunique()\n",
    "        self.x_frames = x_frames\n",
    "        self.y_frames = y_frames\n",
    "        \n",
    "        if status_idx == 'train':\n",
    "            self.status_idx = 0\n",
    "            print(self.status_idx)\n",
    "            \n",
    "        elif status_idx == 'validation':\n",
    "            self.status_idx = 845\n",
    "            print(self.status_idx)\n",
    "            \n",
    "        else:\n",
    "            self.status_idx = 1087\n",
    "            print(self.status_idx)\n",
    "        \n",
    "        \n",
    "        \n",
    "    def __len__(self):\n",
    "        return self.sample_length\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        \n",
    "        idx += self.status_idx\n",
    "        \n",
    "        one_sample_df = self.grouped_dataframe.get_group(idx)\n",
    "        \n",
    "        one_sample_df = one_sample_df[-(self.x_frames + self.y_frames):]\n",
    "        one_sample_arr = one_sample_df.drop(['sample_id','time', 'coin_index'], axis = 1).values\n",
    "\n",
    "        # get values\n",
    "        X = one_sample_arr[:self.x_frames, 0]\n",
    "        y = one_sample_arr[self.x_frames:, 0]\n",
    "        \n",
    "        return X, y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### hpyperparameter setting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "device is cpu\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# ====== initialization\n",
    "parser = argparse.ArgumentParser()\n",
    "args = parser.parse_args(\"\")\n",
    "args.device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "print(\"device is\",args.device)\n",
    "\n",
    "seed = 777\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "\n",
    "\n",
    "# ====== Model Capacity options ===== #\n",
    "args.input_dim = 1\n",
    "args.hidden_dim = 50\n",
    "args.output_dim = 1\n",
    "args.num_layers = 1\n",
    "args.batch_size = 64\n",
    "args.dropout = 0.2\n",
    "args.target_len = 120\n",
    "args.use_bn = True\n",
    "\n",
    "# ====== Dataset Generating options ====== #\n",
    "args.x_frames = 255\n",
    "args.y_frames = 120\n",
    "\n",
    "# ====== Model training options ===== #\n",
    "args.num_epoch = 30\n",
    "args.learning_rate = 0.001\n",
    "args.L2_rate = 0.0001\n",
    "args.training_prediction = 'mixed_teacher_forcing'\n",
    "args.teacher_forcing_ratio = 0.6\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "845\n",
      "1087\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# dataset partitioning\n",
    "trainset = DatasetGenerater(train_df, x_frames = args.x_frames, y_frames = args.y_frames, status_idx = 'train')\n",
    "valset = DatasetGenerater(val_df, x_frames = args.x_frames, y_frames = args.y_frames, status_idx = 'validation')\n",
    "testset = DatasetGenerater(test_df, x_frames = args.x_frames, y_frames = args.y_frames, status_idx = 'test')\n",
    "partition = {'train': trainset, 'val':valset, 'test':testset}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(255,) (120,)\n"
     ]
    }
   ],
   "source": [
    "for i, (X,y) in enumerate(trainset):\n",
    "    print(X.shape,y.shape)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    ''' Encodes time-serise sequence '''\n",
    "    \n",
    "    def __init__(self, input_dim, hidden_dim, num_layers):\n",
    "        \n",
    "        '''\n",
    "        : param input_size:     the number of features in the input X\n",
    "        : param hidden_size:    the number of features in the hidden state h\n",
    "        : param num_layers:     number of recurrent layers (i.e., 2 means there are 2 stacked LSTMs)\n",
    "        '''\n",
    "        \n",
    "        super(Encoder, self).__init__()\n",
    "        self.input_dim = input_dim\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.num_layers = num_layers\n",
    "\n",
    "        \n",
    "        # define LSTM layer\n",
    "        self.rnn = nn.GRU(self.input_dim, self.hidden_dim, self.num_layers)\n",
    "    \n",
    "    def forward(self, X):\n",
    "        \n",
    "        '''\n",
    "        : param x_input:               input of shape (seq_len, # in batch, input_size)\n",
    "        : return lstm_out, hidden:     lstm_out gives all the hidden states in the sequence;\n",
    "        :                              hidden gives the hidden state and cell state for the last\n",
    "        :                              element in the sequence \n",
    "        '''\n",
    "        \n",
    "        rnn_out, self.hidden = self.rnn(X)\n",
    "        return rnn_out, self.hidden\n",
    "    \n",
    "    def init_hidden(self, batch_size):\n",
    "        \n",
    "        '''\n",
    "        initialize hidden state\n",
    "        : param batch_size:    x_input.shape[1]\n",
    "        : return:              zeroed hidden state and cell state \n",
    "        '''\n",
    "        \n",
    "        return (torch.zeros(self.num_layers, batch_size, self.hidden_dim),\n",
    "                torch.zeros(self.num_layers, batch_size, self.hidden_dim))\n",
    "\n",
    "    \n",
    "class Decoder(nn.Module):\n",
    "    ''' Decodes hidden state output by encoder '''\n",
    "    \n",
    "    def __init__(self, output_dim, hidden_dim, num_layers, dropout, use_bn):\n",
    "        \n",
    "        '''\n",
    "        : param output_dim:     the number of features in the input X\n",
    "        : param hidden_dim:    the number of features in the hidden state h\n",
    "        : param num_layers:     number of recurrent layers (i.e., 2 means there are\n",
    "        :                       2 stacked LSTMs)\n",
    "        '''\n",
    "        \n",
    "        super(Decoder, self).__init__()\n",
    "        self.output_dim = output_dim \n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.num_layers = num_layers\n",
    "        self.dropout = dropout\n",
    "        self.use_bn = use_bn\n",
    "        \n",
    "        self.rnn = nn.GRU(self.output_dim, self.hidden_dim, self.num_layers)\n",
    "        self.fc_out = self.regressor()\n",
    "        \n",
    "    def forward(self, x_input, encoder_hidden_states):\n",
    "               \n",
    "        '''        \n",
    "        : param x_input:                    should be 2D (batch_size, input_size)\n",
    "        : param encoder_hidden_states:      hidden states\n",
    "        : return output, hidden:            output gives all the hidden states in the sequence;\n",
    "        :                                   hidden gives the hidden state and cell state for the last\n",
    "        :                                   element in the sequence \n",
    " \n",
    "        '''\n",
    "        \n",
    "        # print(f'decoder input size {x_input.shape} to {x_input.reshape(1, -1, self.output_dim).shape}')\n",
    "        rnn_out, self.hidden = self.rnn(x_input.reshape(1, -1, self.output_dim), encoder_hidden_states)\n",
    "        output = self.fc_out(rnn_out.squeeze(0)) \n",
    "        \n",
    "        return output, self.hidden\n",
    "    \n",
    "    def regressor(self):\n",
    "\n",
    "        layers = []\n",
    "        if self.use_bn:\n",
    "            layers.append(nn.BatchNorm1d(self.hidden_dim))\n",
    "        layers.append(nn.Dropout(self.dropout))\n",
    "        \n",
    "        layers.append(nn.Linear(self.hidden_dim, self.hidden_dim // 2))\n",
    "        layers.append(nn.ReLU())\n",
    "        layers.append(nn.Linear(self.hidden_dim // 2, self.output_dim))\n",
    "        regressor = nn.Sequential(*layers)\n",
    "\n",
    "        return regressor\n",
    "\n",
    "class Seq2Seq(nn.Module):\n",
    "    \n",
    "    def __init__(self, encoder, decoder, device, target_len, training_prediction, teacher_forcing_ratio):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "        self.device = device\n",
    "        self.target_len = target_len\n",
    "        self.training_prediction = training_prediction\n",
    "        self.teacher_forcing_ratio = teacher_forcing_ratio\n",
    "        \n",
    "        assert encoder.hidden_dim == decoder.hidden_dim, \\\n",
    "            \"Hidden dimensions of encoder and decoder must be equal!\"\n",
    "        assert encoder.num_layers == decoder.num_layers, \\\n",
    "            \"Encoder and decoder must have equal number of layers!\"\n",
    "\n",
    "    \n",
    "    def forward(self, X, y):\n",
    "            \n",
    "            if y is None:\n",
    "                self.training_prediction = 'recursive'\n",
    "                \n",
    "                \n",
    "            # initialize\n",
    "            outputs = torch.zeros(self.target_len, X.shape[1], self.decoder.output_dim).to(self.device)\n",
    "            \n",
    "            encoder_hidden = self.encoder.init_hidden(X.shape[1])\n",
    "        \n",
    "            # encoder process\n",
    "            encoder_output, encoder_hidden = self.encoder(X)  \n",
    "            \n",
    "            #decoder process\n",
    "            ''' last X sequence, shape = [batch_size, open_index = 0] '''    \n",
    "            decoder_input = X[-1, :, 0]  \n",
    "            decoder_hidden = encoder_hidden  \n",
    "            \n",
    "\n",
    "            # ======================================================================================#\n",
    "            if self.training_prediction == 'recursive':\n",
    "                # predict recursively\n",
    "                for t in range(self.target_len): \n",
    "                    decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden)\n",
    "                    \n",
    "                    outputs[t] = decoder_output \n",
    "                    decoder_input = decoder_output\n",
    "                    \n",
    "           # ======================================================================================#\n",
    "            if self.training_prediction == 'teacher_forcing':\n",
    "                # use teacher forcing\n",
    "                if random.random() < self.teacher_forcing_ratio:\n",
    "                    for t in range(self.target_len): \n",
    "                        decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden)\n",
    "                        outputs[t] = decoder_output\n",
    "                        decoder_input = y[t, :].unsqueeze(1)\n",
    "                        \n",
    "                # predict recursively \n",
    "                else:\n",
    "                    for t in range(self.target_len): \n",
    "                        decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden)\n",
    "                        outputs[t] = decoder_output\n",
    "                        decoder_input = decoder_output\n",
    "                        \n",
    "            # ======================================================================================#\n",
    "            if self.training_prediction == 'mixed_teacher_forcing':\n",
    "                # predict using mixed teacher forcing\n",
    "                for t in range(self.target_len):\n",
    "\n",
    "                    decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden)\n",
    "                    outputs[t] = decoder_output \n",
    "                                    \n",
    "                    # predict with teacher forcing\n",
    "                    if random.random() < self.teacher_forcing_ratio:\n",
    "                        decoder_input = y[t, :].unsqueeze(1)\n",
    "\n",
    "                    # predict recursively \n",
    "                    else:\n",
    "                        decoder_input = decoder_output\n",
    "\n",
    "            # ======================================================================================#      \n",
    "            return outputs\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([255, 64, 1]) torch.Size([120, 64])\n",
      "torch.Size([120, 64, 1])\n",
      "torch.Size([255, 64, 1]) torch.Size([120, 64])\n",
      "torch.Size([120, 64, 1])\n",
      "torch.Size([255, 64, 1]) torch.Size([120, 64])\n",
      "torch.Size([120, 64, 1])\n",
      "torch.Size([255, 64, 1]) torch.Size([120, 64])\n",
      "torch.Size([120, 64, 1])\n",
      "torch.Size([255, 64, 1]) torch.Size([120, 64])\n",
      "torch.Size([120, 64, 1])\n",
      "torch.Size([255, 64, 1]) torch.Size([120, 64])\n",
      "torch.Size([120, 64, 1])\n",
      "torch.Size([255, 64, 1]) torch.Size([120, 64])\n",
      "torch.Size([120, 64, 1])\n",
      "torch.Size([255, 64, 1]) torch.Size([120, 64])\n",
      "torch.Size([120, 64, 1])\n",
      "torch.Size([255, 64, 1]) torch.Size([120, 64])\n",
      "torch.Size([120, 64, 1])\n",
      "torch.Size([255, 64, 1]) torch.Size([120, 64])\n",
      "torch.Size([120, 64, 1])\n",
      "torch.Size([255, 64, 1]) torch.Size([120, 64])\n",
      "torch.Size([120, 64, 1])\n",
      "torch.Size([255, 64, 1]) torch.Size([120, 64])\n",
      "torch.Size([120, 64, 1])\n",
      "torch.Size([255, 64, 1]) torch.Size([120, 64])\n",
      "torch.Size([120, 64, 1])\n"
     ]
    }
   ],
   "source": [
    "\n",
    "trainloader = DataLoader(partition['train'], batch_size = args.batch_size, shuffle = False, drop_last = True)\n",
    "\n",
    "# Encoder\n",
    "enc = Encoder(args.input_dim, args.hidden_dim, args.num_layers)\n",
    "\n",
    "# Decoder\n",
    "dec = Decoder(args.output_dim, args.hidden_dim, args.num_layers, args.dropout, args.use_bn)\n",
    "\n",
    "# Seq2Seq model\n",
    "model = Seq2Seq(enc, dec, args.device, args.target_len, args.training_prediction, args.teacher_forcing_ratio)\n",
    "model.to(args.device)\n",
    "\n",
    "loss_fn = nn.MSELoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=args.learning_rate, weight_decay=args.L2_rate)\n",
    "\n",
    "# model's mode setting\n",
    "model.train()\n",
    "\n",
    "# model initialization\n",
    "model.zero_grad()\n",
    "train_loss = 0.0\n",
    "\n",
    "# train\n",
    "for i, (X, y) in enumerate(trainloader):\n",
    "\n",
    "    X = X.transpose(0, 1).float().to(args.device)\n",
    "    X = X.unsqueeze(2)\n",
    "\n",
    "    \n",
    "    \n",
    "    y = y.transpose(0, 1).float().to(args.device)\n",
    "    y_true = y.unsqueeze(2)\n",
    "    \n",
    "    print(X.shape, y.shape)\n",
    "    \n",
    "    \n",
    "    # zero the gradient\n",
    "    optimizer.zero_grad()\n",
    "\n",
    "    # en-decoder outputs tensor \n",
    "    y_pred = model(X, y_true)\n",
    "    \n",
    "    print(y_pred.shape)\n",
    "    # compute the loss \n",
    "    loss = loss_fn(y_true, y_pred)\n",
    "\n",
    "    #backpropagation\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "    # get the batch loss\n",
    "    train_loss += loss.item()\n",
    "\n",
    "train_loss = train_loss / len(trainloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def train(model, partition, optimizer, loss_fn, args):\n",
    "    ''' model training '''\n",
    "    \n",
    "   \n",
    "    # data load\n",
    "    trainloader = DataLoader(partition['train'], batch_size = args.batch_size, shuffle = True, drop_last = True)\n",
    "    \n",
    "    # model's mode setting\n",
    "    model.train()\n",
    "\n",
    "    # model initialization\n",
    "    model.zero_grad()\n",
    "    train_loss = 0.0\n",
    "    \n",
    "    # train\n",
    "    for i, (X, y) in enumerate(trainloader):\n",
    "        \n",
    "        X = X.transpose(0, 1).float().to(args.device)\n",
    "        X = X.unsqueeze(2)\n",
    "\n",
    "\n",
    "\n",
    "        y = y.transpose(0, 1).float().to(args.device)\n",
    "        y_true = y.unsqueeze(2)\n",
    "        \n",
    "        # zero the gradient\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # en-decoder outputs tensor \n",
    "        y_pred = model(X, y_true)\n",
    "\n",
    "        # compute the loss \n",
    "        loss = loss_fn(y_true, y_pred)\n",
    "        \n",
    "        #backpropagation\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        # get the batch loss\n",
    "        train_loss += loss.item()\n",
    "        \n",
    "    train_loss = train_loss / len(trainloader)\n",
    "    return model, train_loss\n",
    "\n",
    "\n",
    "def validate(model, partition, loss_fn, args):\n",
    "    ''' model validate '''\n",
    "    \n",
    "    # data load\n",
    "    valloader = DataLoader(partition['val'], batch_size = args.batch_size, shuffle = False, drop_last = True)\n",
    "    \n",
    "    # model's mode setting\n",
    "    model.eval()\n",
    "    val_loss = 0.0\n",
    "    \n",
    "    # evaluate\n",
    "    with torch.no_grad():\n",
    "        for i, (X, y) in enumerate(valloader):\n",
    "            \n",
    "            X = X.transpose(0, 1).float().to(args.device)\n",
    "            X = X.unsqueeze(2)\n",
    "\n",
    "\n",
    "\n",
    "            y = y.transpose(0, 1).float().to(args.device)\n",
    "            y_true = y.unsqueeze(2)\n",
    "\n",
    "            # en-decoder outputs tensor \n",
    "            y_pred = model(X, None)\n",
    "            \n",
    "            # compute the loss \n",
    "            loss = loss_fn(y_true, y_pred)\n",
    "\n",
    "            # get the batch loss\n",
    "            val_loss += loss.item()\n",
    "            \n",
    "    val_loss = val_loss / len(valloader)\n",
    " \n",
    "    return val_loss\n",
    "\n",
    "\n",
    "def test(model, partition, scaler, args):\n",
    "    ''' model test '''\n",
    "    \n",
    "    # data load\n",
    "    testloader = DataLoader(partition['test'], batch_size = 1, shuffle = False, drop_last = False)\n",
    "    \n",
    "    # model's mode setting\n",
    "    model.eval()\n",
    "    \n",
    "    test_mae = 0.0\n",
    "    score_list = list()\n",
    "    item_loss_list = list()\n",
    "    \n",
    "    # evaluate\n",
    "    with torch.no_grad():\n",
    "        for i, (X, y) in enumerate(testloader):\n",
    "            \n",
    "            X = X.transpose(0, 1).float().to(args.device)\n",
    "            X = X.unsqueeze(2)\n",
    "\n",
    "\n",
    "\n",
    "            y = y.transpose(0, 1).float().to(args.device)\n",
    "            y_true = y\n",
    "            \n",
    "            # en-decoder outputs tensor \n",
    "            y_pred = model(X, None)\n",
    "            y_pred = y_pred.squeeze(2)\n",
    "            \n",
    "            # y values to cpu\n",
    "            y_true = y_true.cpu().detach().numpy()\n",
    "            y_pred = y_pred.cpu().detach().numpy()\n",
    "\n",
    "            # get the batch loss\n",
    "            test_mae += mean_absolute_error(y_true, y_pred)\n",
    "#             score = r2_score(y_pred = y_pred.transpose(), y_true = y_true.transpose(), multioutput = 'uniform_average')\n",
    "#             score_list.append(score)\n",
    "            item_loss_list.append(abs(y_true - y_pred))\n",
    "            \n",
    "            \n",
    "    test_mae /= len(testloader)\n",
    "#     score /= len(testloader)\n",
    "    \n",
    "    return test_mae, item_loss_list\n",
    "\n",
    "\n",
    "def experiment(partition, args):\n",
    "\n",
    "    # Encoder\n",
    "    enc = Encoder(args.input_dim, args.hidden_dim, args.num_layers)\n",
    "    \n",
    "    # Decoder\n",
    "    dec = Decoder(args.output_dim, args.hidden_dim, args.num_layers, args.dropout, args.use_bn)\n",
    "    \n",
    "    # Seq2Seq model\n",
    "    model = Seq2Seq(enc, dec, args.device, args.target_len, args.training_prediction, args.teacher_forcing_ratio)\n",
    "    model.to(args.device)\n",
    "    \n",
    "    loss_fn = nn.MSELoss()\n",
    "    optimizer = optim.Adam(model.parameters(), lr=args.learning_rate, weight_decay=args.L2_rate)\n",
    "    \n",
    "    # epoch-wise loss\n",
    "    train_losses = []\n",
    "    val_losses = []\n",
    "\n",
    "    for epoch in range(args.num_epoch):\n",
    "        \n",
    "        start_time = time.time()\n",
    "\n",
    "        model, train_loss = train(model, partition, optimizer, loss_fn, args)\n",
    "        val_loss = validate(model, partition, loss_fn, args)\n",
    "        \n",
    "        end_time = time.time()\n",
    "        \n",
    "        # add epoch loss\n",
    "        train_losses.append(train_loss)\n",
    "        val_losses.append(val_loss)\n",
    "        \n",
    "        print('Epoch {},Loss(train/val) {:.3f}/{:.3f}. Took {:.2f} sec'.format(epoch+1, train_loss, val_loss, end_time-start_time))\n",
    "    \n",
    "    # test part\n",
    "    # test_mae, item_loss_list = test(model, partition, scaler, args)\n",
    "    \n",
    "    # ======= Add Result to Dictionary ======= #\n",
    "    result = {}\n",
    "    \n",
    "    result['train_losses'] = train_losses #epoch 수에 의존\n",
    "    result['val_losses'] = val_losses \n",
    "    \n",
    "    #result['test_mae'] = test_mae.round(3).item()\n",
    "    \n",
    "    # result['r2'] = np.array(score_list).mean().round(3)\n",
    "    # item_loss = np.array(item_loss_list).mean(axis=0).mean(axis=0).astype(int)\n",
    "    # item_loss = list([int(x) for x in item_loss])\n",
    "    # result['item_loss'] = item_loss\n",
    "     \n",
    "    return vars(args), result, model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1,Loss(train/val) 0.683/2503.568. Took 6.82 sec\n",
      "Epoch 2,Loss(train/val) 0.619/39.267. Took 6.57 sec\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-31-270c68a0fb56>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mexperiment\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpartition\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-30-d429d912ab31>\u001b[0m in \u001b[0;36mexperiment\u001b[1;34m(partition, args)\u001b[0m\n\u001b[0;32m     22\u001b[0m         \u001b[0mstart_time\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     23\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 24\u001b[1;33m         \u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_loss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpartition\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     25\u001b[0m         \u001b[0mval_loss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalidate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpartition\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     26\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-27-1f44411a89a9>\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(model, partition, optimizer, loss_fn, args)\u001b[0m\n\u001b[0;32m     34\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m         \u001b[1;31m#backpropagation\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 36\u001b[1;33m         \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     37\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\venv\\lib\\site-packages\\torch\\tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[0;32m    243\u001b[0m                 \u001b[0mcreate_graph\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    244\u001b[0m                 inputs=inputs)\n\u001b[1;32m--> 245\u001b[1;33m         \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    246\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    247\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\venv\\lib\\site-packages\\torch\\autograd\\__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[0;32m    145\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[0;32m    146\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 147\u001b[1;33m         allow_unreachable=True, accumulate_grad=True)  # allow_unreachable flag\n\u001b[0m\u001b[0;32m    148\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    149\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "experiment(partition, args)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}