train_cnn_gru_shifter_table_3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Subset, DataLoader, ConcatDataset\n",
    "from mouse_model.data_utils_new import MouseDatasetSegNewBehav\n",
    "import numpy as np\n",
    "from mouse_model.evaluation import cor_in_time\n",
    "from sklearn.metrics import r2_score, mean_squared_error\n",
    "import random\n",
    "import os\n",
    "import time\n",
    "import torch.nn.init as init\n",
    "from torch.nn import functional as F\n",
    "from kornia.geometry.transform import get_affine_matrix2d, warp_affine\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Shifter(nn.Module):\n",
    "    def __init__(self, input_dim=4, output_dim=3, hidden_dim=256, seq_len=8):\n",
    "        super().__init__()\n",
    "        self.seq_len = seq_len\n",
    "        self.input_dim = input_dim\n",
    "        self.output_dim = output_dim\n",
    "        \n",
    "        self.layers = nn.Sequential(\n",
    "            nn.BatchNorm1d(input_dim),\n",
    "            nn.Linear(input_dim, hidden_dim),\n",
    "            nn.BatchNorm1d(hidden_dim),\n",
    "            nn.Tanh(),\n",
    "            nn.Linear(hidden_dim, hidden_dim),\n",
    "            nn.BatchNorm1d(hidden_dim),\n",
    "            nn.Tanh(),\n",
    "            nn.Linear(hidden_dim, output_dim),\n",
    "            nn.Tanh(),\n",
    "        )\n",
    "        self.bias = nn.Parameter(torch.zeros(3))\n",
    "    def forward(self, x):\n",
    "        x = x.reshape(-1,self.input_dim )\n",
    "        x = self.layers(x)\n",
    "        x0 = (x[...,0] + self.bias[0]) * 80/5.5\n",
    "        x1 = (x[...,1] + self.bias[1]) * 60/5.5\n",
    "        x2 = (x[...,2] + self.bias[2]) * 180/4\n",
    "        x = torch.stack([x0, x1, x2], dim=-1)\n",
    "        x = x.reshape(-1,self.seq_len,self.output_dim)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Shifter(\n",
       "  (layers): Sequential(\n",
       "    (0): BatchNorm1d(4, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (1): Linear(in_features=4, out_features=256, bias=True)\n",
       "    (2): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (3): Tanh()\n",
       "    (4): Linear(in_features=256, out_features=256, bias=True)\n",
       "    (5): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (6): Tanh()\n",
       "    (7): Linear(in_features=256, out_features=3, bias=True)\n",
       "    (8): Tanh()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Shifter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PrintLayer(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super(PrintLayer, self).__init__()\n",
    "    \n",
    "    def forward(self, x):\n",
    "        print(x.shape)\n",
    "        return x\n",
    "    \n",
    "def size_helper(in_length, kernel_size, padding=0, dilation=1, stride=1):\n",
    "    # https://pytorch.org/docs/stable/generated/torch.nn.Conv2d.html#torch.nn.Conv2d\n",
    "    res = in_length + 2 * padding - dilation * (kernel_size - 1) - 1\n",
    "    res /= stride\n",
    "    res += 1\n",
    "    return np.floor(res)\n",
    "\n",
    "# CNN, the last fully connected layer maps to output_dim\n",
    "class VisualEncoder(nn.Module):\n",
    "    \n",
    "    def __init__(self, output_dim, input_shape=(60, 80), k1=7, k2=7, k3=7):\n",
    "        \n",
    "        super().__init__()\n",
    "        \n",
    "        self.input_shape = (60, 80)\n",
    "        out_shape_0 = size_helper(in_length=input_shape[0], kernel_size=k1, stride=2)\n",
    "        out_shape_0 = size_helper(in_length=out_shape_0, kernel_size=k2, stride=2)\n",
    "        out_shape_0 = size_helper(in_length=out_shape_0, kernel_size=k3, stride=2)\n",
    "        out_shape_1 = size_helper(in_length=input_shape[1], kernel_size=k1, stride=2)\n",
    "        out_shape_1 = size_helper(in_length=out_shape_1, kernel_size=k2, stride=2)\n",
    "        out_shape_1 = size_helper(in_length=out_shape_1, kernel_size=k3, stride=2)\n",
    "        self.output_shape = (int(out_shape_0), int(out_shape_1)) # shape of the final feature map\n",
    "        \n",
    "        self.layers = nn.Sequential(\n",
    "            nn.Conv2d(in_channels=1, out_channels=128, kernel_size=k1, stride=2),\n",
    "            nn.BatchNorm2d(128),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(p=0.5),\n",
    "            nn.Conv2d(in_channels=128, out_channels=64, kernel_size=k2, stride=2),\n",
    "            nn.BatchNorm2d(64),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(p=0.5),\n",
    "            nn.Conv2d(in_channels=64, out_channels=32, kernel_size=k3, stride=2),\n",
    "            nn.BatchNorm2d(32),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(p=0.5),\n",
    "            nn.Flatten(),\n",
    "            nn.Linear(480, output_dim)\n",
    "        )\n",
    "        \n",
    "    def forward(self, x):\n",
    "\n",
    "        x = self.layers(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "    \n",
    "# may consider adding an activation after linear\n",
    "class BehavEncoder(nn.Module):\n",
    "    \n",
    "    def __init__(self, behav_dim, output_dim):\n",
    "        \n",
    "        super().__init__()\n",
    "        \n",
    "        self.layers = nn.Sequential(\n",
    "            nn.BatchNorm1d(behav_dim),\n",
    "            nn.Linear(behav_dim, output_dim),\n",
    "        )\n",
    "        \n",
    "    def forward(self, x):\n",
    "\n",
    "        x = self.layers(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "class LSTMPerNeuronCombiner(nn.Module):\n",
    "    \n",
    "    def __init__(self, num_neurons, behav_dim, k1, k2, k3, seq_len, hidden_size=512):\n",
    "        \n",
    "        super().__init__()\n",
    "        \n",
    "        self.seq_len = seq_len\n",
    "        self.num_neurons = num_neurons\n",
    "        self.shifter = Shifter(seq_len = seq_len)\n",
    "        self.visual_encoder = VisualEncoder(output_dim=num_neurons, k1=k1, k2=k2, k3=k3)\n",
    "        self.behav_encoder = BehavEncoder(behav_dim=behav_dim, output_dim=num_neurons)\n",
    "        self.bn = nn.BatchNorm1d(3) # apply bn to vis_feats, beh_feats, prod\n",
    "        self.lstm_net = nn.GRU(input_size=num_neurons*3, hidden_size=hidden_size, num_layers=1, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_size, num_neurons)\n",
    "        self.softplus = nn.Softplus() # we could also do relu or elu offset by 1\n",
    "        \n",
    "    def forward(self, images, behav):\n",
    "        if args.shifter:\n",
    "            bs = images.size()[0]\n",
    "            behav_shifter = torch.concat((behav[...,4].unsqueeze(-1),   # theta\n",
    "                                          behav[...,3].unsqueeze(-1),   # phi\n",
    "                                          behav[...,1].unsqueeze(-1),  # pitch\n",
    "                                         behav[...,2].unsqueeze(-1),  # roll\n",
    "                                         ), dim=-1)  \n",
    "            shift_param = self.shifter(behav_shifter)  \n",
    "            shift_param = shift_param.reshape(-1,3)\n",
    "            scale_param = torch.ones_like(shift_param[..., 0:2]).to(shift_param.device)\n",
    "            affine_mat = get_affine_matrix2d(\n",
    "                                            translations=shift_param[..., 0:2] ,\n",
    "                                             scale = scale_param, \n",
    "                                             center =torch.repeat_interleave(torch.tensor([[30,40]], dtype=torch.float), \n",
    "                                                                            bs*self.seq_len, dim=0).to(shift_param.device), \n",
    "                                             angle=shift_param[..., 2])\n",
    "            affine_mat = affine_mat[:, :2, :]\n",
    "            images = warp_affine(images.reshape(-1,1,60,80), affine_mat, dsize=(60,80)).reshape(bs, self.seq_len,1,60,80)\n",
    "        \n",
    "        # get visual behavioral features in time\n",
    "        vis_beh_feats = []\n",
    "        for i in range(self.seq_len):\n",
    "            v = self.visual_encoder(images[:, i, :, :, :])\n",
    "            b = self.behav_encoder(behav[:, i, :])\n",
    "            vb = v * b\n",
    "            vis_beh_feat = torch.stack([v, b, vb], axis=1)\n",
    "            vis_beh_feat = self.bn(vis_beh_feat)\n",
    "            vis_beh_feats.append(vis_beh_feat)\n",
    "        vis_beh_feats = torch.stack(vis_beh_feats, axis=1)\n",
    "        \n",
    "        # flatten features to (batch_size, seq_len, num_neurons*3)\n",
    "        vis_beh_feats = torch.flatten(vis_beh_feats, start_dim=2)\n",
    "        \n",
    "        # get LSTM output\n",
    "        output, _ = self.lstm_net(vis_beh_feats)\n",
    "        output = output[:, -1, :] # extract the last hidden state\n",
    "        \n",
    "        # fully connected layer and activation function\n",
    "        output = self.fc(output)\n",
    "        pred_spikes = self.softplus(output)\n",
    "\n",
    "        return pred_spikes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "class Args:\n",
    "    \n",
    "    seed = 0\n",
    "    file_id = None\n",
    "    epochs = 50\n",
    "    batch_size = 256\n",
    "    learning_rate = 0.0002\n",
    "    l1_reg_w = 1\n",
    "    seq_len = None\n",
    "    num_neurons = None\n",
    "    behav_mode = None\n",
    "    behav_dim = None\n",
    "    best_val_path = None\n",
    "    best_train_path = None\n",
    "    vid_type = \"vid_mean\"\n",
    "    segment_num = 10\n",
    "    hidden_size = 512\n",
    "    shifter = True\n",
    "    \n",
    "args=Args()\n",
    "\n",
    "def set_random_seed(seed: int, deterministic: bool = True):\n",
    "    # from nnfabrik package\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    if deterministic:\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "    torch.manual_seed(seed)  # this sets both CPU and CUDA seeds for PyTorch\n",
    "\n",
    "seed = args.seed\n",
    "set_random_seed(seed)\n",
    "random.seed(seed)\n",
    "os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed(seed)\n",
    "\n",
    "torch.cuda.empty_cache()\n",
    "print(torch.cuda.is_available())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_train_val_ds():\n",
    "    ds_list = [MouseDatasetSegNewBehav(file_id=args.file_id, segment_num=args.segment_num, seg_idx=i, data_split=\"train\", \n",
    "                               vid_type=args.vid_type, seq_len=args.seq_len, predict_offset=1, \n",
    "                                       behav_mode=args.behav_mode, norm_mode=\"01\") \n",
    "               for i in range(args.segment_num)]\n",
    "    train_ds, val_ds = [], []\n",
    "    for ds in ds_list:\n",
    "        train_ratio = 0.8\n",
    "        train_ds_len = int(len(ds) * train_ratio)\n",
    "        train_ds.append(Subset(ds, np.arange(0, train_ds_len, 1)))\n",
    "        val_ds.append(Subset(ds, np.arange(train_ds_len, len(ds), 1)))\n",
    "    train_ds = ConcatDataset(train_ds)\n",
    "    val_ds = ConcatDataset(val_ds)\n",
    "    print(len(train_ds), len(val_ds))\n",
    "    return train_ds, val_ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_test_ds():\n",
    "    test_ds = [MouseDatasetSegNewBehav(file_id=args.file_id, segment_num=args.segment_num, seg_idx=i, data_split=\"test\", \n",
    "                               vid_type=args.vid_type, seq_len=args.seq_len, predict_offset=1, \n",
    "                                       behav_mode=args.behav_mode, norm_mode=\"01\") \n",
    "               for i in range(args.segment_num)]\n",
    "    test_ds = ConcatDataset(test_ds)\n",
    "    return test_ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model():\n",
    "    \n",
    "    torch.manual_seed(args.seed)\n",
    "    if torch.cuda.is_available():\n",
    "        torch.cuda.manual_seed(args.seed)\n",
    "    \n",
    "    train_ds, val_ds = load_train_val_ds()\n",
    "\n",
    "    train_dataloader = DataLoader(dataset=train_ds, batch_size=args.batch_size, shuffle=True, num_workers=8)\n",
    "    val_dataloader = DataLoader(dataset=val_ds, batch_size=args.batch_size, shuffle=False, num_workers=8)\n",
    "    \n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate)\n",
    "\n",
    "    best_train_spike_loss = np.inf\n",
    "    best_val_spike_loss = np.inf\n",
    "    train_loss_list = []\n",
    "    val_loss_list = []\n",
    "\n",
    "    # start training\n",
    "    ct = 0\n",
    "    \n",
    "    for epoch in range(args.epochs):\n",
    "\n",
    "        print(\"Start epoch\", epoch)\n",
    "\n",
    "        model.train()\n",
    "\n",
    "        epoch_train_loss, epoch_train_spike_loss = 0, 0\n",
    "\n",
    "        for (image, behav, spikes) in train_dataloader:\n",
    "\n",
    "            image, behav, spikes = image.to(device), behav.to(device), spikes.to(device)\n",
    "            \n",
    "            pred = model(image, behav)\n",
    "\n",
    "            spike_loss = nn.functional.poisson_nll_loss(pred, spikes, reduction='mean', log_input=False)\n",
    "            \n",
    "            l1_reg, l1_reg_num_param = 0.0, 0\n",
    "            for name, param in model.named_parameters():\n",
    "                if name == \"behav_encoder.layers.1.weight\":\n",
    "                    l1_reg += param.abs().sum()\n",
    "                    l1_reg_num_param += param.shape[0]*param.shape[1]\n",
    "            l1_reg /= l1_reg_num_param\n",
    "\n",
    "            total_loss = spike_loss + args.l1_reg_w * l1_reg\n",
    "\n",
    "            epoch_train_loss += total_loss.item()\n",
    "            epoch_train_spike_loss += spike_loss.item()\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            total_loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "        epoch_train_loss = epoch_train_loss / len(train_dataloader)\n",
    "        epoch_train_spike_loss = epoch_train_spike_loss / len(train_dataloader)\n",
    "\n",
    "        train_loss_list.append(epoch_train_loss)\n",
    "        \n",
    "        print(\"Epoch {} train loss: {}\".format(epoch, epoch_train_loss))\n",
    "\n",
    "        if epoch_train_spike_loss < best_train_spike_loss:\n",
    "\n",
    "            print(\"save train model at epoch\", epoch)\n",
    "            torch.save(model.state_dict(), args.best_train_path)\n",
    "            best_train_spike_loss = epoch_train_spike_loss\n",
    "\n",
    "        model.eval()\n",
    "\n",
    "        epoch_val_spike_loss = 0\n",
    "\n",
    "        with torch.no_grad():      \n",
    "\n",
    "            for (image, behav, spikes) in val_dataloader:\n",
    "\n",
    "                image, behav, spikes = image.to(device), behav.to(device), spikes.to(device)\n",
    "\n",
    "                pred = model(image, behav)\n",
    "\n",
    "                loss = nn.functional.poisson_nll_loss(pred, spikes, reduction='mean', log_input=False)\n",
    "\n",
    "                epoch_val_spike_loss += loss.item()\n",
    "\n",
    "        epoch_val_spike_loss = epoch_val_spike_loss / len(val_dataloader)\n",
    "\n",
    "        val_loss_list.append(epoch_val_spike_loss)\n",
    "        \n",
    "        print(\"Epoch {} val loss: {}\".format(epoch, epoch_val_spike_loss))\n",
    "        \n",
    "        if epoch_val_spike_loss < best_val_spike_loss:\n",
    "            ct = 0\n",
    "\n",
    "            print(\"save val model at epoch\", epoch)\n",
    "            torch.save(model.state_dict(), args.best_val_path)\n",
    "            best_val_spike_loss = epoch_val_spike_loss\n",
    "        else:\n",
    "            ct += 1\n",
    "            if ct >=5:\n",
    "                print('stop training')\n",
    "                break\n",
    "\n",
    "        print(\"End epoch\", epoch)\n",
    "        \n",
    "    return train_loss_list, val_loss_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "070921_J553RT all_prod 1\n",
      "30120 7540\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.7431484688136537\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.6559784253438313\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.676811802690312\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.6420734246571859\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.6523936643438825\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.6371280570824941\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.6362312577538571\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.6290084004402161\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.6236386743642516\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.6207810123761495\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.6161649297859709\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.6180520315965017\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.6108139854366497\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.6163019140561422\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.6068727257898299\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.6146854817867279\n",
      "save val model at epoch 7\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.6036328107623731\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.6127613226572672\n",
      "save val model at epoch 8\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.6008077506291665\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.6080401082833607\n",
      "save val model at epoch 9\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.5983858608593375\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.6066788693269094\n",
      "save val model at epoch 10\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.5961209429522693\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.6062704642613729\n",
      "save val model at epoch 11\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.5939227852780941\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.6045370201269785\n",
      "save val model at epoch 12\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.5925275309611175\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.6032923062642416\n",
      "save val model at epoch 13\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.5908145950002185\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.6033066948254903\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.5892323356563762\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.6048796554406484\n",
      "End epoch 15\n",
      "Start epoch 16\n",
      "Epoch 16 train loss: 0.5877695679664612\n",
      "save train model at epoch 16\n",
      "Epoch 16 val loss: 0.6006186227003734\n",
      "save val model at epoch 16\n",
      "End epoch 16\n",
      "Start epoch 17\n",
      "Epoch 17 train loss: 0.5862837474224931\n",
      "save train model at epoch 17\n",
      "Epoch 17 val loss: 0.599836403131485\n",
      "save val model at epoch 17\n",
      "End epoch 17\n",
      "Start epoch 18\n",
      "Epoch 18 train loss: 0.5852212148197626\n",
      "save train model at epoch 18\n",
      "Epoch 18 val loss: 0.5994010329246521\n",
      "save val model at epoch 18\n",
      "End epoch 18\n",
      "Start epoch 19\n",
      "Epoch 19 train loss: 0.5837935689142195\n",
      "save train model at epoch 19\n",
      "Epoch 19 val loss: 0.5984606345494589\n",
      "save val model at epoch 19\n",
      "End epoch 19\n",
      "Start epoch 20\n",
      "Epoch 20 train loss: 0.5827492172435179\n",
      "save train model at epoch 20\n",
      "Epoch 20 val loss: 0.600040469566981\n",
      "End epoch 20\n",
      "Start epoch 21\n",
      "Epoch 21 train loss: 0.5813947592751455\n",
      "save train model at epoch 21\n",
      "Epoch 21 val loss: 0.5985481361548106\n",
      "End epoch 21\n",
      "Start epoch 22\n",
      "Epoch 22 train loss: 0.5805512798034539\n",
      "save train model at epoch 22\n",
      "Epoch 22 val loss: 0.6003512799739837\n",
      "End epoch 22\n",
      "Start epoch 23\n",
      "Epoch 23 train loss: 0.5792851852158368\n",
      "save train model at epoch 23\n",
      "Epoch 23 val loss: 0.5982806702454885\n",
      "save val model at epoch 23\n",
      "End epoch 23\n",
      "Start epoch 24\n",
      "Epoch 24 train loss: 0.5786375711529942\n",
      "save train model at epoch 24\n",
      "Epoch 24 val loss: 0.6014466106891632\n",
      "End epoch 24\n",
      "Start epoch 25\n",
      "Epoch 25 train loss: 0.5777574378555104\n",
      "save train model at epoch 25\n",
      "Epoch 25 val loss: 0.5990119636058807\n",
      "End epoch 25\n",
      "Start epoch 26\n",
      "Epoch 26 train loss: 0.5765309444928574\n",
      "save train model at epoch 26\n",
      "Epoch 26 val loss: 0.5974705139795939\n",
      "save val model at epoch 26\n",
      "End epoch 26\n",
      "Start epoch 27\n",
      "Epoch 27 train loss: 0.5756031401076559\n",
      "save train model at epoch 27\n",
      "Epoch 27 val loss: 0.60081214706103\n",
      "End epoch 27\n",
      "Start epoch 28\n",
      "Epoch 28 train loss: 0.574790521698483\n",
      "save train model at epoch 28\n",
      "Epoch 28 val loss: 0.5966964522997539\n",
      "save val model at epoch 28\n",
      "End epoch 28\n",
      "Start epoch 29\n",
      "Epoch 29 train loss: 0.5737625522128607\n",
      "save train model at epoch 29\n",
      "Epoch 29 val loss: 0.603026028474172\n",
      "End epoch 29\n",
      "Start epoch 30\n",
      "Epoch 30 train loss: 0.5732380121441211\n",
      "save train model at epoch 30\n",
      "Epoch 30 val loss: 0.5971440692742666\n",
      "End epoch 30\n",
      "Start epoch 31\n",
      "Epoch 31 train loss: 0.5721593484029932\n",
      "save train model at epoch 31\n",
      "Epoch 31 val loss: 0.5981256326039632\n",
      "End epoch 31\n",
      "Start epoch 32\n",
      "Epoch 32 train loss: 0.5711557046841766\n",
      "save train model at epoch 32\n",
      "Epoch 32 val loss: 0.5962706387043\n",
      "save val model at epoch 32\n",
      "End epoch 32\n",
      "Start epoch 33\n",
      "Epoch 33 train loss: 0.5706184390237776\n",
      "save train model at epoch 33\n",
      "Epoch 33 val loss: 0.5974632600943247\n",
      "End epoch 33\n",
      "Start epoch 34\n",
      "Epoch 34 train loss: 0.5698573756015907\n",
      "save train model at epoch 34\n",
      "Epoch 34 val loss: 0.5962005833784739\n",
      "save val model at epoch 34\n",
      "End epoch 34\n",
      "Start epoch 35\n",
      "Epoch 35 train loss: 0.5693220829559584\n",
      "save train model at epoch 35\n",
      "Epoch 35 val loss: 0.6040061930815379\n",
      "End epoch 35\n",
      "Start epoch 36\n",
      "Epoch 36 train loss: 0.5681257091336331\n",
      "save train model at epoch 36\n",
      "Epoch 36 val loss: 0.5964443425337473\n",
      "End epoch 36\n",
      "Start epoch 37\n",
      "Epoch 37 train loss: 0.5674225511187214\n",
      "save train model at epoch 37\n",
      "Epoch 37 val loss: 0.5996627728144328\n",
      "End epoch 37\n",
      "Start epoch 38\n",
      "Epoch 38 train loss: 0.5663870537685136\n",
      "save train model at epoch 38\n",
      "Epoch 38 val loss: 0.6027400771776835\n",
      "End epoch 38\n",
      "Start epoch 39\n",
      "Epoch 39 train loss: 0.5656849393400095\n",
      "save train model at epoch 39\n",
      "Epoch 39 val loss: 0.5964667359987895\n",
      "stop training\n",
      "070921_J553RT all_prod 2\n",
      "30120 7530\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.7294599706843748\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.650673900047938\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.6660893741300551\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.6316997786362966\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.6408072063478373\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.6242357095082601\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.6248593971890918\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.6158926288286845\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.6146502772630271\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.6136733194192251\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.6070271169735213\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.613171128431956\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.6019137269359524\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.609506360689799\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.5974625455120862\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.6062026043732961\n",
      "save val model at epoch 7\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.5941927928035542\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.6044411520163219\n",
      "save val model at epoch 8\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.59078369706364\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.6023576537768046\n",
      "save val model at epoch 9\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.5880819160049244\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.6000987768173218\n",
      "save val model at epoch 10\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.5852847937810219\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.5984567205111185\n",
      "save val model at epoch 11\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.5829391181468964\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.5978292326132456\n",
      "save val model at epoch 12\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.5808715062626337\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.5988364378611247\n",
      "End epoch 13\n",
      "Start epoch 14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 14 train loss: 0.578990152831805\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.5963725646336874\n",
      "save val model at epoch 14\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.5767980652340388\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.5945991237958272\n",
      "save val model at epoch 15\n",
      "End epoch 15\n",
      "Start epoch 16\n",
      "Epoch 16 train loss: 0.5747399436215223\n",
      "save train model at epoch 16\n",
      "Epoch 16 val loss: 0.5952917257944743\n",
      "End epoch 16\n",
      "Start epoch 17\n",
      "Epoch 17 train loss: 0.5733452893919864\n",
      "save train model at epoch 17\n",
      "Epoch 17 val loss: 0.5987995445728302\n",
      "End epoch 17\n",
      "Start epoch 18\n",
      "Epoch 18 train loss: 0.5714053819745274\n",
      "save train model at epoch 18\n",
      "Epoch 18 val loss: 0.5939799507459005\n",
      "save val model at epoch 18\n",
      "End epoch 18\n",
      "Start epoch 19\n",
      "Epoch 19 train loss: 0.569688734866805\n",
      "save train model at epoch 19\n",
      "Epoch 19 val loss: 0.5932040015856425\n",
      "save val model at epoch 19\n",
      "End epoch 19\n",
      "Start epoch 20\n",
      "Epoch 20 train loss: 0.5682554441993519\n",
      "save train model at epoch 20\n",
      "Epoch 20 val loss: 0.5962185164292654\n",
      "End epoch 20\n",
      "Start epoch 21\n",
      "Epoch 21 train loss: 0.5666924103841944\n",
      "save train model at epoch 21\n",
      "Epoch 21 val loss: 0.5952638705571492\n",
      "End epoch 21\n",
      "Start epoch 22\n",
      "Epoch 22 train loss: 0.5651213324676125\n",
      "save train model at epoch 22\n",
      "Epoch 22 val loss: 0.594878884156545\n",
      "End epoch 22\n",
      "Start epoch 23\n",
      "Epoch 23 train loss: 0.5635198491104578\n",
      "save train model at epoch 23\n",
      "Epoch 23 val loss: 0.5944202204545339\n",
      "End epoch 23\n",
      "Start epoch 24\n",
      "Epoch 24 train loss: 0.5619687432959929\n",
      "save train model at epoch 24\n",
      "Epoch 24 val loss: 0.5988235890865325\n",
      "stop training\n",
      "070921_J553RT all_prod 3\n",
      "30110 7530\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.7242964248535997\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.6501995424429575\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.6636842029579615\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.631713581085205\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.638577098058442\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.625767982006073\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.622314350584806\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.6160397509733836\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.6111757916919256\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.610942029953003\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.603308412483183\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.6065157334009806\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.5978217185553858\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.6052174349625905\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.5931099920959796\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.6026177326838176\n",
      "save val model at epoch 7\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.5894370407371198\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.601645306746165\n",
      "save val model at epoch 8\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.5860295745275789\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.5984321693579356\n",
      "save val model at epoch 9\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.5831972845530106\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.598052708307902\n",
      "save val model at epoch 10\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.5801922441539118\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.5970988313357035\n",
      "save val model at epoch 11\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.5778101394742222\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.5972041269143422\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.5756512905581522\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.5968624730904897\n",
      "save val model at epoch 13\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.5731206218064842\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.5964151004950206\n",
      "save val model at epoch 14\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.5706745502302202\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.5951782087484996\n",
      "save val model at epoch 15\n",
      "End epoch 15\n",
      "Start epoch 16\n",
      "Epoch 16 train loss: 0.569353270328651\n",
      "save train model at epoch 16\n",
      "Epoch 16 val loss: 0.5934673249721527\n",
      "save val model at epoch 16\n",
      "End epoch 16\n",
      "Start epoch 17\n",
      "Epoch 17 train loss: 0.5669933208974741\n",
      "save train model at epoch 17\n",
      "Epoch 17 val loss: 0.5953375657399496\n",
      "End epoch 17\n",
      "Start epoch 18\n",
      "Epoch 18 train loss: 0.5651857368016647\n",
      "save train model at epoch 18\n",
      "Epoch 18 val loss: 0.5934825877348582\n",
      "End epoch 18\n",
      "Start epoch 19\n",
      "Epoch 19 train loss: 0.5634268981925512\n",
      "save train model at epoch 19\n",
      "Epoch 19 val loss: 0.5930482149124146\n",
      "save val model at epoch 19\n",
      "End epoch 19\n",
      "Start epoch 20\n",
      "Epoch 20 train loss: 0.5612809945971279\n",
      "save train model at epoch 20\n",
      "Epoch 20 val loss: 0.5936392744382223\n",
      "End epoch 20\n",
      "Start epoch 21\n",
      "Epoch 21 train loss: 0.55968506710004\n",
      "save train model at epoch 21\n",
      "Epoch 21 val loss: 0.5939679185549418\n",
      "End epoch 21\n",
      "Start epoch 22\n",
      "Epoch 22 train loss: 0.5578532163369454\n",
      "save train model at epoch 22\n",
      "Epoch 22 val loss: 0.6024699687957764\n",
      "End epoch 22\n",
      "Start epoch 23\n",
      "Epoch 23 train loss: 0.5561975144733817\n",
      "save train model at epoch 23\n",
      "Epoch 23 val loss: 0.5941893617312114\n",
      "End epoch 23\n",
      "Start epoch 24\n",
      "Epoch 24 train loss: 0.5545054310459202\n",
      "save train model at epoch 24\n",
      "Epoch 24 val loss: 0.5938546240329743\n",
      "stop training\n",
      "070921_J553RT all_prod 4\n",
      "30100 7530\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.7208146038701979\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.6442032337188721\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.6612893615738821\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.6312688827514649\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.637837766590765\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.6208058357238769\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.6211705601821511\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.6160677433013916\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.6100111724966664\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.6125034193197886\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.6022338336807186\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.6059284508228302\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.5963085313974801\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.6034262935320537\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.5916792118953447\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.6032711843649546\n",
      "save val model at epoch 7\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.5878092070757333\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.5994347532590231\n",
      "save val model at epoch 8\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.5843392246860569\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.5982197403907776\n",
      "save val model at epoch 9\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.5813155931941534\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.5970502495765686\n",
      "save val model at epoch 10\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.5781783931336161\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.5952983140945435\n",
      "save val model at epoch 11\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.5757452017169887\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.5964547236760457\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.5729779964786464\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.5948145349820455\n",
      "save val model at epoch 13\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.5706772874977629\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.5958298663298289\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.5682063734127303\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.5929633915424347\n",
      "save val model at epoch 15\n",
      "End epoch 15\n",
      "Start epoch 16\n",
      "Epoch 16 train loss: 0.5661240119045063\n",
      "save train model at epoch 16\n",
      "Epoch 16 val loss: 0.5934797485669454\n",
      "End epoch 16\n",
      "Start epoch 17\n",
      "Epoch 17 train loss: 0.5637658708176371\n",
      "save train model at epoch 17\n",
      "Epoch 17 val loss: 0.5943336109320323\n",
      "End epoch 17\n",
      "Start epoch 18\n",
      "Epoch 18 train loss: 0.5616058287984234\n",
      "save train model at epoch 18\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 18 val loss: 0.5924500584602356\n",
      "save val model at epoch 18\n",
      "End epoch 18\n",
      "Start epoch 19\n",
      "Epoch 19 train loss: 0.5598759757260144\n",
      "save train model at epoch 19\n",
      "Epoch 19 val loss: 0.5920013944307964\n",
      "save val model at epoch 19\n",
      "End epoch 19\n",
      "Start epoch 20\n",
      "Epoch 20 train loss: 0.5574767670388949\n",
      "save train model at epoch 20\n",
      "Epoch 20 val loss: 0.5945514837900797\n",
      "End epoch 20\n",
      "Start epoch 21\n",
      "Epoch 21 train loss: 0.5558825231204598\n",
      "save train model at epoch 21\n",
      "Epoch 21 val loss: 0.594162768125534\n",
      "End epoch 21\n",
      "Start epoch 22\n",
      "Epoch 22 train loss: 0.5538186228881448\n",
      "save train model at epoch 22\n",
      "Epoch 22 val loss: 0.5940540770689646\n",
      "End epoch 22\n",
      "Start epoch 23\n",
      "Epoch 23 train loss: 0.5519520044326782\n",
      "save train model at epoch 23\n",
      "Epoch 23 val loss: 0.5938971916834513\n",
      "End epoch 23\n",
      "Start epoch 24\n",
      "Epoch 24 train loss: 0.5501341254024182\n",
      "save train model at epoch 24\n",
      "Epoch 24 val loss: 0.5956300477186839\n",
      "stop training\n",
      "070921_J553RT all_prod 5\n",
      "30090 7530\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.7205607249575147\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.6496146857738495\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.6612001617076033\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.6304642558097839\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.6361523427195468\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.6215948601563771\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.6195528860819541\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.6137849112351735\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.6087566932379189\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.6106064677238464\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.6011770165572732\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.6083682000637054\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.5950731599735002\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.6063943088054657\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.590452501329325\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.6010210692882538\n",
      "save val model at epoch 7\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.5861979051161621\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.5992944081624348\n",
      "save val model at epoch 8\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.5826437639988075\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.5977775454521179\n",
      "save val model at epoch 9\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.579620776540142\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.5954374770323435\n",
      "save val model at epoch 10\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.5766292052753901\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.5946158905824025\n",
      "save val model at epoch 11\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.5734105534472708\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.5934387087821961\n",
      "save val model at epoch 12\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.5707725230920113\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.5937506020069122\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.568381107459634\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.593577512105306\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.5660734267558082\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.594410224755605\n",
      "End epoch 15\n",
      "Start epoch 16\n",
      "Epoch 16 train loss: 0.563657475224996\n",
      "save train model at epoch 16\n",
      "Epoch 16 val loss: 0.5923452317714691\n",
      "save val model at epoch 16\n",
      "End epoch 16\n",
      "Start epoch 17\n",
      "Epoch 17 train loss: 0.5614581638473576\n",
      "save train model at epoch 17\n",
      "Epoch 17 val loss: 0.5927557945251465\n",
      "End epoch 17\n",
      "Start epoch 18\n",
      "Epoch 18 train loss: 0.559386679681681\n",
      "save train model at epoch 18\n",
      "Epoch 18 val loss: 0.592365046342214\n",
      "End epoch 18\n",
      "Start epoch 19\n",
      "Epoch 19 train loss: 0.557144055427131\n",
      "save train model at epoch 19\n",
      "Epoch 19 val loss: 0.5936063468456269\n",
      "End epoch 19\n",
      "Start epoch 20\n",
      "Epoch 20 train loss: 0.5552020077988252\n",
      "save train model at epoch 20\n",
      "Epoch 20 val loss: 0.5927198270956675\n",
      "End epoch 20\n",
      "Start epoch 21\n",
      "Epoch 21 train loss: 0.552988322609562\n",
      "save train model at epoch 21\n",
      "Epoch 21 val loss: 0.5919403076171875\n",
      "save val model at epoch 21\n",
      "End epoch 21\n",
      "Start epoch 22\n",
      "Epoch 22 train loss: 0.5509789742655673\n",
      "save train model at epoch 22\n",
      "Epoch 22 val loss: 0.5946496943632762\n",
      "End epoch 22\n",
      "Start epoch 23\n",
      "Epoch 23 train loss: 0.5491709805141061\n",
      "save train model at epoch 23\n",
      "Epoch 23 val loss: 0.5933773497740428\n",
      "End epoch 23\n",
      "Start epoch 24\n",
      "Epoch 24 train loss: 0.5471013011568684\n",
      "save train model at epoch 24\n",
      "Epoch 24 val loss: 0.5957662522792816\n",
      "End epoch 24\n",
      "Start epoch 25\n",
      "Epoch 25 train loss: 0.5453729967949754\n",
      "save train model at epoch 25\n",
      "Epoch 25 val loss: 0.5944469769795736\n",
      "End epoch 25\n",
      "Start epoch 26\n",
      "Epoch 26 train loss: 0.5435363079531718\n",
      "save train model at epoch 26\n",
      "Epoch 26 val loss: 0.5947765032450358\n",
      "stop training\n",
      "070921_J553RT all_prod 6\n",
      "30080 7530\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.7201416199490175\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.6511828660964966\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.6631755162093599\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.6277598639329275\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.6339289864241067\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.6183847387631735\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.618582075430175\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.6159444749355316\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.607401710445598\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.607178250948588\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.5994011007123075\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.6064220825831096\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.5932328145382768\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.6050206542015075\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.588717268180039\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.6007123033205668\n",
      "save val model at epoch 7\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.5844072285345046\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.5973321874936421\n",
      "save val model at epoch 8\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.5806964625746517\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.5975806732972463\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.5774952713715829\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.5947018325328827\n",
      "save val model at epoch 10\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.5743909437777632\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.59719926516215\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.5716085540036023\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.5941221992174784\n",
      "save val model at epoch 12\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.5687670096502466\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.5929822166760762\n",
      "save val model at epoch 13\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.5662385230347261\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.5922987500826518\n",
      "save val model at epoch 14\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.5639769056085813\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.594006218512853\n",
      "End epoch 15\n",
      "Start epoch 16\n",
      "Epoch 16 train loss: 0.5615710813110157\n",
      "save train model at epoch 16\n",
      "Epoch 16 val loss: 0.5924253900845845\n",
      "End epoch 16\n",
      "Start epoch 17\n",
      "Epoch 17 train loss: 0.5590397043753479\n",
      "save train model at epoch 17\n",
      "Epoch 17 val loss: 0.5933414359887441\n",
      "End epoch 17\n",
      "Start epoch 18\n",
      "Epoch 18 train loss: 0.5570525799767446\n",
      "save train model at epoch 18\n",
      "Epoch 18 val loss: 0.5937402625878652\n",
      "End epoch 18\n",
      "Start epoch 19\n",
      "Epoch 19 train loss: 0.5546001726287907\n",
      "save train model at epoch 19\n",
      "Epoch 19 val loss: 0.5927979330221812\n",
      "stop training\n",
      "070921_J553RT all_prod 7\n",
      "30080 7520\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.7210003011307474\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.6473994135856629\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.6613474796384068\n",
      "save train model at epoch 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 val loss: 0.6295513908068339\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.635275269969035\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.6196413139502207\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.6192219903913595\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.6134775300820668\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.607691914348279\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.6089808483918507\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.599547598321559\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.6051565845807393\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.5934569204257707\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.6022655745347341\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.5883054723173885\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.5993504981199901\n",
      "save val model at epoch 7\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.584256345942869\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.5979000508785248\n",
      "save val model at epoch 8\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.5802369047019441\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.5961639285087585\n",
      "save val model at epoch 9\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.5770312199148081\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.5946789105733236\n",
      "save val model at epoch 10\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.5739359471757534\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.5948779424031575\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.5709517279923972\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.5967180649439494\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.5679678275423535\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.5937965253988902\n",
      "save val model at epoch 13\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.5653929735644388\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.5940113961696625\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.5629122358257488\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.5942379852135976\n",
      "End epoch 15\n",
      "Start epoch 16\n",
      "Epoch 16 train loss: 0.5602735548706378\n",
      "save train model at epoch 16\n",
      "Epoch 16 val loss: 0.5921012600262959\n",
      "save val model at epoch 16\n",
      "End epoch 16\n",
      "Start epoch 17\n",
      "Epoch 17 train loss: 0.5575676413915925\n",
      "save train model at epoch 17\n",
      "Epoch 17 val loss: 0.5941604693730672\n",
      "End epoch 17\n",
      "Start epoch 18\n",
      "Epoch 18 train loss: 0.555461902739638\n",
      "save train model at epoch 18\n",
      "Epoch 18 val loss: 0.5930985470612844\n",
      "End epoch 18\n",
      "Start epoch 19\n",
      "Epoch 19 train loss: 0.5529891232312736\n",
      "save train model at epoch 19\n",
      "Epoch 19 val loss: 0.5942627668380738\n",
      "End epoch 19\n",
      "Start epoch 20\n",
      "Epoch 20 train loss: 0.5508367217193215\n",
      "save train model at epoch 20\n",
      "Epoch 20 val loss: 0.5932807485262553\n",
      "End epoch 20\n",
      "Start epoch 21\n",
      "Epoch 21 train loss: 0.5488300899327812\n",
      "save train model at epoch 21\n",
      "Epoch 21 val loss: 0.5954864084720611\n",
      "stop training\n",
      "070921_J553RT all_prod 8\n",
      "30070 7520\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.7175909972797005\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.6432381014029185\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.6570650258306729\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.625092742840449\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.6328251018362531\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.6195721089839935\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.6175564860893508\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.611568558216095\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.6063035448728982\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.6084955632686615\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.5982688321905621\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.6059487740198771\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.5923406360513073\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.6012339770793915\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.5875390141697253\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.5996393700440724\n",
      "save val model at epoch 7\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.5832470943362026\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.5971541941165924\n",
      "save val model at epoch 8\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.5796648819567793\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.5969023525714874\n",
      "save val model at epoch 9\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.5762449209972963\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.5953978757063548\n",
      "save val model at epoch 10\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.5727788831217814\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.5945549527804057\n",
      "save val model at epoch 11\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.5701052283836623\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.5938581089178722\n",
      "save val model at epoch 12\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.5670178451780545\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.5948886533578237\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.5637908481945426\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.5935376405715942\n",
      "save val model at epoch 14\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.5616725415496503\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.5947992901007334\n",
      "End epoch 15\n",
      "Start epoch 16\n",
      "Epoch 16 train loss: 0.5591834081431567\n",
      "save train model at epoch 16\n",
      "Epoch 16 val loss: 0.5930446982383728\n",
      "save val model at epoch 16\n",
      "End epoch 16\n",
      "Start epoch 17\n",
      "Epoch 17 train loss: 0.5565579336578563\n",
      "save train model at epoch 17\n",
      "Epoch 17 val loss: 0.5933965941270193\n",
      "End epoch 17\n",
      "Start epoch 18\n",
      "Epoch 18 train loss: 0.5541265611931429\n",
      "save train model at epoch 18\n",
      "Epoch 18 val loss: 0.5940377891063691\n",
      "End epoch 18\n",
      "Start epoch 19\n",
      "Epoch 19 train loss: 0.5515423669653424\n",
      "save train model at epoch 19\n",
      "Epoch 19 val loss: 0.5945320347944896\n",
      "End epoch 19\n",
      "Start epoch 20\n",
      "Epoch 20 train loss: 0.5494729724981017\n",
      "save train model at epoch 20\n",
      "Epoch 20 val loss: 0.5950650751590729\n",
      "End epoch 20\n",
      "Start epoch 21\n",
      "Epoch 21 train loss: 0.5468905224638471\n",
      "save train model at epoch 21\n",
      "Epoch 21 val loss: 0.5952599128087361\n",
      "stop training\n",
      "101521_J559NC all_prod 1\n",
      "42410 10610\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.8022951602217663\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.7213594998632159\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.7458544204752129\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.712896200872603\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.7201025349548064\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.70118805482274\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.7042709951659283\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.6971436100346702\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.6943773480064898\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.7062911121618181\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.6877714819218739\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.6982465372199104\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.6829448496720877\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.6901471955435616\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.6791979325823037\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.6884233483246395\n",
      "save val model at epoch 7\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.675512556570122\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.690774435088748\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.6728453198111201\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.6876911435808454\n",
      "save val model at epoch 9\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.6704067311373102\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.7040623923142751\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.6680669418300491\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.6816613631589072\n",
      "save val model at epoch 11\n",
      "End epoch 11\n",
      "Start epoch 12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12 train loss: 0.6663903457572661\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.6859872852052961\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.6642881653395044\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.6812135633968172\n",
      "save val model at epoch 13\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.6627669779651136\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.6842233013539087\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.660872571080564\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.6818091088817233\n",
      "End epoch 15\n",
      "Start epoch 16\n",
      "Epoch 16 train loss: 0.6597490364528564\n",
      "save train model at epoch 16\n",
      "Epoch 16 val loss: 0.6854045831021809\n",
      "End epoch 16\n",
      "Start epoch 17\n",
      "Epoch 17 train loss: 0.6579938997705299\n",
      "save train model at epoch 17\n",
      "Epoch 17 val loss: 0.6788630740983146\n",
      "save val model at epoch 17\n",
      "End epoch 17\n",
      "Start epoch 18\n",
      "Epoch 18 train loss: 0.6567348658320415\n",
      "save train model at epoch 18\n",
      "Epoch 18 val loss: 0.6778451587472644\n",
      "save val model at epoch 18\n",
      "End epoch 18\n",
      "Start epoch 19\n",
      "Epoch 19 train loss: 0.6551488731280867\n",
      "save train model at epoch 19\n",
      "Epoch 19 val loss: 0.6785825675442105\n",
      "End epoch 19\n",
      "Start epoch 20\n",
      "Epoch 20 train loss: 0.6540097350097565\n",
      "save train model at epoch 20\n",
      "Epoch 20 val loss: 0.6785496700377691\n",
      "End epoch 20\n",
      "Start epoch 21\n",
      "Epoch 21 train loss: 0.6528908506215337\n",
      "save train model at epoch 21\n",
      "Epoch 21 val loss: 0.6774356010414305\n",
      "save val model at epoch 21\n",
      "End epoch 21\n",
      "Start epoch 22\n",
      "Epoch 22 train loss: 0.6516352109162205\n",
      "save train model at epoch 22\n",
      "Epoch 22 val loss: 0.6765259419168744\n",
      "save val model at epoch 22\n",
      "End epoch 22\n",
      "Start epoch 23\n",
      "Epoch 23 train loss: 0.650651598192123\n",
      "save train model at epoch 23\n",
      "Epoch 23 val loss: 0.6778591119107746\n",
      "End epoch 23\n",
      "Start epoch 24\n",
      "Epoch 24 train loss: 0.6497464531875519\n",
      "save train model at epoch 24\n",
      "Epoch 24 val loss: 0.6771810324419112\n",
      "End epoch 24\n",
      "Start epoch 25\n",
      "Epoch 25 train loss: 0.648313874939838\n",
      "save train model at epoch 25\n",
      "Epoch 25 val loss: 0.6772555609544119\n",
      "End epoch 25\n",
      "Start epoch 26\n",
      "Epoch 26 train loss: 0.64756924811616\n",
      "save train model at epoch 26\n",
      "Epoch 26 val loss: 0.6752885792936597\n",
      "save val model at epoch 26\n",
      "End epoch 26\n",
      "Start epoch 27\n",
      "Epoch 27 train loss: 0.6462473499487682\n",
      "save train model at epoch 27\n",
      "Epoch 27 val loss: 0.6755488089152745\n",
      "End epoch 27\n",
      "Start epoch 28\n",
      "Epoch 28 train loss: 0.6455354033464409\n",
      "save train model at epoch 28\n",
      "Epoch 28 val loss: 0.6740479639598301\n",
      "save val model at epoch 28\n",
      "End epoch 28\n",
      "Start epoch 29\n",
      "Epoch 29 train loss: 0.6440768970782498\n",
      "save train model at epoch 29\n",
      "Epoch 29 val loss: 0.6755109812532153\n",
      "End epoch 29\n",
      "Start epoch 30\n",
      "Epoch 30 train loss: 0.6434270872409085\n",
      "save train model at epoch 30\n",
      "Epoch 30 val loss: 0.6756966241768428\n",
      "End epoch 30\n",
      "Start epoch 31\n",
      "Epoch 31 train loss: 0.6424761953124081\n",
      "save train model at epoch 31\n",
      "Epoch 31 val loss: 0.6885517622743335\n",
      "End epoch 31\n",
      "Start epoch 32\n",
      "Epoch 32 train loss: 0.6412055876599737\n",
      "save train model at epoch 32\n",
      "Epoch 32 val loss: 0.6806644158703941\n",
      "End epoch 32\n",
      "Start epoch 33\n",
      "Epoch 33 train loss: 0.6408430201461516\n",
      "save train model at epoch 33\n",
      "Epoch 33 val loss: 0.6770407557487488\n",
      "stop training\n",
      "101521_J559NC all_prod 2\n",
      "42400 10610\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.7933085733149425\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.7159039732955751\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.7301764348185206\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.7000963006700788\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.7070379713213587\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.6942302641414461\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.6932547394051609\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.6896295292036874\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.6838087737560272\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.6872237580163139\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.676844995783036\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.6821101946490151\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.6716168583875679\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.6824274715923128\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.6672649297369532\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.6787455550261906\n",
      "save val model at epoch 7\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.6638593494173992\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.6784511591706958\n",
      "save val model at epoch 8\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.660805657326457\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.6773137805007753\n",
      "save val model at epoch 9\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.6580530455313534\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.6752254083043053\n",
      "save val model at epoch 10\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.655477630086692\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.6778714756170908\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.6529059679393309\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.6764752722921825\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.6510262571903597\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.6743079580011822\n",
      "save val model at epoch 13\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.648508476205619\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.6761197859332675\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.6465507717017668\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.6818265687851679\n",
      "End epoch 15\n",
      "Start epoch 16\n",
      "Epoch 16 train loss: 0.6443500766553074\n",
      "save train model at epoch 16\n",
      "Epoch 16 val loss: 0.6711023237024035\n",
      "save val model at epoch 16\n",
      "End epoch 16\n",
      "Start epoch 17\n",
      "Epoch 17 train loss: 0.6426022178437336\n",
      "save train model at epoch 17\n",
      "Epoch 17 val loss: 0.6772194192523048\n",
      "End epoch 17\n",
      "Start epoch 18\n",
      "Epoch 18 train loss: 0.6404750986271593\n",
      "save train model at epoch 18\n",
      "Epoch 18 val loss: 0.6729152117456708\n",
      "End epoch 18\n",
      "Start epoch 19\n",
      "Epoch 19 train loss: 0.6389293465987745\n",
      "save train model at epoch 19\n",
      "Epoch 19 val loss: 0.677417000134786\n",
      "End epoch 19\n",
      "Start epoch 20\n",
      "Epoch 20 train loss: 0.6371279843600399\n",
      "save train model at epoch 20\n",
      "Epoch 20 val loss: 0.6742090880870819\n",
      "End epoch 20\n",
      "Start epoch 21\n",
      "Epoch 21 train loss: 0.6351732690650297\n",
      "save train model at epoch 21\n",
      "Epoch 21 val loss: 0.6725633896532512\n",
      "stop training\n",
      "101521_J559NC all_prod 3\n",
      "42400 10600\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.78698179053973\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.7144930717490968\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.7280267117971397\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.69794857218152\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.7032746539776584\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.6916156893684751\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.6884151007755693\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.6876233220100403\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.6790157840194473\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.6842078069845835\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.6717085170458599\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.68159552415212\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.6662499369627022\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.6810183567660195\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.6619587522673319\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.6757453424589974\n",
      "save val model at epoch 7\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.6583693167531347\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.6749826385861352\n",
      "save val model at epoch 8\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.6551893780748528\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.6750169524124691\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.6519996294056076\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.6769033925873893\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.6493949168417827\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.676435452131998\n",
      "End epoch 11\n",
      "Start epoch 12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12 train loss: 0.6466147472341377\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.6729027572132292\n",
      "save val model at epoch 12\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.6440404420157513\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.6743664202235994\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.6415298448269626\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.6754869818687439\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.6392319952867117\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.6714543671835036\n",
      "save val model at epoch 15\n",
      "End epoch 15\n",
      "Start epoch 16\n",
      "Epoch 16 train loss: 0.6368268448186208\n",
      "save train model at epoch 16\n",
      "Epoch 16 val loss: 0.6718713385718209\n",
      "End epoch 16\n",
      "Start epoch 17\n",
      "Epoch 17 train loss: 0.6345041483999735\n",
      "save train model at epoch 17\n",
      "Epoch 17 val loss: 0.6721480332669758\n",
      "End epoch 17\n",
      "Start epoch 18\n",
      "Epoch 18 train loss: 0.6326008525239416\n",
      "save train model at epoch 18\n",
      "Epoch 18 val loss: 0.675529108161018\n",
      "End epoch 18\n",
      "Start epoch 19\n",
      "Epoch 19 train loss: 0.6307029720530453\n",
      "save train model at epoch 19\n",
      "Epoch 19 val loss: 0.6728957848889487\n",
      "End epoch 19\n",
      "Start epoch 20\n",
      "Epoch 20 train loss: 0.6278799348566906\n",
      "save train model at epoch 20\n",
      "Epoch 20 val loss: 0.6741410621574947\n",
      "stop training\n",
      "101521_J559NC all_prod 4\n",
      "42390 10600\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.7839393719851252\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.7121964834985279\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.7250686366874052\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.7024772820018587\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.7002361370138375\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.687923184462956\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.6856338263276112\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.6889648409116835\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.6757899267845843\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.6780793141751063\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.669505060796278\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.675813322975522\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.6633990166417087\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.675834215822674\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.6588404268385416\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.6759159096649715\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.6549513598522508\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.6733646733420235\n",
      "save val model at epoch 8\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.6511683471231575\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.676698176633744\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.6478909803442208\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.6750166458742959\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.6449354524353901\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.6737737684022813\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.6417818719364075\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.6721400547595251\n",
      "save val model at epoch 12\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.6388199372463915\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.6724632722990853\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.6364218409521034\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.6718561507406688\n",
      "save val model at epoch 14\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.6335881600178868\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.6723648153600239\n",
      "End epoch 15\n",
      "Start epoch 16\n",
      "Epoch 16 train loss: 0.6309519165969757\n",
      "save train model at epoch 16\n",
      "Epoch 16 val loss: 0.6756570069562822\n",
      "End epoch 16\n",
      "Start epoch 17\n",
      "Epoch 17 train loss: 0.6284414159964367\n",
      "save train model at epoch 17\n",
      "Epoch 17 val loss: 0.6730378369490305\n",
      "End epoch 17\n",
      "Start epoch 18\n",
      "Epoch 18 train loss: 0.6259891064052122\n",
      "save train model at epoch 18\n",
      "Epoch 18 val loss: 0.6715576180389949\n",
      "save val model at epoch 18\n",
      "End epoch 18\n",
      "Start epoch 19\n",
      "Epoch 19 train loss: 0.6233664632561695\n",
      "save train model at epoch 19\n",
      "Epoch 19 val loss: 0.6729115375450679\n",
      "End epoch 19\n",
      "Start epoch 20\n",
      "Epoch 20 train loss: 0.6209923761436739\n",
      "save train model at epoch 20\n",
      "Epoch 20 val loss: 0.672339562858854\n",
      "End epoch 20\n",
      "Start epoch 21\n",
      "Epoch 21 train loss: 0.6183332786502609\n",
      "save train model at epoch 21\n",
      "Epoch 21 val loss: 0.6738511976741609\n",
      "End epoch 21\n",
      "Start epoch 22\n",
      "Epoch 22 train loss: 0.6160062149346593\n",
      "save train model at epoch 22\n",
      "Epoch 22 val loss: 0.6763247961089724\n",
      "End epoch 22\n",
      "Start epoch 23\n",
      "Epoch 23 train loss: 0.6135839805545578\n",
      "save train model at epoch 23\n",
      "Epoch 23 val loss: 0.6742645104726156\n",
      "stop training\n",
      "101521_J559NC all_prod 5\n",
      "42380 10600\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.7829067215143916\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.7096356465702965\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.723532124815217\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.6937211354573568\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.7005126605550926\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.6890530813307989\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.6865263243755663\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.6852581018493289\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.6762208791382341\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.6805153262047541\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.6691423532474472\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.6789167069253468\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.6634054148053549\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.6770274724279132\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.6585546078452145\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.683732746612458\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.6545827087149563\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.6722314669972375\n",
      "save val model at epoch 8\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.6503414741481643\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.6765355297497341\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.6474346141499209\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.6730099774542309\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.6438692596303411\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.6731719573338827\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.6405334785042038\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.6734611477170672\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.6380907566432493\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.6743113895257314\n",
      "stop training\n",
      "101521_J559NC all_prod 6\n",
      "42370 10600\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.7830484416111406\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.7143462498982748\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.7236828574215073\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.6948392476354327\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.7000854205654328\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.6911951785995847\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.6853307859007135\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.6825479041962397\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.6753484771912357\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.6798625049136934\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.6674216292708753\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.6780341892015367\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.6619362253022482\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.6769883448169345\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.6570497499891075\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.6735074349812099\n",
      "save val model at epoch 7\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.6529593083513788\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.6730773250261942\n",
      "save val model at epoch 8\n",
      "End epoch 8\n",
      "Start epoch 9\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9 train loss: 0.6489110782922033\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.673529421999341\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.64550879956728\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.6723382714248839\n",
      "save val model at epoch 10\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.6416267188916723\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.6829407385417393\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.6385589457419981\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.6754292774768103\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.6350795170628881\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.6738817521503994\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.6318697498505375\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.6740118023895082\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.6289995948234236\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.6731084599381402\n",
      "stop training\n",
      "101521_J559NC all_prod 7\n",
      "42360 10600\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.7826450761542263\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.7084183352334159\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.7213991474674408\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.6970398355097998\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.6983143522078732\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.6912152369817098\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.6838074696351246\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.6838518225011372\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.6741679016366062\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.6832861020451501\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.6672943008233265\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.6783187502906436\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.6613453809755394\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.6756435902345748\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.6561189329767801\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.6758157908916473\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.6517483060618481\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.6767162453560602\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.6480841895183885\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.6725337221508935\n",
      "save val model at epoch 9\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.6443043570202517\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.6743645384198144\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.6410118955445577\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.6735089832828158\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.637013037879783\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.6738527317841848\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.633837393967502\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.6736396877538591\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.6303868125002068\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.6737887646470752\n",
      "stop training\n",
      "101521_J559NC all_prod 8\n",
      "42360 10590\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.7829311895801361\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.714048357236953\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.7270977608410709\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.69944015003386\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.7046755681554955\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.6920169265497298\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.6905998433210764\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.6868461526575542\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.6803852793682053\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.6943506924878984\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.672972697450454\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.6831214541480655\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.6665038750114212\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.67903385417802\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.6611346760428095\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.6799487755412147\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.6561414430658501\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.6811795660427639\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.651830221515104\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.6767039071945917\n",
      "save val model at epoch 9\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.6477663010717875\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.675470096724374\n",
      "save val model at epoch 10\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.6438649142362985\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.675046580178397\n",
      "save val model at epoch 11\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.6399662936308298\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.6747401796636128\n",
      "save val model at epoch 12\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.6365222392312015\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.6759697851680574\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.6331600952579315\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.6777705720492772\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.6295759530670672\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.6775453402882531\n",
      "End epoch 15\n",
      "Start epoch 16\n",
      "Epoch 16 train loss: 0.6263052085077906\n",
      "save train model at epoch 16\n",
      "Epoch 16 val loss: 0.6762676891826448\n",
      "End epoch 16\n",
      "Start epoch 17\n",
      "Epoch 17 train loss: 0.622707156531782\n",
      "save train model at epoch 17\n",
      "Epoch 17 val loss: 0.67649036078226\n",
      "stop training\n",
      "110421_J569LT all_prod 1\n",
      "32940 8240\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.8328179347422696\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.7575865694970796\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.7824054706928342\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.7491919813734113\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.7582839778227399\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.7413868127447186\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.7421732988468436\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.7356947967500398\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.7318352354589359\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.7355064821965767\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.7235571608062863\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.7353008299162893\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.7177188100740891\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.7321745572668134\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.7127016413119412\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.729636569817861\n",
      "save val model at epoch 7\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.7083817734274753\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.7299657193097201\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.7057313369226086\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.733710836280476\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.703039386475733\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.7290404782150731\n",
      "save val model at epoch 10\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.7003827561703764\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.7247396364356532\n",
      "save val model at epoch 11\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.6984914393388024\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.7275555531183878\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.6964149964872257\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.7274546749664076\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.6942783478618593\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.727279291008458\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.6923796358958695\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.7244835965561144\n",
      "save val model at epoch 15\n",
      "End epoch 15\n",
      "Start epoch 16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 16 train loss: 0.6906477568685546\n",
      "save train model at epoch 16\n",
      "Epoch 16 val loss: 0.7331881487008297\n",
      "End epoch 16\n",
      "Start epoch 17\n",
      "Epoch 17 train loss: 0.6888295860253564\n",
      "save train model at epoch 17\n",
      "Epoch 17 val loss: 0.7312970414306178\n",
      "End epoch 17\n",
      "Start epoch 18\n",
      "Epoch 18 train loss: 0.6881138169488241\n",
      "save train model at epoch 18\n",
      "Epoch 18 val loss: 0.7231675350304806\n",
      "save val model at epoch 18\n",
      "End epoch 18\n",
      "Start epoch 19\n",
      "Epoch 19 train loss: 0.6861006178597148\n",
      "save train model at epoch 19\n",
      "Epoch 19 val loss: 0.7225732586600564\n",
      "save val model at epoch 19\n",
      "End epoch 19\n",
      "Start epoch 20\n",
      "Epoch 20 train loss: 0.6845957858617916\n",
      "save train model at epoch 20\n",
      "Epoch 20 val loss: 0.7265421964905479\n",
      "End epoch 20\n",
      "Start epoch 21\n",
      "Epoch 21 train loss: 0.6834238698316175\n",
      "save train model at epoch 21\n",
      "Epoch 21 val loss: 0.7228349754304597\n",
      "End epoch 21\n",
      "Start epoch 22\n",
      "Epoch 22 train loss: 0.6821787070858386\n",
      "save train model at epoch 22\n",
      "Epoch 22 val loss: 0.7374747052337184\n",
      "End epoch 22\n",
      "Start epoch 23\n",
      "Epoch 23 train loss: 0.680815061857534\n",
      "save train model at epoch 23\n",
      "Epoch 23 val loss: 0.7253364703871987\n",
      "End epoch 23\n",
      "Start epoch 24\n",
      "Epoch 24 train loss: 0.6797023027442223\n",
      "save train model at epoch 24\n",
      "Epoch 24 val loss: 0.7223115075718273\n",
      "save val model at epoch 24\n",
      "End epoch 24\n",
      "Start epoch 25\n",
      "Epoch 25 train loss: 0.67831722479458\n",
      "save train model at epoch 25\n",
      "Epoch 25 val loss: 0.7234044165322275\n",
      "End epoch 25\n",
      "Start epoch 26\n",
      "Epoch 26 train loss: 0.6771496571311655\n",
      "save train model at epoch 26\n",
      "Epoch 26 val loss: 0.7270930608113607\n",
      "End epoch 26\n",
      "Start epoch 27\n",
      "Epoch 27 train loss: 0.6766097051228663\n",
      "save train model at epoch 27\n",
      "Epoch 27 val loss: 0.7191014903964419\n",
      "save val model at epoch 27\n",
      "End epoch 27\n",
      "Start epoch 28\n",
      "Epoch 28 train loss: 0.6753856553587803\n",
      "save train model at epoch 28\n",
      "Epoch 28 val loss: 0.724962799838095\n",
      "End epoch 28\n",
      "Start epoch 29\n",
      "Epoch 29 train loss: 0.6745187912800515\n",
      "save train model at epoch 29\n",
      "Epoch 29 val loss: 0.7222182949384054\n",
      "End epoch 29\n",
      "Start epoch 30\n",
      "Epoch 30 train loss: 0.6728717911151029\n",
      "save train model at epoch 30\n",
      "Epoch 30 val loss: 0.7188237905502319\n",
      "save val model at epoch 30\n",
      "End epoch 30\n",
      "Start epoch 31\n",
      "Epoch 31 train loss: 0.6722444337467814\n",
      "save train model at epoch 31\n",
      "Epoch 31 val loss: 0.7216638976877386\n",
      "End epoch 31\n",
      "Start epoch 32\n",
      "Epoch 32 train loss: 0.6709690154060837\n",
      "save train model at epoch 32\n",
      "Epoch 32 val loss: 0.7280990156260404\n",
      "End epoch 32\n",
      "Start epoch 33\n",
      "Epoch 33 train loss: 0.6703260041946588\n",
      "save train model at epoch 33\n",
      "Epoch 33 val loss: 0.7203542138590957\n",
      "End epoch 33\n",
      "Start epoch 34\n",
      "Epoch 34 train loss: 0.6695556672968606\n",
      "save train model at epoch 34\n",
      "Epoch 34 val loss: 0.7196367346879208\n",
      "End epoch 34\n",
      "Start epoch 35\n",
      "Epoch 35 train loss: 0.668700210345808\n",
      "save train model at epoch 35\n",
      "Epoch 35 val loss: 0.721363694378824\n",
      "stop training\n",
      "110421_J569LT all_prod 2\n",
      "32930 8240\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.8198670111885367\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.7520835796991984\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.7673460097275964\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.7394724199266145\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.7438654395961022\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.7327689546527285\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.7285105646118637\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.7296339977871288\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.7180348426796669\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.7275488033439174\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.7102512435395588\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.7273756843624692\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.7043837502945302\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.7261782198241262\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.6987616798674413\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.725785083843\n",
      "save val model at epoch 7\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.6942716148472572\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.7232495672775038\n",
      "save val model at epoch 8\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.6906100936638292\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.7227352695031599\n",
      "save val model at epoch 9\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.6873006811437681\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.7187341560016979\n",
      "save val model at epoch 10\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.6845924992894017\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.719313484249693\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.6819539966509324\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.7224637613152013\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.6791358007941135\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.7252253312053103\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.677561878233917\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.7233304941292965\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.6748489617377289\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.7226929809107925\n",
      "stop training\n",
      "110421_J569LT all_prod 3\n",
      "32920 8240\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.8146119043808575\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.7511038075793873\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.7618212325628414\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.740341892748168\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.7396713957306027\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.7309848575881033\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.7257700165112814\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.7299696676658861\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.7165448628654776\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.7272469437483585\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.7080816002779229\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.7235408082152858\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.7020896862643634\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.7226535063801389\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.6966630451438963\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.7209528323375818\n",
      "save val model at epoch 7\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.6924248919930569\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.7211300911325397\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.688475512256918\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.7178726557529334\n",
      "save val model at epoch 9\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.6849801896154418\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.7213165976784446\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.6816509241281554\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.7193500526023634\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.6791338232136512\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.7204762317917563\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.6761525305666665\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.7213759855790571\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.6731914217157882\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.7198390021468654\n",
      "stop training\n",
      "110421_J569LT all_prod 4\n",
      "32920 8230\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.8129935181418131\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.7428257880788861\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.757097488687944\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.7318762576941288\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.7347302275110584\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.7328752622459874\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.7197209573531336\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.7264003392421838\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.7089988533840623\n",
      "save train model at epoch 4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4 val loss: 0.7245057351661451\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.7006916717965473\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.7219607089505051\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.6945571973342304\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.7189166455557852\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.6889236726502116\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.7181993740977664\n",
      "save val model at epoch 7\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.6845214611800142\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.718849929896268\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.6801352759664373\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.7161482048757148\n",
      "save val model at epoch 9\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.6774860066036845\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.71550114588304\n",
      "save val model at epoch 10\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.6734386257422987\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.7154956405813043\n",
      "save val model at epoch 11\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.670195038004439\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.7150680513092966\n",
      "save val model at epoch 12\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.6672858762186628\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.7200686732927958\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.66473601354185\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.7184459205829736\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.6620974231135938\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.713136840950359\n",
      "save val model at epoch 15\n",
      "End epoch 15\n",
      "Start epoch 16\n",
      "Epoch 16 train loss: 0.6598673245703527\n",
      "save train model at epoch 16\n",
      "Epoch 16 val loss: 0.7154965671626005\n",
      "End epoch 16\n",
      "Start epoch 17\n",
      "Epoch 17 train loss: 0.6573259553243948\n",
      "save train model at epoch 17\n",
      "Epoch 17 val loss: 0.7119524171858123\n",
      "save val model at epoch 17\n",
      "End epoch 17\n",
      "Start epoch 18\n",
      "Epoch 18 train loss: 0.6553189491116723\n",
      "save train model at epoch 18\n",
      "Epoch 18 val loss: 0.7147749517903184\n",
      "End epoch 18\n",
      "Start epoch 19\n",
      "Epoch 19 train loss: 0.6527504616005476\n",
      "save train model at epoch 19\n",
      "Epoch 19 val loss: 0.714061742479151\n",
      "End epoch 19\n",
      "Start epoch 20\n",
      "Epoch 20 train loss: 0.6507009609725124\n",
      "save train model at epoch 20\n",
      "Epoch 20 val loss: 0.7139605374047251\n",
      "End epoch 20\n",
      "Start epoch 21\n",
      "Epoch 21 train loss: 0.6485033538914466\n",
      "save train model at epoch 21\n",
      "Epoch 21 val loss: 0.7145351207617557\n",
      "End epoch 21\n",
      "Start epoch 22\n",
      "Epoch 22 train loss: 0.6461813796398251\n",
      "save train model at epoch 22\n",
      "Epoch 22 val loss: 0.7164552789745908\n",
      "stop training\n",
      "110421_J569LT all_prod 5\n",
      "32910 8230\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.810270234133846\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.7436705340038646\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.7567216860231503\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.7327210252935236\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.7335905132367629\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.7250763900352247\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.718624145485634\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.7207934368740428\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.7074904488038647\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.7196431196097172\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.6985902398131615\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.7223584615822994\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.6922070337820423\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.715527082934524\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.6866301621577536\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.7164027167089058\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.6824756597363671\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.716376416610949\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.6786501749541408\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.7168006878910642\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.6749498285064401\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.7146959864731991\n",
      "save val model at epoch 10\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.6714725956436276\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.716470864686099\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.6684517523114996\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.7126830075726365\n",
      "save val model at epoch 12\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.6654246985450272\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.712481419245402\n",
      "save val model at epoch 13\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.6624770261520563\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.7141536979964285\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.6598715306252472\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.7133520931908579\n",
      "End epoch 15\n",
      "Start epoch 16\n",
      "Epoch 16 train loss: 0.6575361042059669\n",
      "save train model at epoch 16\n",
      "Epoch 16 val loss: 0.7147137540759463\n",
      "End epoch 16\n",
      "Start epoch 17\n",
      "Epoch 17 train loss: 0.6550019979476929\n",
      "save train model at epoch 17\n",
      "Epoch 17 val loss: 0.7164071310650219\n",
      "End epoch 17\n",
      "Start epoch 18\n",
      "Epoch 18 train loss: 0.6522986851921377\n",
      "save train model at epoch 18\n",
      "Epoch 18 val loss: 0.7143868767853939\n",
      "stop training\n",
      "110421_J569LT all_prod 6\n",
      "32900 8230\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.8109012823696284\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.7464342659169977\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.757953496866448\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.7328466628537034\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.7354245925134466\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.7304852333935824\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.7206104670384134\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.7298904151627512\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.7095436079557552\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.7226103146870931\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.7009999752044678\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.7217386057882598\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.6942797598912734\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.7201667120962432\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.6886263180148694\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.7209984027978146\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.6833938212357751\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.7194178772695137\n",
      "save val model at epoch 8\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.6793404526488726\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.721986503312082\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.6752037955809009\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.7180915745821866\n",
      "save val model at epoch 10\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.6719090166942093\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.7199783000079069\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.6686136916626332\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.7194226655093107\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.6652183084524879\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.718339715943192\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.662112948968429\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.7164611039739667\n",
      "save val model at epoch 14\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.6590872767359711\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.7161767356323473\n",
      "save val model at epoch 15\n",
      "End epoch 15\n",
      "Start epoch 16\n",
      "Epoch 16 train loss: 0.6566943872806638\n",
      "save train model at epoch 16\n",
      "Epoch 16 val loss: 0.7129881219430403\n",
      "save val model at epoch 16\n",
      "End epoch 16\n",
      "Start epoch 17\n",
      "Epoch 17 train loss: 0.653743771619575\n",
      "save train model at epoch 17\n",
      "Epoch 17 val loss: 0.7121626247059215\n",
      "save val model at epoch 17\n",
      "End epoch 17\n",
      "Start epoch 18\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 18 train loss: 0.6512138834295347\n",
      "save train model at epoch 18\n",
      "Epoch 18 val loss: 0.7138871297691808\n",
      "End epoch 18\n",
      "Start epoch 19\n",
      "Epoch 19 train loss: 0.6480577130650365\n",
      "save train model at epoch 19\n",
      "Epoch 19 val loss: 0.7186040751861803\n",
      "End epoch 19\n",
      "Start epoch 20\n",
      "Epoch 20 train loss: 0.6452609041864558\n",
      "save train model at epoch 20\n",
      "Epoch 20 val loss: 0.7175055507457617\n",
      "End epoch 20\n",
      "Start epoch 21\n",
      "Epoch 21 train loss: 0.6427546536275582\n",
      "save train model at epoch 21\n",
      "Epoch 21 val loss: 0.7186246153080103\n",
      "End epoch 21\n",
      "Start epoch 22\n",
      "Epoch 22 train loss: 0.640399574771408\n",
      "save train model at epoch 22\n",
      "Epoch 22 val loss: 0.7161332007610437\n",
      "stop training\n",
      "110421_J569LT all_prod 7\n",
      "32890 8230\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.8086218196292256\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.7455958326657613\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.7589819551438324\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.732901004227725\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.7365673974502919\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.7288216731765054\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.7212044901626055\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.7279808376774644\n",
      "save val model at epoch 3\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.710617740948995\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.7217213077978655\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.7022365724393563\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.7181990814931465\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.6955735567928285\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.7215041265343175\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.6901167977687924\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.7231793584245624\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.6852347712184108\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.7187735293850754\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.6813721887825072\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.7199288696953745\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.6769189427989398\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.7166346076763037\n",
      "save val model at epoch 10\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.673738648263059\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.7193648941589125\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.6700736921887065\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.7194935896179893\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.6672349932581879\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.7175451625477184\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.6646326637083246\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.717805746829871\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.6611006749692814\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.7221756685863842\n",
      "stop training\n",
      "110421_J569LT all_prod 8\n",
      "32880 8230\n",
      "Start epoch 0\n",
      "Epoch 0 train loss: 0.8095081896744958\n",
      "save train model at epoch 0\n",
      "Epoch 0 val loss: 0.7378883072824189\n",
      "save val model at epoch 0\n",
      "End epoch 0\n",
      "Start epoch 1\n",
      "Epoch 1 train loss: 0.7563898535661919\n",
      "save train model at epoch 1\n",
      "Epoch 1 val loss: 0.7307389830098008\n",
      "save val model at epoch 1\n",
      "End epoch 1\n",
      "Start epoch 2\n",
      "Epoch 2 train loss: 0.7346362781155017\n",
      "save train model at epoch 2\n",
      "Epoch 2 val loss: 0.7243605797941034\n",
      "save val model at epoch 2\n",
      "End epoch 2\n",
      "Start epoch 3\n",
      "Epoch 3 train loss: 0.7193964964659639\n",
      "save train model at epoch 3\n",
      "Epoch 3 val loss: 0.7270055575804277\n",
      "End epoch 3\n",
      "Start epoch 4\n",
      "Epoch 4 train loss: 0.7083110157833543\n",
      "save train model at epoch 4\n",
      "Epoch 4 val loss: 0.7227339238831492\n",
      "save val model at epoch 4\n",
      "End epoch 4\n",
      "Start epoch 5\n",
      "Epoch 5 train loss: 0.6997685219890387\n",
      "save train model at epoch 5\n",
      "Epoch 5 val loss: 0.7215534827925942\n",
      "save val model at epoch 5\n",
      "End epoch 5\n",
      "Start epoch 6\n",
      "Epoch 6 train loss: 0.6932310790054558\n",
      "save train model at epoch 6\n",
      "Epoch 6 val loss: 0.7193668936238145\n",
      "save val model at epoch 6\n",
      "End epoch 6\n",
      "Start epoch 7\n",
      "Epoch 7 train loss: 0.6874811080999153\n",
      "save train model at epoch 7\n",
      "Epoch 7 val loss: 0.7208079406709382\n",
      "End epoch 7\n",
      "Start epoch 8\n",
      "Epoch 8 train loss: 0.6830421968023906\n",
      "save train model at epoch 8\n",
      "Epoch 8 val loss: 0.722183549042904\n",
      "End epoch 8\n",
      "Start epoch 9\n",
      "Epoch 9 train loss: 0.6786718387012334\n",
      "save train model at epoch 9\n",
      "Epoch 9 val loss: 0.7203760020660631\n",
      "End epoch 9\n",
      "Start epoch 10\n",
      "Epoch 10 train loss: 0.6751903535783753\n",
      "save train model at epoch 10\n",
      "Epoch 10 val loss: 0.7189410437237133\n",
      "save val model at epoch 10\n",
      "End epoch 10\n",
      "Start epoch 11\n",
      "Epoch 11 train loss: 0.6708023136900377\n",
      "save train model at epoch 11\n",
      "Epoch 11 val loss: 0.7179675788590403\n",
      "save val model at epoch 11\n",
      "End epoch 11\n",
      "Start epoch 12\n",
      "Epoch 12 train loss: 0.6676196673119715\n",
      "save train model at epoch 12\n",
      "Epoch 12 val loss: 0.7180364998904142\n",
      "End epoch 12\n",
      "Start epoch 13\n",
      "Epoch 13 train loss: 0.6641339456388192\n",
      "save train model at epoch 13\n",
      "Epoch 13 val loss: 0.7246012163884712\n",
      "End epoch 13\n",
      "Start epoch 14\n",
      "Epoch 14 train loss: 0.661347019117932\n",
      "save train model at epoch 14\n",
      "Epoch 14 val loss: 0.7168285395159866\n",
      "save val model at epoch 14\n",
      "End epoch 14\n",
      "Start epoch 15\n",
      "Epoch 15 train loss: 0.6577992979870286\n",
      "save train model at epoch 15\n",
      "Epoch 15 val loss: 0.7149882695891641\n",
      "save val model at epoch 15\n",
      "End epoch 15\n",
      "Start epoch 16\n",
      "Epoch 16 train loss: 0.6552744572476823\n",
      "save train model at epoch 16\n",
      "Epoch 16 val loss: 0.7163330497163715\n",
      "End epoch 16\n",
      "Start epoch 17\n",
      "Epoch 17 train loss: 0.6517285639925521\n",
      "save train model at epoch 17\n",
      "Epoch 17 val loss: 0.7162103562644033\n",
      "End epoch 17\n",
      "Start epoch 18\n",
      "Epoch 18 train loss: 0.6492799857790156\n",
      "save train model at epoch 18\n",
      "Epoch 18 val loss: 0.7170736663269274\n",
      "End epoch 18\n",
      "Start epoch 19\n",
      "Epoch 19 train loss: 0.6463463440421939\n",
      "save train model at epoch 19\n",
      "Epoch 19 val loss: 0.7206997636592749\n",
      "End epoch 19\n",
      "Start epoch 20\n",
      "Epoch 20 train loss: 0.643356439679168\n",
      "save train model at epoch 20\n",
      "Epoch 20 val loss: 0.7178577834909613\n",
      "stop training\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')    \n",
    "\n",
    "for file_id, num_neurons in  [(\"070921_J553RT\", 68), (\"101521_J559NC\", 49), (\"110421_J569LT\", 32)]:\n",
    "    for behav_mode, behav_dim in [(\"all_prod\", 66)]:\n",
    "        for seq_len in range(1, 9): \n",
    "            print(file_id, behav_mode, seq_len)\n",
    "            \n",
    "            args.file_id = file_id\n",
    "            args.num_neurons = num_neurons\n",
    "            args.shifter=True\n",
    "\n",
    "            args.behav_mode = behav_mode\n",
    "            args.behav_dim = behav_dim\n",
    "            \n",
    "            args.seq_len = seq_len\n",
    "\n",
    "            args.best_train_path = \"weights_cnn_gru_shifter/train_{}_{}_seq_{}.pth\".format(\n",
    "                args.file_id, args.behav_mode, args.seq_len)\n",
    "            args.best_val_path = \"weights_cnn_gru_shifter/val_{}_{}_seq_{}.pth\".format(\n",
    "                args.file_id, args.behav_mode, args.seq_len)\n",
    "\n",
    "            model = LSTMPerNeuronCombiner(num_neurons=args.num_neurons, \n",
    "                                          behav_dim=args.behav_dim, \n",
    "                                          k1=7, k2=7, k3=7, \n",
    "                                          seq_len=args.seq_len,\n",
    "                                          hidden_size=args.hidden_size).to(device)\n",
    "\n",
    "            train_loss_list, val_loss_list = train_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LSTMPerNeuronCombiner(\n",
       "  (shifter): Shifter(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(4, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Linear(in_features=4, out_features=256, bias=True)\n",
       "      (2): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (3): Tanh()\n",
       "      (4): Linear(in_features=256, out_features=256, bias=True)\n",
       "      (5): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (6): Tanh()\n",
       "      (7): Linear(in_features=256, out_features=3, bias=True)\n",
       "      (8): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (visual_encoder): VisualEncoder(\n",
       "    (layers): Sequential(\n",
       "      (0): Conv2d(1, 128, kernel_size=(7, 7), stride=(2, 2))\n",
       "      (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): ReLU()\n",
       "      (3): Dropout(p=0.5, inplace=False)\n",
       "      (4): Conv2d(128, 64, kernel_size=(7, 7), stride=(2, 2))\n",
       "      (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (6): ReLU()\n",
       "      (7): Dropout(p=0.5, inplace=False)\n",
       "      (8): Conv2d(64, 32, kernel_size=(7, 7), stride=(2, 2))\n",
       "      (9): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (10): ReLU()\n",
       "      (11): Dropout(p=0.5, inplace=False)\n",
       "      (12): Flatten(start_dim=1, end_dim=-1)\n",
       "      (13): Linear(in_features=480, out_features=32, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (behav_encoder): BehavEncoder(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(66, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Linear(in_features=66, out_features=32, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (bn): BatchNorm1d(3, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (lstm_net): GRU(96, 512, batch_first=True)\n",
       "  (fc): Linear(in_features=512, out_features=32, bias=True)\n",
       "  (softplus): Softplus(beta=1, threshold=20)\n",
       ")"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default is smoothing with 2 second, 48 ms per frame\n",
    "def smoothing_with_np_conv(nsp, size=int(2000/48)):\n",
    "    np_conv_res = []\n",
    "    for i in range(nsp.shape[1]):\n",
    "        np_conv_res.append(np.convolve(nsp[:, i], np.ones(size)/size, mode=\"same\"))        \n",
    "    np_conv_res = np.transpose(np.array(np_conv_res))\n",
    "    return np_conv_res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_model(model, weights_path, dataset, device):\n",
    "\n",
    "    dl = DataLoader(dataset=dataset, batch_size=256, shuffle=False, num_workers=4)\n",
    "    \n",
    "    model.load_state_dict(torch.load(weights_path))\n",
    "\n",
    "    ground_truth_all = []\n",
    "    pred_all = []\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    with torch.no_grad():      \n",
    "        \n",
    "        for (image, behav, spikes) in dl:\n",
    "            \n",
    "            image = image.to(device)\n",
    "            behav = behav.to(device)\n",
    "            \n",
    "            pred = model(image, behav)\n",
    "            \n",
    "            ground_truth_all.append(spikes.numpy())\n",
    "            pred_all.append(pred.cpu().numpy())\n",
    "    \n",
    "    return np.concatenate(pred_all, axis=0), np.concatenate(ground_truth_all, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "070921_J553RT all_prod 1\n",
      "30120 7540\n",
      "MSE 0.054336\n",
      "mean corr, 0.646+-0.136\n",
      "070921_J553RT all_prod 2\n",
      "30120 7530\n",
      "MSE 0.052839\n",
      "mean corr, 0.649+-0.139\n",
      "070921_J553RT all_prod 3\n",
      "30110 7530\n",
      "MSE 0.052778\n",
      "mean corr, 0.653+-0.139\n",
      "070921_J553RT all_prod 4\n",
      "30100 7530\n",
      "MSE 0.052547\n",
      "mean corr, 0.650+-0.142\n",
      "070921_J553RT all_prod 5\n",
      "30090 7530\n",
      "MSE 0.055567\n",
      "mean corr, 0.645+-0.144\n",
      "070921_J553RT all_prod 6\n",
      "30080 7530\n",
      "MSE 0.052630\n",
      "mean corr, 0.654+-0.142\n",
      "070921_J553RT all_prod 7\n",
      "30080 7520\n",
      "MSE 0.053443\n",
      "mean corr, 0.644+-0.148\n",
      "070921_J553RT all_prod 8\n",
      "30070 7520\n",
      "MSE 0.054729\n",
      "mean corr, 0.646+-0.146\n",
      "110421_J569LT all_prod 1\n",
      "32940 8240\n",
      "MSE 0.091719\n",
      "mean corr, 0.508+-0.166\n",
      "110421_J569LT all_prod 2\n",
      "32930 8240\n",
      "MSE 0.089837\n",
      "mean corr, 0.506+-0.174\n",
      "110421_J569LT all_prod 3\n",
      "32920 8240\n",
      "MSE 0.084313\n",
      "mean corr, 0.528+-0.160\n",
      "110421_J569LT all_prod 4\n",
      "32920 8230\n",
      "MSE 0.079940\n",
      "mean corr, 0.566+-0.169\n",
      "110421_J569LT all_prod 5\n",
      "32910 8230\n",
      "MSE 0.093304\n",
      "mean corr, 0.519+-0.177\n",
      "110421_J569LT all_prod 6\n",
      "32900 8230\n",
      "MSE 0.082298\n",
      "mean corr, 0.549+-0.175\n",
      "110421_J569LT all_prod 7\n",
      "32890 8230\n",
      "MSE 0.093120\n",
      "mean corr, 0.533+-0.169\n",
      "110421_J569LT all_prod 8\n",
      "32880 8230\n",
      "MSE 0.083966\n",
      "mean corr, 0.546+-0.179\n",
      "101521_J559NC all_prod 1\n",
      "42410 10610\n",
      "MSE 0.080114\n",
      "mean corr, 0.607+-0.132\n",
      "101521_J559NC all_prod 2\n",
      "42400 10610\n",
      "MSE 0.081119\n",
      "mean corr, 0.607+-0.133\n",
      "101521_J559NC all_prod 3\n",
      "42400 10600\n",
      "MSE 0.079014\n",
      "mean corr, 0.604+-0.134\n",
      "101521_J559NC all_prod 4\n",
      "42390 10600\n",
      "MSE 0.077284\n",
      "mean corr, 0.614+-0.136\n",
      "101521_J559NC all_prod 5\n",
      "42380 10600\n",
      "MSE 0.080692\n",
      "mean corr, 0.598+-0.134\n",
      "101521_J559NC all_prod 6\n",
      "42370 10600\n",
      "MSE 0.079824\n",
      "mean corr, 0.598+-0.138\n",
      "101521_J559NC all_prod 7\n",
      "42360 10600\n",
      "MSE 0.080605\n",
      "mean corr, 0.596+-0.133\n",
      "101521_J559NC all_prod 8\n",
      "42360 10590\n",
      "MSE 0.082547\n",
      "mean corr, 0.594+-0.141\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')    \n",
    "\n",
    "for file_id, num_neurons in  [(\"070921_J553RT\", 68), (\"110421_J569LT\", 32), (\"101521_J559NC\", 49)]:\n",
    "    \n",
    "\n",
    "    for behav_mode, behav_dim in [(\"all_prod\", 66)]:\n",
    "\n",
    "        for seq_len in range(1, 9): \n",
    "\n",
    "            print(file_id, behav_mode, seq_len)\n",
    "            \n",
    "            args.file_id = file_id\n",
    "            args.num_neurons = num_neurons\n",
    "\n",
    "            args.behav_mode = behav_mode\n",
    "            args.behav_dim = behav_dim\n",
    "            \n",
    "            args.seq_len = seq_len\n",
    "\n",
    "            args.best_val_path = \"weights_cnn_gru_shifter/val_{}_{}_seq_{}.pth\".format(\n",
    "                args.file_id, args.behav_mode, args.seq_len)\n",
    "\n",
    "            model = LSTMPerNeuronCombiner(num_neurons=args.num_neurons, \n",
    "                                          behav_dim=args.behav_dim, \n",
    "                                          k1=7, k2=7, k3=7, \n",
    "                                          seq_len=args.seq_len,\n",
    "                                          hidden_size=args.hidden_size).to(device)\n",
    "            \n",
    "            train_ds, val_ds = load_train_val_ds()\n",
    "            test_ds = load_test_ds()\n",
    "            \n",
    "            pred, label = evaluate_model(model, weights_path=args.best_val_path, dataset=test_ds, device=device)\n",
    "            cor_array = cor_in_time(pred, label)\n",
    "        #     print(\"best val model on test dataset, {:.3f}+-{:.3f}\".format(np.mean(cor_array), np.std(cor_array)))\n",
    "            pred = smoothing_with_np_conv(pred)\n",
    "            label = smoothing_with_np_conv(label)\n",
    "            # print(\"R2\", \"{:.6f}\".format(r2_score(label.T, pred.T)))\n",
    "            print(\"MSE\", \"{:.6f}\".format(mean_squared_error(label, pred)))\n",
    "            cor_array = cor_in_time(pred, label)\n",
    "            print(\"mean corr, {:.3f}+-{:.3f}\".format(np.mean(cor_array), np.std(cor_array)))\n",
    "            # print(\"max corr\", \"{:.6f}\".format(np.max(cor_array)))\n",
    "            # print(\"min corr\", \"{:.6f}\".format(np.min(cor_array)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "save per neuron correlation for future plotting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "070921_J553RT all_prod 1\n",
      "30120 7540\n",
      "MSE 0.054336\n",
      "mean corr, 0.646+-0.136\n",
      "(1, 68)\n",
      "110421_J569LT all_prod 1\n",
      "32940 8240\n",
      "MSE 0.091719\n",
      "mean corr, 0.508+-0.166\n",
      "(1, 32)\n",
      "101521_J559NC all_prod 1\n",
      "42410 10610\n",
      "MSE 0.080114\n",
      "mean corr, 0.607+-0.132\n",
      "(1, 49)\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')    \n",
    "\n",
    "for file_id, num_neurons in  [(\"070921_J553RT\", 68), (\"110421_J569LT\", 32), (\"101521_J559NC\", 49)]:\n",
    "    \n",
    "\n",
    "    for behav_mode, behav_dim in [(\"all_prod\", 66)]:\n",
    "\n",
    "        seq_len = 1\n",
    "\n",
    "        print(file_id, behav_mode, seq_len)\n",
    "\n",
    "        args.file_id = file_id\n",
    "        args.num_neurons = num_neurons\n",
    "\n",
    "        args.behav_mode = behav_mode\n",
    "        args.behav_dim = behav_dim\n",
    "\n",
    "        args.seq_len = seq_len\n",
    "\n",
    "        args.best_val_path = \"weights_cnn_gru_shifter/val_{}_{}_seq_{}.pth\".format(\n",
    "            args.file_id, args.behav_mode, args.seq_len)\n",
    "\n",
    "        model = LSTMPerNeuronCombiner(num_neurons=args.num_neurons, \n",
    "                                      behav_dim=args.behav_dim, \n",
    "                                      k1=7, k2=7, k3=7, \n",
    "                                      seq_len=args.seq_len,\n",
    "                                      hidden_size=args.hidden_size).to(device)\n",
    "\n",
    "        train_ds, val_ds = load_train_val_ds()\n",
    "        test_ds = load_test_ds()\n",
    "\n",
    "        pred, label = evaluate_model(model, weights_path=args.best_val_path, dataset=test_ds, device=device)\n",
    "        cor_array = cor_in_time(pred, label)\n",
    "    #     print(\"best val model on test dataset, {:.3f}+-{:.3f}\".format(np.mean(cor_array), np.std(cor_array)))\n",
    "        pred = smoothing_with_np_conv(pred)\n",
    "        label = smoothing_with_np_conv(label)\n",
    "        # print(\"R2\", \"{:.6f}\".format(r2_score(label.T, pred.T)))\n",
    "        print(\"MSE\", \"{:.6f}\".format(mean_squared_error(label, pred)))\n",
    "        cor_array = cor_in_time(pred, label)\n",
    "        print(\"mean corr, {:.3f}+-{:.3f}\".format(np.mean(cor_array), np.std(cor_array)))\n",
    "        print(cor_array.shape)\n",
    "        np.save(\"corr_cnn_gru_shifter/{}_seq_{}.npy\".format(file_id, seq_len), cor_array)\n",
    "        # print(\"max corr\", \"{:.6f}\".format(np.max(cor_array)))\n",
    "        # print(\"min corr\", \"{:.6f}\".format(np.min(cor_array)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}