CNN_model_TSCV_train_predict.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential, load_model, Model\n",
    "from keras.layers import Input, Dense, Activation, Conv2D, Dropout, Flatten\n",
    "from keras import optimizers, utils, initializers, regularizers\n",
    "import keras.backend as K\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED_NUM = 10000\n",
    "pred_file_name = 'CNN_TSCV_prediction_Initialization_SEED' + str(SEED_NUM)\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = 'v10_170713_5x5_include_na_dataset.npz'\n",
    "label_path = \"v10_170713_5x5_include_na_label.npz\"\n",
    "x_tr_blended = np.load(data_path)['arr_0']\n",
    "y_tr_blended = np.load(label_path)['arr_0']\n",
    "\n",
    "all_dates = np.unique(x_tr_blended[:, 28*12+3])\n",
    "random.seed(1000)\n",
    "random_dates = random.sample(list(all_dates), len(all_dates)) \n",
    "x_tr = []; y_tr = []\n",
    "for day in random_dates:\n",
    "    x_tr.append(x_tr_blended[x_tr_blended[:, 28*12+3]==day])\n",
    "    y_tr.append(y_tr_blended[x_tr_blended[:, 28*12+3]==day])\n",
    "x_tr = np.vstack(x_tr)\n",
    "y_tr =np.hstack( y_tr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_train_validation(data_set, label_set, fold, k):\n",
    "    \"\"\"split train set and validation set\"\"\"\n",
    "    quo = int(len(data_set) / k)\n",
    "    \n",
    "    x_train = np.delete(data_set, range(quo*fold,quo*(fold+1)), 0)\n",
    "    y_train = np.delete(label_set, range(quo*fold,quo*(fold+1)), 0)\n",
    "    x_test = data_set[quo*fold:quo*(fold+1)]\n",
    "    y_test = label_set[quo*fold:quo*(fold+1)]\n",
    "    return x_train, y_train, x_test, y_test\n",
    "\n",
    "def norm_by_std_nan(train, val):\n",
    "    mask = np.ma.array(train, mask=np.isnan(train))\n",
    "    mean = np.mean(mask, 0)\n",
    "    std = np.std(mask, 0)\n",
    "\n",
    "    train = (train - mean) / std\n",
    "    train = np.where(train == np.nan, 0, train)\n",
    "    train = np.nan_to_num(train)\n",
    "\n",
    "    val = (val-mean)/std\n",
    "    val = np.where(val == np.nan, 0, val)\n",
    "    val = np.nan_to_num(val)\n",
    "    return train, val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "fold = 10\n",
    "epochs = 200\n",
    "tr_batch_size = 100\n",
    "ev_batch_size = 100\n",
    "Input_width = 5\n",
    "Input_height = 5\n",
    "num_channels = 28\n",
    "noise_std = 0.1\n",
    "\n",
    "n_conv = 64\n",
    "n_hidden = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /compuworks/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /compuworks/anaconda3/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From /compuworks/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Epoch 1/1\n",
      "224100/224100 [==============================] - 23s 102us/step - loss: 78.0858 - mean_absolute_error: 2.6644\n",
      "epoch:0, validation set r-squared:0.678767661843797\n",
      "Epoch 1/1\n",
      "224100/224100 [==============================] - 19s 87us/step - loss: 44.3956 - mean_absolute_error: 2.2982\n",
      "epoch:1, validation set r-squared:0.690820473038622\n",
      "Epoch 1/1\n",
      "224100/224100 [==============================] - 19s 84us/step - loss: 27.4595 - mean_absolute_error: 2.2512\n",
      "epoch:2, validation set r-squared:0.6952972658101993\n",
      "Epoch 1/1\n",
      "224100/224100 [==============================] - 19s 83us/step - loss: 20.9557 - mean_absolute_error: 2.2352\n",
      "epoch:3, validation set r-squared:0.6948371755806336\n",
      "Epoch 1/1\n",
      "224100/224100 [==============================] - 19s 85us/step - loss: 18.1932 - mean_absolute_error: 2.2227\n",
      "epoch:4, validation set r-squared:0.6979099784956586\n",
      "Epoch 1/1\n",
      "224100/224100 [==============================] - 18s 82us/step - loss: 16.6933 - mean_absolute_error: 2.2045\n",
      "epoch:5, validation set r-squared:0.702533250875217\n",
      "Epoch 1/1\n",
      " 77400/224100 [=========>....................] - ETA: 12s - loss: 15.3745 - mean_absolute_error: 2.1811"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-91430a352d05>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     21\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0;34m\"\"\"train\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m         \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtr_batch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m         \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mev_batch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_val\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0mval_r2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msquare\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_val\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mpred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msquare\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_val\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_val\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/compuworks/anaconda3/lib/python3.7/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1037\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1038\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1039\u001b[0;31m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1040\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1041\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m/compuworks/anaconda3/lib/python3.7/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m    202\u001b[0m                     \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 204\u001b[0;31m                 \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    205\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mcallback_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop_training\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    206\u001b[0m                     \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/compuworks/anaconda3/lib/python3.7/site-packages/keras/callbacks.py\u001b[0m in \u001b[0;36mon_batch_end\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m    115\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_delta_ts_batch_end\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mt_before_callbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 117\u001b[0;31m         \u001b[0mdelta_t_median\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmedian\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_delta_ts_batch_end\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    118\u001b[0m         if (self._delta_t_batch > 0. and\n\u001b[1;32m    119\u001b[0m            (delta_t_median > 0.95 * self._delta_t_batch and delta_t_median > 0.1)):\n",
      "\u001b[0;32m/compuworks/anaconda3/lib/python3.7/site-packages/numpy/lib/function_base.py\u001b[0m in \u001b[0;36mmedian\u001b[0;34m(a, axis, out, overwrite_input, keepdims)\u001b[0m\n\u001b[1;32m   3495\u001b[0m     \"\"\"\n\u001b[1;32m   3496\u001b[0m     r, k = _ureduce(a, func=_median, axis=axis, out=out,\n\u001b[0;32m-> 3497\u001b[0;31m                     overwrite_input=overwrite_input)\n\u001b[0m\u001b[1;32m   3498\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3499\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/compuworks/anaconda3/lib/python3.7/site-packages/numpy/lib/function_base.py\u001b[0m in \u001b[0;36m_ureduce\u001b[0;34m(a, func, **kwargs)\u001b[0m\n\u001b[1;32m   3403\u001b[0m         \u001b[0mkeepdim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3404\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3405\u001b[0;31m     \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3406\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeepdim\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3407\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/compuworks/anaconda3/lib/python3.7/site-packages/numpy/lib/function_base.py\u001b[0m in \u001b[0;36m_median\u001b[0;34m(a, axis, out, overwrite_input)\u001b[0m\n\u001b[1;32m   3548\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0missubdtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minexact\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0msz\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3549\u001b[0m         \u001b[0;31m# warn and return nans like mean would\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3550\u001b[0;31m         \u001b[0mrout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpart\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3551\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_median_nancheck\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3552\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for fold_num in range(fold):\n",
    "    save_directory = 'prediction/TSCV/fold'+str(fold_num)\n",
    "    if not os.path.exists(save_directory):\n",
    "        os.makedirs(save_directory)\n",
    "    x_train, y_train, x_val, y_val = split_train_validation(x_tr, y_tr, fold_num, fold)\n",
    "    x_train, x_val = norm_by_std_nan(x_train, x_val)\n",
    "    x_train = x_train.reshape(len(x_train), Input_width, Input_height, num_channels)\n",
    "    x_val   = x_val.reshape(len(x_val), Input_width, Input_height, num_channels)\n",
    "    \n",
    "    model = Sequential([\n",
    "    Conv2D(n_conv, (3,3), kernel_initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.1, seed=SEED_NUM), bias_initializer=initializers.Constant(0.1), input_shape=(Input_width, Input_height, num_channels), padding='same', kernel_regularizer=regularizers.l1(0.005)),\n",
    "    Activation('relu'),\n",
    "    Flatten(),\n",
    "    Dropout(0.5, seed=SEED_NUM),\n",
    "    Dense(n_hidden, activation='elu', kernel_initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.1, seed=SEED_NUM), bias_initializer=initializers.Constant(0.1), kernel_regularizer=regularizers.l1(0.005)),\n",
    "    Dense(1, activation='linear')\n",
    "    ])\n",
    "    nadam = optimizers.Nadam(lr=0.0002, beta_1=0.9, beta_2=0.999, epsilon=None, schedule_decay=0.004)\n",
    "    model.compile(optimizer=nadam, loss='mse', metrics=['mae'])\n",
    "    \n",
    "    for epoch in range(epochs):\n",
    "        \"\"\"train\"\"\"\n",
    "        model.fit(x_train, y_train, epochs=1, batch_size=tr_batch_size, verbose=1)\n",
    "        pred = model.predict(x_val, batch_size=ev_batch_size).reshape(len(x_val),)\n",
    "        val_r2 = 1 - (np.sum(np.square(y_val - pred)) / np.sum(np.square(y_val - np.mean(y_val))))\n",
    "        print(\"epoch:{}, validation set r-squared:{}\".format(epoch, val_r2))\n",
    "\n",
    "        sv_pred = np.array(pred).reshape(len(pred), 1)\n",
    "        np.savetxt(save_directory + '/' + pred_file_name + '_epoch' + str(epoch) + \".csv\", sv_pred, delimiter=',')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}