19_DL_LSTM_starter.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1c0b6eac",
   "metadata": {},
   "source": [
    "# Deep learning - LSTM "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a9e6925",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.keras import Model, Sequential\n",
    "\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.losses import MeanSquaredError\n",
    "from tensorflow.keras.metrics import MeanAbsoluteError\n",
    "\n",
    "from tensorflow.keras.layers import Dense, Conv1D, LSTM, RNN\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6c933c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams[\"figure.figsize\"] = (9,6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7729e378",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "tf.keras.utils.set_random_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50ed85ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv('data/train.csv')\n",
    "val_df = pd.read_csv('data/val.csv')\n",
    "test_df = pd.read_csv('data/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f48ef7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_timeseries_data(df, input_length, output_length, target_names):\n",
    "    \n",
    "    if target_names is not None:\n",
    "        target_indices = {name: i for i, name in enumerate(target_names)}\n",
    "    col_indices = {name: i for i, name in enumerate(df.columns)}\n",
    "    \n",
    "    total_length = input_length + output_length\n",
    "    \n",
    "    input_slice = slice(0, input_length)\n",
    "    output_slice = slice(input_length, None)\n",
    "    \n",
    "    data = np.array(df, dtype=np.float32)\n",
    "    ds = tf.keras.preprocessing.timeseries_dataset_from_array(\n",
    "        data=data,\n",
    "        targets=None,\n",
    "        sequence_length=total_length,\n",
    "        sequence_stride=1,\n",
    "        shuffle=False,\n",
    "        batch_size=32\n",
    "    )\n",
    "    \n",
    "    def split_to_input_output(x):\n",
    "            \n",
    "        inputs = x[:, input_slice, :]\n",
    "        outputs = x[:, output_slice, :]\n",
    "        \n",
    "        if target_names is not None:\n",
    "            outputs = tf.stack(\n",
    "                [outputs[:,:,col_indices[name]] for name in target_names],\n",
    "                axis=-1\n",
    "            )\n",
    "\n",
    "        inputs.set_shape([None, input_length, None])\n",
    "        outputs.set_shape([None, output_length, None])\n",
    "    \n",
    "        return inputs, outputs\n",
    "    \n",
    "    ds = ds.map(split_to_input_output)\n",
    "    \n",
    "    return ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b66a19f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.callbacks import LearningRateScheduler\n",
    "\n",
    "def scheduler(epoch, lr):\n",
    "    if epoch < 2:\n",
    "        return lr\n",
    "    else:\n",
    "        return lr * tf.math.exp(-0.1)\n",
    "    \n",
    "lr_scheduler = LearningRateScheduler(scheduler)\n",
    "\n",
    "def train_model(model, train_ds, val_ds, patience=5, max_epochs=50, use_scheduler=False):\n",
    "    \n",
    "    early_stopping = EarlyStopping(monitor='val_loss', patience=patience, mode='min')\n",
    "    \n",
    "    model.compile(loss=MeanSquaredError(), optimizer=Adam(), metrics=[MeanAbsoluteError()])\n",
    "    \n",
    "    history = model.fit(train_ds, epochs=max_epochs, validation_data=val_ds, callbacks=[early_stopping])\n",
    "    \n",
    "    if use_scheduler == True:\n",
    "        history = model.fit(train_ds, epochs=max_epochs, validation_data=val_ds, callbacks=[early_stopping, lr_scheduler])\n",
    "    \n",
    "    return history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4661c862",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_history(history):\n",
    "\n",
    "    fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2)\n",
    "\n",
    "    ax1.plot(history.history['loss'], label='Train')\n",
    "    ax1.plot(history.history['val_loss'], label='Validation')\n",
    "    ax1.set_xlabel('Epochs')\n",
    "    ax1.set_ylabel('Loss')\n",
    "    ax1.set_title('Loss')\n",
    "    ax1.legend(loc='best')\n",
    "\n",
    "    ax2.plot(history.history['mean_absolute_error'], label='Train')\n",
    "    ax2.plot(history.history['val_mean_absolute_error'], label='Validation')\n",
    "    ax2.set_xlabel('Epochs')\n",
    "    ax2.set_ylabel('MAE')\n",
    "    ax2.set_title('Mean absolute error')\n",
    "    ax2.legend(loc='best')\n",
    "\n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6583960f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_predictions(model, sample_batch, model_type):\n",
    "    \n",
    "    inputs, outputs = sample_batch\n",
    "    preds = model(inputs)\n",
    "    \n",
    "    if model_type == 'single_step':      \n",
    "\n",
    "        fig, ax = plt.subplots()\n",
    "\n",
    "        actual_scaled = outputs.numpy().flatten()\n",
    "        actual = actual_scaled * (OT_max_test - OT_min_test) + OT_min_test\n",
    "        \n",
    "        predictions_scaled = preds.numpy().flatten()\n",
    "        predictions = predictions_scaled * (OT_max_train - OT_min_train) + OT_min_train\n",
    "        \n",
    "        ax.plot(actual, label='Actual')\n",
    "        ax.plot(predictions, label='Predicted')\n",
    "        ax.set_xlabel('Time steps')\n",
    "        ax.set_ylabel('Oil temperature')\n",
    "        ax.set_title('Predictions on a sample batch')\n",
    "        ax.legend(loc='best')\n",
    "        \n",
    "    elif model_type == 'multi_step':\n",
    "\n",
    "        fig, ax = plt.subplots()\n",
    "        \n",
    "        actual_scaled = outputs.numpy().flatten()[:48]\n",
    "        actual = actual_scaled * (OT_max_test - OT_min_test) + OT_min_test\n",
    "        \n",
    "        predictions_scaled = preds.numpy().flatten()[:48]\n",
    "        predictions = predictions_scaled * (OT_max_train - OT_min_train) + OT_min_train\n",
    "\n",
    "        ax.plot(actual, label='Actual')\n",
    "        ax.plot(predictions, label='Predicted')\n",
    "        ax.set_xlabel('Time steps')\n",
    "        ax.set_ylabel('Oil temperature')\n",
    "        ax.set_title('Predictions on a sample batch')\n",
    "        ax.legend(loc='best')\n",
    "        \n",
    "    elif model_type == 'multi_output':\n",
    "        \n",
    "        fig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1)\n",
    "        \n",
    "        OT_actual_scaled = outputs.numpy().flatten()[0::2]\n",
    "        OT_actual = OT_actual_scaled * (OT_max_test - OT_min_test) + OT_min_test\n",
    "        \n",
    "        OT_predictions_scaled = preds.numpy().flatten()[0::2]\n",
    "        OT_predictions = OT_predictions_scaled * (OT_max_train - OT_min_train) + OT_min_train\n",
    "\n",
    "        MULL_actual_scaled = outputs.numpy().flatten()[1::2]\n",
    "        MULL_actual = MULL_actual_scaled * (MULL_max_test - MULL_min_test) + MULL_min_test\n",
    "        \n",
    "        MULL_predictions_scaled = preds.numpy().flatten()[1::2]\n",
    "        MULL_predictions = MULL_predictions_scaled * (MULL_max_train - MULL_min_train) + MULL_min_train\n",
    "        \n",
    "        ax1.plot(OT_actual, label='Actual')\n",
    "        ax1.plot(OT_predictions, label='Predicted')\n",
    "        ax1.set_xlabel('Time')\n",
    "        ax1.set_ylabel('Oil temperature')\n",
    "        ax1.legend(loc='best')\n",
    "        ax1.set_title('Predictions on a sample batch')\n",
    "        \n",
    "        ax2.plot(MULL_actual, label='Actual')\n",
    "        ax2.plot(MULL_predictions, label='Predicted')\n",
    "        ax2.set_xlabel('Time')\n",
    "        ax2.set_ylabel('MULL')\n",
    "        ax2.legend(loc='best')\n",
    "        \n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80084664",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_evaluation(model_list, mae):\n",
    "    \n",
    "    fig, ax = plt.subplots()\n",
    "    \n",
    "    ax.bar(model_list, mae, width=0.3)\n",
    "    ax.set_ylabel('MAE')\n",
    "    ax.set_xlabel('Models')\n",
    "    ax.set_ylim(0, max(mae)+0.15)\n",
    "    for index, value in enumerate(mae):\n",
    "        ax.text(x=index, y=value+0.005, s=str(round(value, 3)), ha='center')\n",
    "    \n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8e8c4ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "MULL_max_train = 7.568999767303468\n",
    "MULL_min_train = -5.934000015258789\n",
    "MULL_max_test = 4.690999984741211\n",
    "MULL_min_test = -3.3399999141693115\n",
    "\n",
    "OT_max_train = 46.00699996948242\n",
    "OT_min_train = -4.079999923706056\n",
    "OT_max_test = 17.165000915527347\n",
    "OT_min_test = 3.025000095367432"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "047505fe",
   "metadata": {},
   "source": [
    "## Single-step model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "438e40d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds_ss = format_timeseries_data(train_df, 1, 1, ['OT'])\n",
    "val_ds_ss = format_timeseries_data(val_df, 1, 1, ['OT'])\n",
    "test_ds_ss = format_timeseries_data(test_df, 1, 1, ['OT'])\n",
    "\n",
    "ss_sample_batch = next(iter(test_ds_ss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46e87942",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Create a single-step LSTM model\n",
    "\n",
    "ss_lstm_history = train_model(ss_lstm, train_ds_ss, val_ds_ss, use_scheduler=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73ae208d",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(ss_lstm_history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "921eb487",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_predictions(ss_lstm, ss_sample_batch, 'single_step')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45bdc199",
   "metadata": {},
   "source": [
    "### Evaluation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5d9eda9",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_list = ['Linear', 'DNN', 'LSTM']\n",
    "\n",
    "ss_mae = [0.032, 0.046]\n",
    "ss_mae.append(ss_lstm.evaluate(test_ds_ss)[1])\n",
    "\n",
    "plot_evaluation(model_list, ss_mae)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64d0f6c6",
   "metadata": {},
   "source": [
    "## Multi-step model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a835ea8",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds_ms = format_timeseries_data(train_df, 24, 24, ['OT'])\n",
    "val_ds_ms = format_timeseries_data(val_df, 24, 24, ['OT'])\n",
    "test_ds_ms = format_timeseries_data(test_df, 24, 24, ['OT'])\n",
    "\n",
    "ms_sample_batch = next(iter(test_ds_ms))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ccfcb1c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Create a multi-step LSTM model\n",
    "\n",
    "ms_lstm_history = train_model(ms_lstm, train_ds_ms, val_ds_ms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7cf2479",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(ms_lstm_history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b7378b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_predictions(ms_lstm, ms_sample_batch, 'multi_step')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f8b7356",
   "metadata": {},
   "outputs": [],
   "source": [
    "ms_mae = [0.123, 0.154]\n",
    "ms_mae.append(ms_lstm.evaluate(test_ds_ms)[1])\n",
    "\n",
    "plot_evaluation(model_list, ms_mae)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "005e4e8e",
   "metadata": {},
   "source": [
    "## Multi-output model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78fabe16",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds_mo = format_timeseries_data(train_df, 1, 1, ['OT', 'MULL'])\n",
    "val_ds_mo = format_timeseries_data(val_df, 1, 1, ['OT', 'MULL'])\n",
    "test_ds_mo = format_timeseries_data(test_df, 1, 1, ['OT', 'MULL'])\n",
    "\n",
    "mo_sample_batch = next(iter(test_ds_mo))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "850009c0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Create a multi-output LSTM model\n",
    "\n",
    "mo_lstm_history = train_model(mo_lstm, train_ds_mo, val_ds_mo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9034a032",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(mo_lstm_history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a80a38a",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_predictions(mo_lstm, mo_sample_batch, 'multi_output')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "858a5965",
   "metadata": {},
   "outputs": [],
   "source": [
    "mo_mae = [0.05, 0.069]\n",
    "mo_mae.append(mo_lstm.evaluate(test_ds_mo)[1])\n",
    "\n",
    "plot_evaluation(model_list, mo_mae)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c9065d1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}