18_DL_DNN_starter.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "48b5fa31",
   "metadata": {},
   "source": [
    "# Deep learning - Deep neural networks "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "871a9a79",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.keras import Model, Sequential\n",
    "\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.losses import MeanSquaredError\n",
    "from tensorflow.keras.metrics import MeanAbsoluteError\n",
    "\n",
    "from tensorflow.keras.layers import Dense, Conv1D, LSTM, RNN\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf145010",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams[\"figure.figsize\"] = (9,6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a20fdf2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "tf.keras.utils.set_random_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d0623fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv('data/train.csv')\n",
    "val_df = pd.read_csv('data/val.csv')\n",
    "test_df = pd.read_csv('data/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec472d65",
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_timeseries_data(df, input_length, output_length, target_names):\n",
    "    \n",
    "    if target_names is not None:\n",
    "        target_indices = {name: i for i, name in enumerate(target_names)}\n",
    "    col_indices = {name: i for i, name in enumerate(df.columns)}\n",
    "    \n",
    "    total_length = input_length + output_length\n",
    "    \n",
    "    input_slice = slice(0, input_length)\n",
    "    output_slice = slice(input_length, None)\n",
    "    \n",
    "    data = np.array(df, dtype=np.float32)\n",
    "    ds = tf.keras.preprocessing.timeseries_dataset_from_array(\n",
    "        data=data,\n",
    "        targets=None,\n",
    "        sequence_length=total_length,\n",
    "        sequence_stride=1,\n",
    "        shuffle=False,\n",
    "        batch_size=32\n",
    "    )\n",
    "    \n",
    "    def split_to_input_output(x):\n",
    "            \n",
    "        inputs = x[:, input_slice, :]\n",
    "        outputs = x[:, output_slice, :]\n",
    "        \n",
    "        if target_names is not None:\n",
    "            outputs = tf.stack(\n",
    "                [outputs[:,:,col_indices[name]] for name in target_names],\n",
    "                axis=-1\n",
    "            )\n",
    "\n",
    "        inputs.set_shape([None, input_length, None])\n",
    "        outputs.set_shape([None, output_length, None])\n",
    "    \n",
    "        return inputs, outputs\n",
    "    \n",
    "    ds = ds.map(split_to_input_output)\n",
    "    \n",
    "    return ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e33d5933",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, train_ds, val_ds, patience=5, max_epochs=50):\n",
    "    \n",
    "    early_stopping = EarlyStopping(monitor='val_loss', patience=patience, mode='min')\n",
    "    \n",
    "    model.compile(loss=MeanSquaredError(), optimizer=Adam(), metrics=[MeanAbsoluteError()])\n",
    "    \n",
    "    history = model.fit(train_ds, epochs=max_epochs, validation_data=val_ds, callbacks=[early_stopping])\n",
    "    \n",
    "    return history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b9f7704",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_history(history):\n",
    "\n",
    "    fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2)\n",
    "\n",
    "    ax1.plot(history.history['loss'], label='Train')\n",
    "    ax1.plot(history.history['val_loss'], label='Validation')\n",
    "    ax1.set_xlabel('Epochs')\n",
    "    ax1.set_ylabel('Loss')\n",
    "    ax1.set_title('Loss')\n",
    "    ax1.legend(loc='best')\n",
    "\n",
    "    ax2.plot(history.history['mean_absolute_error'], label='Train')\n",
    "    ax2.plot(history.history['val_mean_absolute_error'], label='Validation')\n",
    "    ax2.set_xlabel('Epochs')\n",
    "    ax2.set_ylabel('MAE')\n",
    "    ax2.set_title('Mean absolute error')\n",
    "    ax2.legend(loc='best')\n",
    "\n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c85f462",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_predictions(model, sample_batch, model_type):\n",
    "    \n",
    "    inputs, outputs = sample_batch\n",
    "    preds = model(inputs)\n",
    "    \n",
    "    if model_type == 'single_step':      \n",
    "\n",
    "        fig, ax = plt.subplots()\n",
    "\n",
    "        actual_scaled = outputs.numpy().flatten()\n",
    "        actual = actual_scaled * (OT_max_test - OT_min_test) + OT_min_test\n",
    "        \n",
    "        predictions_scaled = preds.numpy().flatten()\n",
    "        predictions = predictions_scaled * (OT_max_train - OT_min_train) + OT_min_train\n",
    "        \n",
    "        ax.plot(actual, label='Actual')\n",
    "        ax.plot(predictions, label='Predicted')\n",
    "        ax.set_xlabel('Time steps')\n",
    "        ax.set_ylabel('Oil temperature')\n",
    "        ax.set_title('Predictions on a sample batch')\n",
    "        ax.legend(loc='best')\n",
    "        \n",
    "    elif model_type == 'multi_step':\n",
    "\n",
    "        fig, ax = plt.subplots()\n",
    "        \n",
    "        actual_scaled = outputs.numpy().flatten()[:48]\n",
    "        actual = actual_scaled * (OT_max_test - OT_min_test) + OT_min_test\n",
    "        \n",
    "        predictions_scaled = preds.numpy().flatten()[:48]\n",
    "        predictions = predictions_scaled * (OT_max_train - OT_min_train) + OT_min_train\n",
    "\n",
    "        ax.plot(actual, label='Actual')\n",
    "        ax.plot(predictions, label='Predicted')\n",
    "        ax.set_xlabel('Time steps')\n",
    "        ax.set_ylabel('Oil temperature')\n",
    "        ax.set_title('Predictions on a sample batch')\n",
    "        ax.legend(loc='best')\n",
    "        \n",
    "    elif model_type == 'multi_output':\n",
    "        \n",
    "        fig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1)\n",
    "        \n",
    "        OT_actual_scaled = outputs.numpy().flatten()[0::2]\n",
    "        OT_actual = OT_actual_scaled * (OT_max_test - OT_min_test) + OT_min_test\n",
    "        \n",
    "        OT_predictions_scaled = preds.numpy().flatten()[0::2]\n",
    "        OT_predictions = OT_predictions_scaled * (OT_max_train - OT_min_train) + OT_min_train\n",
    "\n",
    "        MULL_actual_scaled = outputs.numpy().flatten()[1::2]\n",
    "        MULL_actual = MULL_actual_scaled * (MULL_max_test - MULL_min_test) + MULL_min_test\n",
    "        \n",
    "        MULL_predictions_scaled = preds.numpy().flatten()[1::2]\n",
    "        MULL_predictions = MULL_predictions_scaled * (MULL_max_train - MULL_min_train) + MULL_min_train\n",
    "        \n",
    "        ax1.plot(OT_actual, label='Actual')\n",
    "        ax1.plot(OT_predictions, label='Predicted')\n",
    "        ax1.set_xlabel('Time')\n",
    "        ax1.set_ylabel('Oil temperature')\n",
    "        ax1.legend(loc='best')\n",
    "        ax1.set_title('Predictions on a sample batch')\n",
    "        \n",
    "        ax2.plot(MULL_actual, label='Actual')\n",
    "        ax2.plot(MULL_predictions, label='Predicted')\n",
    "        ax2.set_xlabel('Time')\n",
    "        ax2.set_ylabel('MULL')\n",
    "        ax2.legend(loc='best')\n",
    "        \n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f5f3306",
   "metadata": {},
   "outputs": [],
   "source": [
    "MULL_max_train = 7.568999767303468\n",
    "MULL_min_train = -5.934000015258789\n",
    "MULL_max_test = 4.690999984741211\n",
    "MULL_min_test = -3.3399999141693115\n",
    "\n",
    "OT_max_train = 46.00699996948242\n",
    "OT_min_train = -4.079999923706056\n",
    "OT_max_test = 17.165000915527347\n",
    "OT_min_test = 3.025000095367432"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89a3072f",
   "metadata": {},
   "source": [
    "## Single-step model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "590c4f75",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds_ss = format_timeseries_data(train_df, 1, 1, ['OT'])\n",
    "val_ds_ss = format_timeseries_data(val_df, 1, 1, ['OT'])\n",
    "test_ds_ss = format_timeseries_data(test_df, 1, 1, ['OT'])\n",
    "\n",
    "ss_sample_batch = next(iter(test_ds_ss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f647530",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Create a single-step DNN\n",
    "\n",
    "ss_dnn_history = train_model(ss_dnn, train_ds_ss, val_ds_ss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "587d7761",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(ss_dnn_history)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a54a68d3",
   "metadata": {},
   "source": [
    "### Add a learning rate scheduler "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c46c200",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add a learning rate scheduler\n",
    "    \n",
    "lr_scheduler = LearningRateScheduler(scheduler)\n",
    "\n",
    "def train_model(model, train_ds, val_ds, patience=5, max_epochs=50, use_scheduler=False):\n",
    "    \n",
    "    early_stopping = EarlyStopping(monitor='val_loss', patience=patience, mode='min')\n",
    "    \n",
    "    model.compile(loss=MeanSquaredError(), optimizer=Adam(), metrics=[MeanAbsoluteError()])\n",
    "    \n",
    "    history = model.fit(train_ds, epochs=max_epochs, validation_data=val_ds, callbacks=[early_stopping])\n",
    "    \n",
    "    if use_scheduler == True:\n",
    "        # Add the scheduler as a callback\n",
    "    \n",
    "    return history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5d70d1d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ss_dnn_history = train_model(ss_dnn, train_ds_ss, val_ds_ss, use_scheduler=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aed16e7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(ss_dnn_history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d03de18c",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_predictions(ss_dnn, ss_sample_batch, 'single_step')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "254c7cb2",
   "metadata": {},
   "source": [
    "### Evaluation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee99f82f",
   "metadata": {},
   "outputs": [],
   "source": [
    "ss_mae = [0.0318]\n",
    "ss_mae.append(ss_dnn.evaluate(test_ds_ss)[1])\n",
    "\n",
    "x = ['Linear', 'DNN']\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "ax.bar(x, ss_mae, width=0.3)\n",
    "ax.set_ylabel('MAE')\n",
    "ax.set_xlabel('Models')\n",
    "ax.set_ylim(0, max(ss_mae)+0.1)\n",
    "for index, value in enumerate(ss_mae):\n",
    "    ax.text(x=index, y=value+0.005, s=str(round(value, 3)), ha='center')\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "041df913",
   "metadata": {},
   "source": [
    "## Multi-step model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab92b207",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds_ms = format_timeseries_data(train_df, 24, 24, ['OT'])\n",
    "val_ds_ms = format_timeseries_data(val_df, 24, 24, ['OT'])\n",
    "test_ds_ms = format_timeseries_data(test_df, 24, 24, ['OT'])\n",
    "\n",
    "ms_sample_batch = next(iter(test_ds_ms))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c3e911f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Create a multi-step DNN model\n",
    "\n",
    "ms_dnn_history = train_model(ms_dnn, train_ds_ms, val_ds_ms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4dbce187",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(ms_dnn_history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fdf6d4d0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ms_dnn_history = train_model(ms_dnn, train_ds_ms, val_ds_ms, use_scheduler=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c94e581",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(ms_dnn_history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0dd4f2b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_predictions(ms_dnn, ms_sample_batch, 'multi_step')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce3b1b1d",
   "metadata": {},
   "source": [
    "### Evaluation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bc4f46a",
   "metadata": {},
   "outputs": [],
   "source": [
    "ms_mae = [0.1229]\n",
    "ms_mae.append(ms_dnn.evaluate(test_ds_ms)[1])\n",
    "\n",
    "x = ['Linear', 'DNN']\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "ax.bar(x, ms_mae, width=0.3)\n",
    "ax.set_ylabel('MAE')\n",
    "ax.set_xlabel('Models')\n",
    "ax.set_ylim(0, max(ms_mae)+0.15)\n",
    "for index, value in enumerate(ms_mae):\n",
    "    ax.text(x=index, y=value+0.005, s=str(round(value, 3)), ha='center')\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8bc24d79",
   "metadata": {},
   "source": [
    "## Multi-output model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f47e849c",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds_mo = format_timeseries_data(train_df, 1, 1, ['OT', 'MULL'])\n",
    "val_ds_mo = format_timeseries_data(val_df, 1, 1, ['OT', 'MULL'])\n",
    "test_ds_mo = format_timeseries_data(test_df, 1, 1, ['OT', 'MULL'])\n",
    "\n",
    "mo_sample_batch = next(iter(test_ds_mo))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f3b0558",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Create a multi-output DNN\n",
    "\n",
    "mo_dnn_history = train_model(mo_dnn, train_ds_mo, val_ds_mo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7ebb859",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(mo_dnn_history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad650da9",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_predictions(mo_dnn, mo_sample_batch, 'multi_output')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a43b23f1",
   "metadata": {},
   "source": [
    "### Evaluation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8524c56e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_evaluation(model_list, mae):\n",
    "    \n",
    "    fig, ax = plt.subplots()\n",
    "    \n",
    "    ax.bar(model_list, mae, width=0.3)\n",
    "    ax.set_ylabel('MAE')\n",
    "    ax.set_xlabel('Models')\n",
    "    ax.set_ylim(0, max(mae)+0.15)\n",
    "    for index, value in enumerate(mae):\n",
    "        ax.text(x=index, y=value+0.005, s=str(round(value, 3)), ha='center')\n",
    "    \n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7eec27a",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_list = ['Linear', 'DNN']\n",
    "\n",
    "mo_mae = [0.0495]\n",
    "mo_mae.append(mo_dnn.evaluate(test_ds_mo)[1])\n",
    "\n",
    "plot_evaluation(model_list, mo_mae)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00674342",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}