LSTM_Example_Model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "LSTM_Model.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "2Bysm3TXx1Pw"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/JasperLS/data_science_intros/blob/main/LSTM_Example_Model.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j6ktCJ6vtajP"
      },
      "source": [
        "import glob\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JLK44J7PGsLF"
      },
      "source": [
        "# **Vorbereitung Daten** (Preprocessing)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3w2n9eETndh5"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from IPython.core.interactiveshell import InteractiveShell\n",
        "InteractiveShell.ast_node_interactivity = \"all\"\n",
        "pd.set_option('display.max_columns', None)\n",
        "from sklearn.utils import shuffle"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iN4Aub5S21WW"
      },
      "source": [
        "# Randomize Index to get shuffled arrays \n",
        "df_ivs = pd.read_csv('/content/drive/My Drive/computer vision intro/data_final_ivs.csv')\n",
        "df_dvs = pd.read_csv('/content/drive/My Drive/computer vision intro/data_final_dvs.csv')\n",
        "lstm_ivs = np.load('/content/drive/My Drive/computer vision intro/lstm_all_ivs.npy')\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xqins9xAb_OX"
      },
      "source": [
        "lstm_ivs.shape\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gp0bmvoCswd3"
      },
      "source": [
        "#Berücksichtigung der Dummy Variable \"category\" im DataFrame der unabhängigen Variablen\n",
        "dummy_var = pd.get_dummies(df_ivs['category'], prefix='category', drop_first=True)\n",
        "df = pd.concat([df_ivs, dummy_var], axis = 1)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "so0ecShvh2DX"
      },
      "source": [
        "#separate Dummy for LSTM\n",
        "df_dummys = df.loc[:,'category_1':]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r1JRkLmBOfri"
      },
      "source": [
        "# Delete some columns\n",
        "del df_ivs['category']\n",
        "del df['category']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VRBnW2hXpGSv"
      },
      "source": [
        "# Select DV\n",
        "df_dvs = df_dvs[['likability']]*100 "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8ez0WjEniUjD"
      },
      "source": [
        "df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0wf4I23gpIR1"
      },
      "source": [
        "# Select IVs\n",
        "df = df[['colorfulness_avg','saturation_avg', 'value_avg', 'black_avg', 'blue_avg', 'brown_avg', 'grey_avg','green_avg', 'orange_avg','pink_avg','purple_avg','red_avg','yellow_avg','white_avg','avg_scene_freq','category_1','category_2','category_3','category_4','category_5','category_6','category_7','category_8','category_9','category_10','category_11','category_13','category_14','category_15','category_16','category_17','category_18','category_19','category_20']]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FSrm8-MYMJF_"
      },
      "source": [
        "df.head(2)\n",
        "df_dvs.head(2)\n",
        "df_dummys.head(2)\n",
        "df.shape\n",
        "df_dvs.shape\n",
        "df_dummys.shape\n",
        "lstm_ivs.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I4iFiAFqM0JG"
      },
      "source": [
        "'''\n",
        "\n",
        "What's the third dimension of LSTM IVs?\n",
        "\n",
        "Why do we have a separate dummy?\n",
        "\n",
        "'''"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "13HH-W7xHLtm"
      },
      "source": [
        "### Training- und Test-Set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MmCfWhZ4rjrM"
      },
      "source": [
        "#Unterteilung in Training- und Test-Set\n",
        "split_size=2487 #80% Train- und 20% Test-Set\n",
        "#Regression\n",
        "y_train_reg = df_dvs[:split_size]\n",
        "y_test_reg = df_dvs[split_size:]\n",
        "X_train_reg = df[:split_size]\n",
        "X_test_reg = df[split_size:]\n",
        "#LSTM\n",
        "y_train_lstm = df_dvs[:split_size]\n",
        "y_test_lstm = df_dvs[split_size:]\n",
        "dummys_train_lstm = df_dummys[:split_size]\n",
        "dummys_test_lstm = df_dummys[split_size:]\n",
        "iv_lstm_train = lstm_ivs[:split_size,:,:]\n",
        "iv_lstm_test = lstm_ivs[split_size:,:,:]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YfTvztxHztkh"
      },
      "source": [
        "# **Regression**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iI4At6eqPjls"
      },
      "source": [
        "%matplotlib inline"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oL6NN_-5zGAx"
      },
      "source": [
        "import statsmodels.api as sm\n",
        "from statsmodels.stats.outliers_influence import variance_inflation_factor\n",
        "from sklearn.linear_model import LinearRegression\n",
        "from sklearn import metrics\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dniSykIezDG8"
      },
      "source": [
        "'''\n",
        "\n",
        "What problem do we probably encounter?\n",
        "\n",
        "'''"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2Bysm3TXx1Pw"
      },
      "source": [
        "## SKIP: Plots"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ifoTz65zfg05"
      },
      "source": [
        "#Plot\n",
        "# plt.figure(figsize=(20,7))\n",
        "\n",
        "# y1 = df_dvs\n",
        "# y2 = df_dvs\n",
        "# x1 = df[['colorfulness_avg']]\n",
        "# x2 = df[['avg_scene_freq']]\n",
        "\n",
        "# plt.subplot(121)\n",
        "# plt.scatter(x1, y1)\n",
        "# plt.xlim(0.0, 1.0)\n",
        "# plt.ylim(0.0, 2.0)\n",
        "# plt.ylabel('Click-Through-Rate', fontsize=14)\n",
        "# plt.xlabel('Colorfulness', fontsize=14)\n",
        "\n",
        "# plt.subplot(122)\n",
        "# plt.scatter(x2, y2)\n",
        "# plt.xlim(0.0, 1.0)\n",
        "# plt.ylim(0.0, 2.0)\n",
        "# plt.ylabel('Click-Through-Rate', fontsize=14)\n",
        "# plt.xlabel('Szenen-Frequenz', fontsize=14)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4y49pu5S4Yek"
      },
      "source": [
        "## Modell"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aTKmDHHmzsHJ"
      },
      "source": [
        "X = df\n",
        "Y = df_dvs\n",
        "# X = sm.add_constant(X)\n",
        "model = sm.OLS(Y.astype(float), X.astype(float))\n",
        "results = model.fit()\n",
        "print(results.summary())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sAxIZQ5lMMV9"
      },
      "source": [
        "# plt.rc('figure', figsize=(12, 7))\n",
        "# plt.text(0.01, 0.05, str(results.summary()), {'fontsize': 10}, fontproperties = 'monospace') \n",
        "# plt.axis('off')\n",
        "# plt.tight_layout()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fWD2Yvf44ge_"
      },
      "source": [
        "## VIF Check"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fk4gd5tvzzZi"
      },
      "source": [
        "vif_data = pd.DataFrame() \n",
        "vif_data[\"feature\"] = X.columns\n",
        "\n",
        "vif_data[\"VIF\"] = [variance_inflation_factor(X.values, i) for i in range(len(X.columns))]\n",
        "\n",
        "print(vif_data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oOf7JPabsGAq"
      },
      "source": [
        "plt.rc('figure', figsize=(12, 7))\n",
        "plt.text(0.01, 0.05, str(vif_data), {'fontsize': 10}, fontproperties = 'monospace') \n",
        "plt.axis('off')\n",
        "plt.tight_layout()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Uxhpftiu4rfo"
      },
      "source": [
        "## Training\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qMgDjrfJ9izP"
      },
      "source": [
        "# Repeat training for train set only for later comparison\n",
        "lin_reg_mod = LinearRegression()\n",
        "lin_reg_mod.fit(X_train_reg, y_train_reg)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MGVLf-kZ4-LS"
      },
      "source": [
        "## Test & Prediction"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AguVMd9K0EyV"
      },
      "source": [
        "y_pred_reg = lin_reg_mod.predict(X_test_reg)\n",
        "\n",
        "df_compare_reg = pd.DataFrame({'Tatsächlicher Wert': y_test_reg.values.flatten(), 'Vorhersage': y_pred_reg.flatten()})\n",
        "df_compare_reg"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fPm5vePa0J4h"
      },
      "source": [
        "# df1 = df_compare_reg.head(50)\n",
        "# df1.plot(kind='bar',figsize=(16,10))\n",
        "# plt.grid(which='major', linestyle='-', linewidth='0.5', color='green')\n",
        "# plt.grid(which='minor', linestyle=':', linewidth='0.5', color='black')\n",
        "# plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ygVs98-r5QtJ"
      },
      "source": [
        "## MSE"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UC0jw5id0Kw_"
      },
      "source": [
        "#Berechnung MSE\n",
        "reg_mse = metrics.mean_squared_error(y_test_reg, y_pred_reg)\n",
        "#Rücktransformation des MSE\n",
        "reg_mse = reg_mse/10000\n",
        "\n",
        "print('Die Regression hat einen MSE von', reg_mse)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1Nb2jKVT0PEO"
      },
      "source": [
        "# **Long Short-Term Memory (LSTM)**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oIjSzIWSthSn"
      },
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.layers import LSTM, Input, Dense, Dropout, Concatenate\n",
        "from tensorflow.keras.utils import to_categorical,plot_model\n",
        "from tensorflow.keras import optimizers\n",
        "\n",
        "pd.options.display.float_format = \"{:,.5f}\".format"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TV5gibi5xKY9"
      },
      "source": [
        "'''\n",
        "\n",
        "\n",
        "Before we continue:\n",
        "What are potential tuning parameters?\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "'''"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UA7YI8yY5Xcu"
      },
      "source": [
        "## Modell"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nrtuNkBpx0ho"
      },
      "source": [
        "## Parameter LSTM\n",
        "lstm_shape = 10 \n",
        "main_shape = 24 # [8,16,24,30]\n",
        "learn = 0.3 # [0.1,0.2,0.3,0.4]\n",
        "epochs = 150 # [50,100,150,200]\n",
        "batch_size = 64 # [16,32,64,128]\n",
        "drop = 0.3 # [0.1,0.2,0.3,0.4]\n",
        "dense = 12\n",
        "opt = optimizers.Adadelta(learning_rate=learn) # [Adadelta,Adam,Adamax,Adagrad,SGD]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mZrrlA0E1ZKu"
      },
      "source": [
        "#Modellaufbau / -struktur\n",
        "first_input = Input(shape=(lstm_ivs.shape[1:]), name='first_input')\n",
        "lstm_out = LSTM(lstm_shape,dropout= drop,return_sequences = False,return_state=False)(first_input)\n",
        "first = Dense(dense)(lstm_out)\n",
        "first = Dropout(drop)(first)\n",
        "\n",
        "second_input = Input(shape = df_dummys.shape[1], name='dummys')\n",
        "second = Dense(main_shape,activation = 'relu')(second_input)\n",
        "\n",
        "l = Concatenate(axis=1)([first,second])\n",
        "l = Dense(main_shape, activation='relu')(l)\n",
        "l = Dropout(drop)(l)\n",
        "l = Dense(main_shape, activation='relu')(l)\n",
        "l = Dropout(drop)(l)\n",
        "\n",
        "main_output = Dense(1,activation = 'relu', name='main_output')(l)\n",
        "\n",
        "model = Model(inputs=[first_input,second_input], outputs=[main_output]) \n",
        "model.compile(optimizer = opt, loss='mean_squared_error')\n",
        "model.reset_states() \n",
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zrExJK60y7M3"
      },
      "source": [
        "plot_model(model,show_shapes=True, show_layer_names=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IPJD66iRHfLR"
      },
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q__d9NCkeOEg"
      },
      "source": [
        "history = model.fit([iv_lstm_train, dummys_train_lstm], y_train_lstm, epochs=epochs, batch_size=batch_size , use_multiprocessing = True, validation_data=([iv_lstm_test, dummys_test_lstm], y_test_lstm))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LO2lirh-gmJ5"
      },
      "source": [
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.title('model train vs validation loss')\n",
        "plt.ylabel('loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'validation'], loc='upper right')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fdNGfNtKH4jb"
      },
      "source": [
        "## Test & Prediction\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fi3dSw3Gv4sr"
      },
      "source": [
        "# Prediction\n",
        "y_pred_lstm = model.predict([iv_lstm_test, dummys_test_lstm])\n",
        "df_compare_lstm = pd.DataFrame({'Tatsächlicher Wert': y_test_lstm.values.flatten(), 'Vorhersage': y_pred_lstm.flatten()})\n",
        "df_compare_lstm"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oUU3cR5T6c25"
      },
      "source": [
        "## MSE"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HjrErFCtRy5W"
      },
      "source": [
        "#Berechnung MSE\n",
        "lstm_mse = metrics.mean_squared_error(y_test_lstm, y_pred_lstm)\n",
        "#Rücktransformation des MSE\n",
        "lstm_mse = lstm_mse/10000\n",
        "\n",
        "print('Das LSTM hat einen MSE von', lstm_mse)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gMBUG1KGNNeb"
      },
      "source": [
        "# Baseline Modell"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rdxg3Llzz_ex"
      },
      "source": [
        "'''\n",
        "\n",
        "What could we use as an alternative baseline model?\n",
        "\n",
        "Why do we use a basline model?\n",
        "\n",
        "'''"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xnm1lw8uu6dO"
      },
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eVPbb1bmRATU"
      },
      "source": [
        "#Test-Set Baseline\n",
        "y_test_base = y_test_reg.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fwe0tuthP0Kt"
      },
      "source": [
        "#Prediction-Set Baseline\n",
        "y_pred_base = y_test_reg.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2oHm934GOvGy"
      },
      "source": [
        "#Durchschnittswert aus Trainings-Set als Predictor\n",
        "y_pred_base.insert(1,'likability_avg',y_train_reg['likability'].mean())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mftml8VZaPCn"
      },
      "source": [
        "#Zwischenschritt für Berechnung der Abweichung der Vorhersage\n",
        "df_compare_base = y_pred_base.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aQMk13U2Oz2d"
      },
      "source": [
        "#Dataframe bereinigen\n",
        "del y_pred_base['likability']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CTRGi09_uyhs"
      },
      "source": [
        "## MSE"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VsvRGRDvPBt6"
      },
      "source": [
        "#Berechnung MSE\n",
        "base_mse = metrics.mean_squared_error(y_test_base, y_pred_base)\n",
        "#Rücktransformation des MSE\n",
        "base_mse = base_mse/10000\n",
        "\n",
        "print('Das Baseline Modell hat einen MSE von', base_mse)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bVm4pvlbV7n7"
      },
      "source": [
        "# **Comparison: Baseline vs. Regression vs. LSTM**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J73Jj8DhTwiJ"
      },
      "source": [
        "modelle = ['Baseline','Regression', 'LSTM']\n",
        "werte = [base_mse, reg_mse, lstm_mse]\n",
        "\n",
        "plt.figure(figsize=(22, 8))\n",
        "\n",
        "plt.subplot(131)\n",
        "\n",
        "plt.bar(modelle, werte)\n",
        "plt.title('Vergleich: Baseline vs. Regression vs. LSTM', fontsize=16)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oixsFltbMklk"
      },
      "source": [
        "#Prozentuale Unterschiede\n",
        "reg_mse_copy = reg_mse\n",
        "lstm_mse_copy = lstm_mse\n",
        "base_mse_copy = base_mse\n",
        "\n",
        "diff_base_reg = (1-reg_mse_copy/base_mse_copy)*100\n",
        "diff_base_lstm = (1-lstm_mse_copy/base_mse_copy)*100\n",
        "diff_reg_lstm = (1-lstm_mse_copy/reg_mse_copy)*100\n",
        "\n",
        "print('Das Baseline Modell hat einen MSE von',\"{:.7f}\".format(base_mse))\n",
        "print('Die Regression hat einen MSE von',\"{:.7f}\".format(reg_mse),'und ist damit',\"{:.2f}\".format(diff_base_reg),'Prozent besser als das Baseline-Modell.')\n",
        "print('Das LSTM hat einen MSE von',\"{:.7f}\".format(lstm_mse),'und ist damit',\"{:.2f}\".format(diff_base_lstm),'Prozent besser als das Baseline-Modell sowie',\"{:.2f}\".format(diff_reg_lstm),'Prozent besser als die Regression.')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}