Homework2_Question.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "492b2f97",
   "metadata": {},
   "source": [
    "Othman Murad"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f8c3cb4",
   "metadata": {},
   "source": [
    "# Homework 2:  Improve Baseline CNN model and compute metrics for assessing the performance of the CNN-based model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac5a0f64",
   "metadata": {},
   "source": [
    "## Notebook Outline\n",
    "\n",
    "    1. Train Basic Model (From Homework 1)\n",
    "\n",
    "    2. Saving and Loading Model\n",
    "    \n",
    "    3. Metrics Access Performance\n",
    "    \n",
    "    4. Hyper-parameter Tuning\n",
    "    \n",
    "    5. Overfitting Prevention\n",
    "    \n",
    "    6. Compare Performance of Basic and Improved Model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1d5214f",
   "metadata": {},
   "source": [
    "## 1. Train Basic Model (From Homework 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a36e42d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-04-28 19:08:38.290670: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "# Load the tensorflow, which is a framework for deep learning.\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import datasets, layers, models\n",
    "# Load numpy library as \"np\", which can handle large matrices and provides some mathematical functions.\n",
    "import numpy as np \n",
    "# Load pandas as \"pd\", which is useful when working with data tables. \n",
    "import pandas as pd \n",
    "# Load random, which provide some randomize functions.\n",
    "import random\n",
    "# Load a function pyplot as \"plt\" to plot figures.\n",
    "import matplotlib.pyplot as plt\n",
    "# Load functions to calculate precision, and recall\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "\n",
    "\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "\n",
    "\n",
    "# Setup the random seed for reproducibility\n",
    "seed = 1234\n",
    "random.seed(seed)\n",
    "np.random.seed(seed)\n",
    "tf.random.set_seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dabcf8f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(620, 600, 100, 4)\n",
      "(84, 600, 100, 4)\n"
     ]
    }
   ],
   "source": [
    "# The root directory of the pmuBAGE data\n",
    "pmuBAGE_data_dir = \"pmuBAGE/data\"\n",
    "\n",
    "# Number of the tensor for voltage and frequency\n",
    "voltage_tensor_number = 31\n",
    "frequency_tensor_number = 21\n",
    "\n",
    "# Load each tensors of voltage events and concatenate them as a big tensor.\n",
    "voltage_tensor_list = []\n",
    "for idx in range(voltage_tensor_number):\n",
    "    voltage_sub_tensor = np.load(f\"{pmuBAGE_data_dir}/voltage/voltage_{idx}.npy\")\n",
    "    voltage_tensor_list.append(voltage_sub_tensor)\n",
    "voltage_tensor = np.concatenate(voltage_tensor_list, axis=0)\n",
    "\n",
    "\n",
    "# Load each tensors of frequency events and concatenate them as a big tensor.\n",
    "frequency_tensor_list = []\n",
    "for idx in range(frequency_tensor_number):\n",
    "    frequency_sub_tensor = np.load(f\"{pmuBAGE_data_dir}/frequency/frequency_{idx}.npy\")\n",
    "    frequency_tensor_list.append(frequency_sub_tensor)\n",
    "frequency_tensor = np.concatenate(frequency_tensor_list, axis=0)\n",
    "\n",
    "# Transpose the big tensor as (event_idx, timestamp, PMU_idx, measurements)\n",
    "voltage_tensor = np.transpose(voltage_tensor, (0, 3, 2, 1))\n",
    "frequency_tensor = np.transpose(frequency_tensor, (0, 3, 2, 1))\n",
    "\n",
    "# Print the shape of the voltage event\n",
    "print(voltage_tensor.shape)\n",
    "print(frequency_tensor.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c49b797d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(620, 600, 100, 4)\n",
      "(84, 600, 100, 4)\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "    Use standardization to pre-process the pmu time series data.\n",
    "    Input  -> two original tensors: voltage_tensor, frequency_tensor\n",
    "    Output -> two standardized tensors: voltage_tensor_standardized, frequency_tensor_standardized\n",
    "    Requirement Details: \n",
    "        The tensor shape is (number_of_event, timestamps (600), pmus (100), measurements (4))\n",
    "        For each time sequence (Single pmu measurement sequence, 600 timestamps), standardize them by Z-Score\n",
    "        z-score = (x - mean) / std\n",
    "\"\"\"\n",
    "\n",
    "# Voltage tensor\n",
    "\n",
    "voltage_mean = np.mean(voltage_tensor, axis=1)\n",
    "voltage_mean = np.expand_dims(voltage_mean, axis=1)\n",
    "voltage_std = np.std(voltage_tensor, axis=1)\n",
    "voltage_std = np.expand_dims(voltage_std, axis=1)\n",
    "voltage_tensor_standardized = np.nan_to_num((voltage_tensor - voltage_mean) / voltage_std)\n",
    "\n",
    "# Frequency tensor\n",
    "\n",
    "frequency_mean = np.mean(frequency_tensor, axis=1)\n",
    "frequency_mean = np.expand_dims(frequency_mean, axis=1)\n",
    "frequency_std = np.std(frequency_tensor, axis=1)\n",
    "frequency_std = np.expand_dims(frequency_std, axis=1)\n",
    "frequency_tensor_standardized = np.nan_to_num((frequency_tensor - frequency_mean) / frequency_std)\n",
    "\n",
    "print(voltage_tensor_standardized.shape)\n",
    "print(frequency_tensor_standardized.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "071115fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1. 0.]\n",
      "[0. 1.]\n",
      "(620, 2)\n",
      "(84, 2)\n"
     ]
    }
   ],
   "source": [
    "# Number of the classes\n",
    "num_classes = 2\n",
    "\n",
    "# Number of the voltage and frequency events in the dataset\n",
    "n_voltage = voltage_tensor_standardized.shape[0]\n",
    "n_frequency = frequency_tensor_standardized.shape[0]\n",
    "\n",
    "# Define the labels\n",
    "# Voltage events' label is defined as: 0\n",
    "voltage_label = np.array([0] * n_voltage)\n",
    "# Frequency events' label is defined as: 1\n",
    "frequency_label = np.array([1] * n_frequency)\n",
    "\n",
    "\"\"\"\n",
    "    Implement the one-hot encoding on the lablel of of the voltage and frequency event labels.\n",
    "    Input  -> Original voltage and frequency labels (voltage_label, frequency_label)\n",
    "    Output -> One-hot encoded voltage and frequency labels (voltage_label_onehot, frequency_label_onthot)\n",
    "    Voltage label: \"0\" -> \"[1, 0]\"\n",
    "    Frequency label: \"1\" -> \"[0, 1]\"\n",
    "    You can use any library or tool for doing this\n",
    "\"\"\"\n",
    "\n",
    "voltage_label_onehot = tf.keras.utils.to_categorical(voltage_label, num_classes=num_classes)\n",
    "frequency_label_onthot = tf.keras.utils.to_categorical(frequency_label, num_classes=num_classes)\n",
    "\n",
    "# Should be [1, 0]\n",
    "print(voltage_label_onehot[0])\n",
    "# Should be [0, 1]\n",
    "print(frequency_label_onthot[0])\n",
    "# Should be (620, 2)\n",
    "print(voltage_label_onehot.shape)\n",
    "# Should be (84, 2)\n",
    "print(frequency_label_onthot.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1b8ba982",
   "metadata": {},
   "outputs": [],
   "source": [
    "voltage_tensor_standarded_permuted = voltage_tensor_standardized[np.random.permutation(n_voltage)]\n",
    "frequency_tensor_standarded_permuted = frequency_tensor_standardized[np.random.permutation(n_frequency)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "07b58c49",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(492, 600, 100, 4)\n",
      "(492, 2)\n",
      "(212, 600, 100, 4)\n",
      "(212, 2)\n"
     ]
    }
   ],
   "source": [
    "# Seperate the data to train and test\n",
    "train_portion = 0.7\n",
    "\n",
    "# Samples\n",
    "X_voltage = voltage_tensor_standarded_permuted\n",
    "X_frequency = frequency_tensor_standarded_permuted\n",
    "# Labels\n",
    "y_voltage = voltage_label_onehot\n",
    "y_frequency = frequency_label_onthot\n",
    "\n",
    "\"\"\"\n",
    "    Seperate the samples and labels to train and test datasets.\n",
    "    70% of the voltage and frequency samples and labels are combined as training dataset\n",
    "    30% remainings are combined as testing dataset\n",
    "    Input  -> X_voltage, X_frequency, y_voltage, y_frequency\n",
    "    Output -> X_train, y_train, X_test, y_test\n",
    "        X_train contains 70% of the X_voltage and X_frequency\n",
    "        y_train contains 70% of the y_voltage and y_frequency\n",
    "        X_test contains 30% of the X_voltage and X_frequency\n",
    "        y_test contains 30% of the y_voltage and y_frequency\n",
    "\"\"\"\n",
    "\n",
    "# X_train\n",
    "X_train_voltage = X_voltage[:int(n_voltage * train_portion)] \n",
    "X_train_frequency = X_frequency[:int(n_frequency * train_portion)]\n",
    "X_train = np.concatenate([X_train_voltage, X_train_frequency], axis=0)\n",
    "\n",
    "# y_train\n",
    "y_train_voltage = y_voltage[:int(n_voltage * train_portion)] \n",
    "y_train_frequency = y_frequency[:int(n_frequency * train_portion)]\n",
    "y_train = np.concatenate([y_train_voltage, y_train_frequency], axis=0)\n",
    "\n",
    "# X_test\n",
    "X_test_voltage = X_voltage[int(n_voltage * train_portion):] \n",
    "X_test_frequency = X_frequency[int(n_frequency * train_portion):]\n",
    "X_test = np.concatenate([X_test_voltage, X_test_frequency], axis=0)\n",
    "\n",
    "# y_test\n",
    "y_test_voltage = y_voltage[int(n_voltage * train_portion):] \n",
    "y_test_frequency = y_frequency[int(n_frequency * train_portion):]\n",
    "y_test = np.concatenate([y_test_voltage, y_test_frequency], axis=0)\n",
    "\n",
    "# Should be (492, 600, 100, 4)\n",
    "print(X_train.shape)\n",
    "# Should be (492, 2)\n",
    "print(y_train.shape)\n",
    "# Should be (212, 600, 100, 4)\n",
    "print(X_test.shape)\n",
    "# Should be (212, 2)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "494860b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/othmanmurad/Downloads/MLforPMU/MLforPMU/lib/python3.12/site-packages/keras/src/layers/convolutional/base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n",
      "  super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "\u001b[1m31/31\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m24s\u001b[0m 693ms/step - categorical_accuracy: 0.8292 - loss: 3.8810\n",
      "Epoch 2/10\n",
      "\u001b[1m31/31\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m40s\u001b[0m 656ms/step - categorical_accuracy: 0.8847 - loss: 0.3790\n",
      "Epoch 3/10\n",
      "\u001b[1m31/31\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m18s\u001b[0m 571ms/step - categorical_accuracy: 0.8847 - loss: 0.3640\n",
      "Epoch 4/10\n",
      "\u001b[1m31/31\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m19s\u001b[0m 613ms/step - categorical_accuracy: 0.8847 - loss: 0.3621\n",
      "Epoch 5/10\n",
      "\u001b[1m31/31\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m20s\u001b[0m 601ms/step - categorical_accuracy: 0.8847 - loss: 0.3622\n",
      "Epoch 6/10\n",
      "\u001b[1m31/31\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m19s\u001b[0m 600ms/step - categorical_accuracy: 0.8847 - loss: 0.3619\n",
      "Epoch 7/10\n",
      "\u001b[1m31/31\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m19s\u001b[0m 605ms/step - categorical_accuracy: 0.8847 - loss: 0.3617\n",
      "Epoch 8/10\n",
      "\u001b[1m31/31\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m20s\u001b[0m 652ms/step - categorical_accuracy: 0.8847 - loss: 0.3614\n",
      "Epoch 9/10\n",
      "\u001b[1m31/31\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m21s\u001b[0m 687ms/step - categorical_accuracy: 0.8847 - loss: 0.3613\n",
      "Epoch 10/10\n",
      "\u001b[1m31/31\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m19s\u001b[0m 605ms/step - categorical_accuracy: 0.8847 - loss: 0.3613\n",
      "\u001b[1m7/7\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 401ms/step - categorical_accuracy: 0.9654 - loss: 0.2054\n",
      "The accuracy of the neural network on the test dataset is: 0.8773584961891174.\n"
     ]
    }
   ],
   "source": [
    "def build_model():\n",
    "    \"\"\"\n",
    "        Add more laybers in the model, at least three convolusional layers.\n",
    "        Then add the Flatten and Dense layers to make the output same with the number of classes.\n",
    "    \"\"\"\n",
    "\n",
    "    model = models.Sequential()\n",
    "    model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(600, 100, 4)))\n",
    "    model.add(layers.MaxPooling2D((2, 2)))\n",
    "    model.add(layers.Conv2D(32, (3, 3), activation='relu'))\n",
    "    model.add(layers.MaxPooling2D((2, 2)))\n",
    "    model.add(layers.Conv2D(32, (3, 3), activation='relu'))\n",
    "    model.add(layers.MaxPooling2D((2, 2)))\n",
    "    model.add(layers.Flatten())\n",
    "    model.add(layers.Dense(32, activation='relu'))\n",
    "    model.add(layers.Dense(num_classes, activation='softmax'))\n",
    "    \n",
    "    return model\n",
    "\n",
    "model = build_model()\n",
    "\n",
    "# Define the Loss function\n",
    "loss_func = tf.keras.losses.CategoricalCrossentropy()\n",
    "\n",
    "# Define the optimizer and learning rate\n",
    "lr = 0.01\n",
    "optimizer = tf.keras.optimizers.Adam(lr)\n",
    "\n",
    "# Compile the neural network model\n",
    "model.compile(optimizer=optimizer, loss=loss_func, metrics=['categorical_accuracy'])\n",
    "\n",
    "# Train the neural network\n",
    "history = model.fit(X_train, y_train, epochs=10, batch_size=16)\n",
    "\n",
    "# Evaluate the neural network\n",
    "loss, accuracy = model.evaluate(X_test, y_test)\n",
    "print(f\"The accuracy of the neural network on the test dataset is: {accuracy}.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ae7d767",
   "metadata": {},
   "source": [
    "## 2. Saving and Loading Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "32189909",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n"
     ]
    }
   ],
   "source": [
    "##-----------------------------------------------------------------------##\n",
    "##---------------------Students start filling below----------------------##\n",
    "##-----------------------------------------------------------------------##\n",
    "\n",
    "\"\"\"\n",
    "    Save trained model to file for future application or further fine-tuning train. \n",
    "\"\"\"\n",
    "\n",
    "# Write the code to save the trained model to file.\n",
    "model.save('homework2.h5')\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "    Load the model from the file and compile it.\n",
    "\"\"\"\n",
    "\n",
    "# Write the code to load the model from file and compile it.\n",
    "model = tf.keras.models.load_model('homework2.h5')\n",
    "\n",
    "\n",
    "\n",
    "##-----------------------------------------------------------------------##\n",
    "##------------------------------End filling------------------------------##\n",
    "##-----------------------------------------------------------------------##"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b5114e9",
   "metadata": {},
   "source": [
    "## 3. Metrics Assess Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e59196fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m7/7\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 362ms/step\n"
     ]
    }
   ],
   "source": [
    "# Get the test samples and labels, and get the model's prediction on test data.\n",
    "X_test = X_test\n",
    "y_test = y_test\n",
    "y_pred = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "77503f61",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m7/7\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 327ms/step\n",
      "The accuracy is: 0.8773584905660378.\n",
      "The precision is: 0.4386792452830189.\n",
      "The recall is: 0.5.\n",
      "The f1 score is: 0.46733668341708545.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/othmanmurad/Downloads/MLforPMU/MLforPMU/lib/python3.12/site-packages/sklearn/metrics/_classification.py:1509: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n"
     ]
    }
   ],
   "source": [
    "##-----------------------------------------------------------------------##\n",
    "##---------------------Students start filling below----------------------##\n",
    "##-----------------------------------------------------------------------##\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "\"\"\"\n",
    "    Homework 1, only calculate the accuracy of the whole dataset.\n",
    "    In this task, you required to calculate the accuracy, precision, recall, and F1-score.\n",
    "\"\"\"\n",
    "\n",
    "# Accuracy\n",
    "# Precision\n",
    "# Recall\n",
    "# F1-Score\n",
    "\n",
    "y_pred = model.predict(X_test)\n",
    "accuracy = accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_pred, axis=1))\n",
    "precision = precision_score(np.argmax(y_test, axis=1), np.argmax(y_pred, axis=1), average='macro')\n",
    "recall = recall_score(np.argmax(y_test, axis=1), np.argmax(y_pred, axis=1), average='macro')\n",
    "f1 = f1_score(np.argmax(y_test, axis=1), np.argmax(y_pred, axis=1), average='macro')\n",
    "\n",
    "\n",
    "print(f\"The accuracy is: {accuracy}.\")\n",
    "print(f\"The precision is: {precision}.\")\n",
    "print(f\"The recall is: {recall}.\")\n",
    "print(f\"The f1 score is: {f1}.\")\n",
    "\n",
    "##-----------------------------------------------------------------------##\n",
    "##------------------------------End filling------------------------------##\n",
    "##-----------------------------------------------------------------------##"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da40ad63",
   "metadata": {},
   "source": [
    "## 4. Hyper-parameter Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "aaecb8b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(395, 600, 100, 4)\n",
      "(395, 2)\n",
      "(97, 600, 100, 4)\n",
      "(97, 2)\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "    Seperate the Training Dataset to Training (80%) and Validation (20%).\n",
    "    Perform the hyper-parameter tuning to find the best parameter combination.\n",
    "    \n",
    "\"\"\"\n",
    "\n",
    "val_portaton = 0.2\n",
    "\n",
    "n_voltage_train = int(n_voltage * train_portion)\n",
    "n_frequency_train = int(n_frequency * train_portion)\n",
    "\n",
    "X_train_voltage = X_voltage[:int(n_voltage * train_portion)] \n",
    "X_train_frequency = X_frequency[:int(n_frequency * train_portion)]\n",
    "X_train = np.concatenate([X_train_voltage, X_train_frequency], axis=0)\n",
    "\n",
    "# X_val_hp\n",
    "X_val_hp_voltage = X_train[:int(n_voltage_train * val_portaton)]\n",
    "X_val_hp_frequency = X_train[n_voltage_train: n_voltage_train + int(n_frequency_train * val_portaton)]\n",
    "X_val_hp = np.concatenate([X_val_hp_voltage, X_val_hp_frequency], axis=0)\n",
    "\n",
    "# y_val_hp\n",
    "y_val_hp_voltage = y_train[:int(n_voltage_train * val_portaton)]\n",
    "y_val_hp_frequency = y_train[n_voltage_train: n_voltage_train + int(n_frequency_train * val_portaton)]\n",
    "y_val_hp = np.concatenate([y_val_hp_voltage, y_val_hp_frequency], axis=0)\n",
    "\n",
    "\n",
    "# X_train_hp\n",
    "X_train_hp_voltage = X_train[int(n_voltage_train * val_portaton): n_voltage_train]\n",
    "X_train_hp_frequency = X_train[n_voltage_train + int(n_frequency_train * val_portaton):]\n",
    "X_train_hp = np.concatenate([X_train_hp_voltage, X_train_hp_frequency], axis=0)\n",
    "\n",
    "# y_train_hp\n",
    "y_train_hp_voltage = y_train[int(n_voltage_train * val_portaton): n_voltage_train]\n",
    "y_train_hp_frequency = y_train[n_voltage_train + int(n_frequency_train * val_portaton):]\n",
    "y_train_hp = np.concatenate([y_train_hp_voltage, y_train_hp_frequency], axis=0)\n",
    "\n",
    "\n",
    "# Should be (492, 600, 100, 4)\n",
    "print(X_train_hp.shape)\n",
    "# Should be (492, 2)\n",
    "print(y_train_hp.shape)\n",
    "# Should be (212, 600, 100, 4)\n",
    "print(X_val_hp.shape)\n",
    "# Should be (212, 2)\n",
    "print(y_val_hp.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "371668f5-d6bc-4db8-9034-ab524c9b6e2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup the random seed for reproducibility\n",
    "seed = 1234\n",
    "random.seed(seed)\n",
    "np.random.seed(seed)\n",
    "tf.random.set_seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2a508dfd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current hyper-parameters: learning_rate: 0.001, batch_size: 16, training_epoch: 10.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/othmanmurad/Downloads/MLforPMU/MLforPMU/lib/python3.12/site-packages/keras/src/layers/convolutional/base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n",
      "  super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m18s\u001b[0m 629ms/step - categorical_accuracy: 0.8315 - loss: 0.7406\n",
      "Epoch 2/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m16s\u001b[0m 621ms/step - categorical_accuracy: 0.8928 - loss: 0.2048\n",
      "Epoch 3/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m16s\u001b[0m 657ms/step - categorical_accuracy: 0.9207 - loss: 0.1808\n",
      "Epoch 4/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m22s\u001b[0m 864ms/step - categorical_accuracy: 0.9394 - loss: 0.1585\n",
      "Epoch 5/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m20s\u001b[0m 772ms/step - categorical_accuracy: 0.9503 - loss: 0.1386\n",
      "Epoch 6/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m18s\u001b[0m 715ms/step - categorical_accuracy: 0.9534 - loss: 0.1344\n",
      "Epoch 7/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m19s\u001b[0m 780ms/step - categorical_accuracy: 0.9550 - loss: 0.1190\n",
      "Epoch 8/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 567ms/step - categorical_accuracy: 0.9683 - loss: 0.0932\n",
      "Epoch 9/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m16s\u001b[0m 633ms/step - categorical_accuracy: 0.9807 - loss: 0.0915\n",
      "Epoch 10/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m14s\u001b[0m 570ms/step - categorical_accuracy: 0.9748 - loss: 0.0942\n",
      "\u001b[1m4/4\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 219ms/step - categorical_accuracy: 0.9814 - loss: 0.0897\n",
      "Current validation accuracy is: 0.969072163105011.\n",
      "\n",
      "Current hyper-parameters: learning_rate: 0.001, batch_size: 16, training_epoch: 20.\n",
      "Epoch 1/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m16s\u001b[0m 576ms/step - categorical_accuracy: 0.7573 - loss: 0.8488\n",
      "Epoch 2/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m16s\u001b[0m 620ms/step - categorical_accuracy: 0.9423 - loss: 0.1974\n",
      "Epoch 3/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m21s\u001b[0m 636ms/step - categorical_accuracy: 0.9502 - loss: 0.1620\n",
      "Epoch 4/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 610ms/step - categorical_accuracy: 0.9098 - loss: 0.1966\n",
      "Epoch 5/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m14s\u001b[0m 564ms/step - categorical_accuracy: 0.9463 - loss: 0.1421\n",
      "Epoch 6/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 613ms/step - categorical_accuracy: 0.9743 - loss: 0.0878\n",
      "Epoch 7/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m18s\u001b[0m 726ms/step - categorical_accuracy: 0.9653 - loss: 0.0756\n",
      "Epoch 8/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m16s\u001b[0m 623ms/step - categorical_accuracy: 0.9920 - loss: 0.0389\n",
      "Epoch 9/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 596ms/step - categorical_accuracy: 0.9920 - loss: 0.0333\n",
      "Epoch 10/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 600ms/step - categorical_accuracy: 0.9920 - loss: 0.0302\n",
      "Epoch 11/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m20s\u001b[0m 589ms/step - categorical_accuracy: 0.9920 - loss: 0.0257\n",
      "Epoch 12/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 599ms/step - categorical_accuracy: 0.9920 - loss: 0.0265\n",
      "Epoch 13/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 597ms/step - categorical_accuracy: 0.9884 - loss: 0.0313\n",
      "Epoch 14/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 589ms/step - categorical_accuracy: 0.9920 - loss: 0.0136\n",
      "Epoch 15/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 595ms/step - categorical_accuracy: 0.9920 - loss: 0.0112\n",
      "Epoch 16/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 606ms/step - categorical_accuracy: 0.9957 - loss: 0.0042\n",
      "Epoch 17/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 606ms/step - categorical_accuracy: 0.9957 - loss: 0.0037\n",
      "Epoch 18/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 601ms/step - categorical_accuracy: 0.9957 - loss: 0.0033\n",
      "Epoch 19/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 611ms/step - categorical_accuracy: 0.9957 - loss: 0.0032\n",
      "Epoch 20/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 604ms/step - categorical_accuracy: 0.9957 - loss: 0.0032\n",
      "\u001b[1m4/4\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 251ms/step - categorical_accuracy: 0.9773 - loss: 0.1049\n",
      "Current validation accuracy is: 0.9587628841400146.\n",
      "\n",
      "Current hyper-parameters: learning_rate: 0.001, batch_size: 32, training_epoch: 10.\n",
      "Epoch 1/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m18s\u001b[0m 1s/step - categorical_accuracy: 0.8470 - loss: 1.3252\n",
      "Epoch 2/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m16s\u001b[0m 1s/step - categorical_accuracy: 0.9325 - loss: 0.2052\n",
      "Epoch 3/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m16s\u001b[0m 1s/step - categorical_accuracy: 0.9576 - loss: 0.1606\n",
      "Epoch 4/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m18s\u001b[0m 1s/step - categorical_accuracy: 0.9573 - loss: 0.1501\n",
      "Epoch 5/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9539 - loss: 0.1369\n",
      "Epoch 6/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m19s\u001b[0m 1s/step - categorical_accuracy: 0.9677 - loss: 0.1152\n",
      "Epoch 7/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m17s\u001b[0m 1s/step - categorical_accuracy: 0.9639 - loss: 0.1033\n",
      "Epoch 8/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m18s\u001b[0m 1s/step - categorical_accuracy: 0.9811 - loss: 0.0818\n",
      "Epoch 9/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m17s\u001b[0m 1s/step - categorical_accuracy: 0.9809 - loss: 0.0597\n",
      "Epoch 10/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9936 - loss: 0.0362\n",
      "\u001b[1m4/4\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 254ms/step - categorical_accuracy: 0.9773 - loss: 0.0814\n",
      "Current validation accuracy is: 0.9587628841400146.\n",
      "\n",
      "Current hyper-parameters: learning_rate: 0.001, batch_size: 32, training_epoch: 20.\n",
      "Epoch 1/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m17s\u001b[0m 1s/step - categorical_accuracy: 0.8504 - loss: 0.7260\n",
      "Epoch 2/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9309 - loss: 0.1995\n",
      "Epoch 3/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9479 - loss: 0.1775\n",
      "Epoch 4/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9544 - loss: 0.1730\n",
      "Epoch 5/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9623 - loss: 0.1445\n",
      "Epoch 6/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9769 - loss: 0.1058\n",
      "Epoch 7/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9811 - loss: 0.0856\n",
      "Epoch 8/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9834 - loss: 0.0611\n",
      "Epoch 9/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9871 - loss: 0.0374\n",
      "Epoch 10/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 1.0000 - loss: 0.0190\n",
      "Epoch 11/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9978 - loss: 0.0120\n",
      "Epoch 12/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9985 - loss: 0.0108\n",
      "Epoch 13/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9756 - loss: 0.0570\n",
      "Epoch 14/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9687 - loss: 0.0761\n",
      "Epoch 15/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9844 - loss: 0.0606\n",
      "Epoch 16/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9780 - loss: 0.0588\n",
      "Epoch 17/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9961 - loss: 0.0249\n",
      "Epoch 18/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9928 - loss: 0.0124\n",
      "Epoch 19/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 1.0000 - loss: 0.0032\n",
      "Epoch 20/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 1.0000 - loss: 0.0010\n",
      "\u001b[1m4/4\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 256ms/step - categorical_accuracy: 0.9897 - loss: 0.1017  \n",
      "Current validation accuracy is: 0.9793814420700073.\n",
      "\n",
      "Current hyper-parameters: learning_rate: 0.01, batch_size: 16, training_epoch: 10.\n",
      "Epoch 1/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m17s\u001b[0m 586ms/step - categorical_accuracy: 0.6892 - loss: 0.9182\n",
      "Epoch 2/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 590ms/step - categorical_accuracy: 0.8871 - loss: 0.3362\n",
      "Epoch 3/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 597ms/step - categorical_accuracy: 0.8861 - loss: 0.2351\n",
      "Epoch 4/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 598ms/step - categorical_accuracy: 0.9175 - loss: 0.2017\n",
      "Epoch 5/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 583ms/step - categorical_accuracy: 0.9440 - loss: 0.1572\n",
      "Epoch 6/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 596ms/step - categorical_accuracy: 0.9704 - loss: 0.1276\n",
      "Epoch 7/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 585ms/step - categorical_accuracy: 0.9617 - loss: 0.1222\n",
      "Epoch 8/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 583ms/step - categorical_accuracy: 0.9797 - loss: 0.1057\n",
      "Epoch 9/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 596ms/step - categorical_accuracy: 0.9726 - loss: 0.1360\n",
      "Epoch 10/10\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 585ms/step - categorical_accuracy: 0.9477 - loss: 0.1682\n",
      "\u001b[1m4/4\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 255ms/step - categorical_accuracy: 0.9503 - loss: 0.1486\n",
      "Current validation accuracy is: 0.9278350472450256.\n",
      "\n",
      "Current hyper-parameters: learning_rate: 0.01, batch_size: 16, training_epoch: 20.\n",
      "Epoch 1/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m16s\u001b[0m 582ms/step - categorical_accuracy: 0.8686 - loss: 8.1021\n",
      "Epoch 2/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 598ms/step - categorical_accuracy: 0.8871 - loss: 0.3520\n",
      "Epoch 3/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 581ms/step - categorical_accuracy: 0.8871 - loss: 0.3517\n",
      "Epoch 4/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 594ms/step - categorical_accuracy: 0.8871 - loss: 0.3519\n",
      "Epoch 5/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 619ms/step - categorical_accuracy: 0.8871 - loss: 0.3521\n",
      "Epoch 6/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 583ms/step - categorical_accuracy: 0.8871 - loss: 0.3524\n",
      "Epoch 7/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 591ms/step - categorical_accuracy: 0.8871 - loss: 0.3519\n",
      "Epoch 8/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m20s\u001b[0m 586ms/step - categorical_accuracy: 0.8871 - loss: 0.3518\n",
      "Epoch 9/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m14s\u001b[0m 577ms/step - categorical_accuracy: 0.8871 - loss: 0.3519\n",
      "Epoch 10/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 587ms/step - categorical_accuracy: 0.8871 - loss: 0.3519\n",
      "Epoch 11/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 581ms/step - categorical_accuracy: 0.8871 - loss: 0.3520\n",
      "Epoch 12/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 592ms/step - categorical_accuracy: 0.8871 - loss: 0.3520\n",
      "Epoch 13/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 585ms/step - categorical_accuracy: 0.8871 - loss: 0.3522\n",
      "Epoch 14/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 588ms/step - categorical_accuracy: 0.8871 - loss: 0.3525\n",
      "Epoch 15/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 579ms/step - categorical_accuracy: 0.8871 - loss: 0.3529\n",
      "Epoch 16/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 594ms/step - categorical_accuracy: 0.8871 - loss: 0.3535\n",
      "Epoch 17/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 592ms/step - categorical_accuracy: 0.8871 - loss: 0.3545\n",
      "Epoch 18/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 594ms/step - categorical_accuracy: 0.8871 - loss: 0.3561\n",
      "Epoch 19/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m17s\u001b[0m 664ms/step - categorical_accuracy: 0.8871 - loss: 0.3583\n",
      "Epoch 20/20\n",
      "\u001b[1m25/25\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m19s\u001b[0m 779ms/step - categorical_accuracy: 0.8871 - loss: 0.3602\n",
      "\u001b[1m4/4\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 518ms/step - categorical_accuracy: 0.9338 - loss: 0.2781\n",
      "Current validation accuracy is: 0.8865979313850403.\n",
      "\n",
      "Current hyper-parameters: learning_rate: 0.01, batch_size: 32, training_epoch: 10.\n",
      "Epoch 1/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m23s\u001b[0m 2s/step - categorical_accuracy: 0.7038 - loss: 8.5057\n",
      "Epoch 2/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m20s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.2611\n",
      "Epoch 3/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m18s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.2584\n",
      "Epoch 4/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.1952\n",
      "Epoch 5/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.1657\n",
      "Epoch 6/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9074 - loss: 0.1447\n",
      "Epoch 7/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m16s\u001b[0m 1s/step - categorical_accuracy: 0.9665 - loss: 0.1283\n",
      "Epoch 8/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9741 - loss: 0.1123\n",
      "Epoch 9/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9741 - loss: 0.0986\n",
      "Epoch 10/10\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9753 - loss: 0.0862\n",
      "\u001b[1m4/4\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 261ms/step - categorical_accuracy: 0.8880 - loss: 0.2141\n",
      "Current validation accuracy is: 0.876288652420044.\n",
      "\n",
      "Current hyper-parameters: learning_rate: 0.01, batch_size: 32, training_epoch: 20.\n",
      "Epoch 1/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m17s\u001b[0m 1s/step - categorical_accuracy: 0.8163 - loss: 20.1459\n",
      "Epoch 2/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.6065\n",
      "Epoch 3/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.4963\n",
      "Epoch 4/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.4276\n",
      "Epoch 5/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.3908\n",
      "Epoch 6/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.3726\n",
      "Epoch 7/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.3637\n",
      "Epoch 8/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.3594\n",
      "Epoch 9/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.3573\n",
      "Epoch 10/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.3563\n",
      "Epoch 11/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.3557\n",
      "Epoch 12/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m21s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.3555\n",
      "Epoch 13/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.3553\n",
      "Epoch 14/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.3552\n",
      "Epoch 15/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.3552\n",
      "Epoch 16/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.3552\n",
      "Epoch 17/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.3552\n",
      "Epoch 18/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.3552\n",
      "Epoch 19/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.3552\n",
      "Epoch 20/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8860 - loss: 0.3552\n",
      "\u001b[1m4/4\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 256ms/step - categorical_accuracy: 0.9338 - loss: 0.2609\n",
      "Current validation accuracy is: 0.8865979313850403.\n",
      "\n",
      "best validation accuracy is: 0.9793814420700073\n",
      "best hyper-parameter setting is: {'learning_rate': 0.001, 'batch_size': 32, 'training_epoch': 20}\n"
     ]
    }
   ],
   "source": [
    "# Hyper-parameters\n",
    "learning_rates = [0.001, 0.01]\n",
    "batch_sizes = [16, 32]\n",
    "training_epochs = [10, 20]\n",
    "\n",
    "##-----------------------------------------------------------------------##\n",
    "##---------------------Students start filling below----------------------##\n",
    "##-----------------------------------------------------------------------##\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "    Try different combination of the hyper-parameters.\n",
    "    Train on training dataset, test on validation dataset.\n",
    "    Choose the best hyper-parameter combination to train the final improved model.\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "best_hyperparameter = {\"learning_rate\": 0, \"batch_size\": 0, \"training_epoch\": 0}\n",
    "best_accuracy_val = 0.0\n",
    "\n",
    "\n",
    "for learning_rate in learning_rates:\n",
    "    for batch_size in batch_sizes:\n",
    "        for training_epoch in training_epochs:\n",
    "            print(f\"Current hyper-parameters: learning_rate: {learning_rate}, batch_size: {batch_size}, training_epoch: {training_epoch}.\")\n",
    "            \n",
    "            \"\"\"\n",
    "                Train the model under hyper-parameter setting, and evaluate over validation dataset.\n",
    "            \"\"\"\n",
    "            \n",
    "            \"\"\" Filling code below \"\"\"\n",
    "            \n",
    "            # build model\n",
    "            model_tuning = build_model()\n",
    "\n",
    "            # Train the model with the above hyper-parameters\n",
    "\n",
    "            # Define the Loss function\n",
    "            loss_func = tf.keras.losses.CategoricalCrossentropy()\n",
    "\n",
    "            # Define the optimizer and learning rate\n",
    "            \n",
    "            optimizer = tf.keras.optimizers.Adam(learning_rate)\n",
    "\n",
    "            # Compile the neural network model\n",
    "            model_tuning.compile(optimizer=optimizer, loss=loss_func, metrics=['categorical_accuracy'])\n",
    "\n",
    "            # Train the neural network\n",
    "            history = model_tuning.fit(X_train_hp, y_train_hp, epochs= training_epoch, batch_size= batch_size)\n",
    "\n",
    "            \n",
    "            \"\"\" End Filling \"\"\"\n",
    "    \n",
    "            # Evaluate the hyper-parameter tuning neural network\n",
    "            loss_val, accuracy_val = model_tuning.evaluate(X_val_hp, y_val_hp)\n",
    "            print(f\"Current validation accuracy is: {accuracy_val}.\\n\")\n",
    "            \n",
    "            if accuracy_val > best_accuracy_val:\n",
    "                best_accuracy_val = accuracy_val\n",
    "                best_hyperparameter[\"learning_rate\"] = learning_rate\n",
    "                best_hyperparameter[\"batch_size\"] = batch_size\n",
    "                best_hyperparameter[\"training_epoch\"] = training_epoch\n",
    "            tf.keras.backend.clear_session()\n",
    "            \n",
    "            \n",
    "print(f\"best validation accuracy is: {best_accuracy_val}\")\n",
    "print(f\"best hyper-parameter setting is: {best_hyperparameter}\")\n",
    "\n",
    "##-----------------------------------------------------------------------##\n",
    "##------------------------------End filling------------------------------##\n",
    "##-----------------------------------------------------------------------##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "15f4a6d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m22s\u001b[0m 1s/step - categorical_accuracy: 0.8834 - loss: 0.9956\n",
      "Epoch 2/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m19s\u001b[0m 1s/step - categorical_accuracy: 0.9338 - loss: 0.1858\n",
      "Epoch 3/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m23s\u001b[0m 1s/step - categorical_accuracy: 0.9568 - loss: 0.1298\n",
      "Epoch 4/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m22s\u001b[0m 1s/step - categorical_accuracy: 0.9576 - loss: 0.1001\n",
      "Epoch 5/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m44s\u001b[0m 2s/step - categorical_accuracy: 0.9721 - loss: 0.0732\n",
      "Epoch 6/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m26s\u001b[0m 2s/step - categorical_accuracy: 0.9870 - loss: 0.0533\n",
      "Epoch 7/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m18s\u001b[0m 1s/step - categorical_accuracy: 0.9965 - loss: 0.0325\n",
      "Epoch 8/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m18s\u001b[0m 1s/step - categorical_accuracy: 0.9965 - loss: 0.0187\n",
      "Epoch 9/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m20s\u001b[0m 1s/step - categorical_accuracy: 0.9990 - loss: 0.0137\n",
      "Epoch 10/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m20s\u001b[0m 1s/step - categorical_accuracy: 1.0000 - loss: 0.0100\n",
      "Epoch 11/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m19s\u001b[0m 1s/step - categorical_accuracy: 0.9990 - loss: 0.0101\n",
      "Epoch 12/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m19s\u001b[0m 1s/step - categorical_accuracy: 0.9912 - loss: 0.0369\n",
      "Epoch 13/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m19s\u001b[0m 1s/step - categorical_accuracy: 0.9855 - loss: 0.0339\n",
      "Epoch 14/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m19s\u001b[0m 1s/step - categorical_accuracy: 0.9982 - loss: 0.0123\n",
      "Epoch 15/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m19s\u001b[0m 1s/step - categorical_accuracy: 0.9986 - loss: 0.0111\n",
      "Epoch 16/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m19s\u001b[0m 1s/step - categorical_accuracy: 0.9599 - loss: 0.0727\n",
      "Epoch 17/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m18s\u001b[0m 1s/step - categorical_accuracy: 0.9946 - loss: 0.0328\n",
      "Epoch 18/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m19s\u001b[0m 1s/step - categorical_accuracy: 0.9654 - loss: 0.0696\n",
      "Epoch 19/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m19s\u001b[0m 1s/step - categorical_accuracy: 0.9857 - loss: 0.0517\n",
      "Epoch 20/20\n",
      "\u001b[1m16/16\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m18s\u001b[0m 1s/step - categorical_accuracy: 0.9785 - loss: 0.0501\n",
      "\u001b[1m7/7\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 351ms/step - categorical_accuracy: 0.8991 - loss: 0.3459\n",
      "Current validation accuracy is: 0.8962264060974121.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#####################################\n",
    "######     Improved Model      ######\n",
    "#####################################\n",
    "\n",
    "learning_rate = 0.001\n",
    "batch_size = 32\n",
    "training_epoch = 20\n",
    "\n",
    "\n",
    "model_improved = build_model()\n",
    "\n",
    "# Train the model with the above hyper-parameters\n",
    "\n",
    "# Define the Loss function\n",
    "loss_func = tf.keras.losses.CategoricalCrossentropy()\n",
    "\n",
    "# Define the optimizer and learning rate\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate)\n",
    "\n",
    "# Compile the neural network model\n",
    "model_improved.compile(optimizer=optimizer, loss=loss_func, metrics=['categorical_accuracy'])\n",
    "\n",
    "# Train the neural network\n",
    "history = model_improved.fit(X_train, y_train, epochs=training_epoch, batch_size= batch_size)\n",
    "\n",
    "loss, accuracy = model_improved.evaluate(X_test, y_test)\n",
    "\n",
    "print(f\"Current validation accuracy is: {accuracy}.\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea655f64",
   "metadata": {},
   "source": [
    "## 5. Overfitting Prevention\n",
    "    Early Stopping to Prevent the overfitting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "53437133",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m25s\u001b[0m 2s/step - categorical_accuracy: 0.8423 - loss: 1.8344 - val_categorical_accuracy: 0.8866 - val_loss: 0.2593\n",
      "Epoch 2/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.8947 - loss: 0.2209 - val_categorical_accuracy: 0.9485 - val_loss: 0.1611\n",
      "Epoch 3/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9400 - loss: 0.1921 - val_categorical_accuracy: 0.9691 - val_loss: 0.1293\n",
      "Epoch 4/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m14s\u001b[0m 1s/step - categorical_accuracy: 0.9584 - loss: 0.1574 - val_categorical_accuracy: 0.9691 - val_loss: 0.1252\n",
      "Epoch 5/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m19s\u001b[0m 2s/step - categorical_accuracy: 0.9656 - loss: 0.1354 - val_categorical_accuracy: 0.9588 - val_loss: 0.1290\n",
      "Epoch 6/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m18s\u001b[0m 1s/step - categorical_accuracy: 0.9681 - loss: 0.1141 - val_categorical_accuracy: 0.9588 - val_loss: 0.1317\n",
      "Epoch 7/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m19s\u001b[0m 1s/step - categorical_accuracy: 0.9732 - loss: 0.0944 - val_categorical_accuracy: 0.9588 - val_loss: 0.1375\n",
      "Epoch 8/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m17s\u001b[0m 1s/step - categorical_accuracy: 0.9809 - loss: 0.0747 - val_categorical_accuracy: 0.9588 - val_loss: 0.1459\n",
      "Epoch 9/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m19s\u001b[0m 1s/step - categorical_accuracy: 0.9871 - loss: 0.0537 - val_categorical_accuracy: 0.9485 - val_loss: 0.1677\n",
      "Epoch 10/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m18s\u001b[0m 1s/step - categorical_accuracy: 0.9871 - loss: 0.0395 - val_categorical_accuracy: 0.9485 - val_loss: 0.2007\n",
      "Epoch 11/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m16s\u001b[0m 1s/step - categorical_accuracy: 0.9871 - loss: 0.0282 - val_categorical_accuracy: 0.9485 - val_loss: 0.2165\n",
      "Epoch 12/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9910 - loss: 0.0197 - val_categorical_accuracy: 0.9485 - val_loss: 0.2040\n",
      "Epoch 13/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9910 - loss: 0.0159 - val_categorical_accuracy: 0.9485 - val_loss: 0.2274\n",
      "Epoch 14/20\n",
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 1s/step - categorical_accuracy: 0.9910 - loss: 0.0147 - val_categorical_accuracy: 0.9588 - val_loss: 0.1616\n"
     ]
    }
   ],
   "source": [
    "# Set the best hyper-parameter from previous tuning. (You may change them based on previous results)\n",
    "learning_rate = 0.001\n",
    "batch_size = 32\n",
    "training_epoch = 20\n",
    "\n",
    "##-----------------------------------------------------------------------##\n",
    "##---------------------Students start filling below----------------------##\n",
    "##-----------------------------------------------------------------------##\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "    Use the early stopping to prevent the overfitting.\n",
    "    Useful resource: https://keras.io/api/callbacks/early_stopping/\n",
    "\"\"\"\n",
    "\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "\n",
    "\n",
    "# Build model\n",
    "model_improved = build_model()\n",
    "# Define the loss function\n",
    "loss_func = tf.keras.losses.CategoricalCrossentropy()\n",
    "# Define the optimizer\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate)\n",
    "# Compile the model\n",
    "model_improved.compile(optimizer=optimizer, loss=loss_func, metrics=['categorical_accuracy'])\n",
    "\n",
    "\"\"\" Filling code below \"\"\"\n",
    "\n",
    "# Define the early stopping callback\n",
    "early_stopping = EarlyStopping(monitor = 'val_loss',patience = 10, restore_best_weights = True)\n",
    "\n",
    "\"\"\" End Filling \"\"\"\n",
    "\n",
    "# Train the model with the tuned hyper-parameters and early-stopping.\n",
    "history = model_improved.fit(X_train_hp, y_train_hp, validation_data=(X_val_hp, y_val_hp), \n",
    "                             epochs=training_epoch, batch_size=batch_size, callbacks=[early_stopping])\n",
    "\n",
    "##-----------------------------------------------------------------------##\n",
    "##------------------------------End filling------------------------------##\n",
    "##-----------------------------------------------------------------------##"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b04b3bcf",
   "metadata": {},
   "source": [
    "## 6. Compare Performance of Basic and Improved Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9e5f2d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Precision, recall, F1-score of the basic model\n",
    "\n",
    "y_pred = model.predict(X_test)\n",
    "accuracy = accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_pred, axis=1))\n",
    "precision = precision_score(np.argmax(y_test, axis=1), np.argmax(y_pred, axis=1), average='macro')\n",
    "recall = recall_score(np.argmax(y_test, axis=1), np.argmax(y_pred, axis=1), average='macro')\n",
    "f1 = f1_score(np.argmax(y_test, axis=1), np.argmax(y_pred, axis=1), average='macro')\n",
    "\n",
    "print(\"Performance of the basic model.\")\n",
    "print(f\"The accuracy is: {accuracy}.\")\n",
    "print(f\"The precision is: {precision}.\")\n",
    "print(f\"The recall is: {recall}.\")\n",
    "print(f\"The f1 score is: {f1}.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cde3fcd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Precision, recall, F1-score of the improved model\n",
    "\n",
    "y_pred_improved = model_improved.predict(X_test)\n",
    "accuracy = accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_pred_improved, axis=1))\n",
    "precision = precision_score(np.argmax(y_test, axis=1), np.argmax(y_pred_improved, axis=1), average='macro')\n",
    "recall = recall_score(np.argmax(y_test, axis=1), np.argmax(y_pred_improved, axis=1), average='macro')\n",
    "f1 = f1_score(np.argmax(y_test, axis=1), np.argmax(y_pred_improved, axis=1), average='macro')\n",
    "\n",
    "print(\"Performance of the improved model.\")\n",
    "print(f\"The accuracy is: {accuracy}.\")\n",
    "print(f\"The precision is: {precision}.\")\n",
    "print(f\"The recall is: {recall}.\")\n",
    "print(f\"The f1 score is: {f1}.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db2af259",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a710e7a7-c75c-4a5b-ac0f-e6999d96b224",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}