Preprocessing_2D.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "JUjihIUu3whg"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import cv2\n",
        "import os"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Connect to Drive if needed"
      ],
      "metadata": {
        "id": "OfqPURXW7zJS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XCyfR4uQ4P_j",
        "outputId": "48168ca4-21ec-4cb3-b5c7-fc9f8cd88f9b"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Get **source_path** and **save_path**"
      ],
      "metadata": {
        "id": "n1qd-RMX78Mq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "source_path='/content/drive/MyDrive/spectrograms'\n",
        "save_path=\"/content/drive/MyDrive\""
      ],
      "metadata": {
        "id": "JknBW59_4QJZ"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Process each image on the **source_path** and append them to **imageid_list**"
      ],
      "metadata": {
        "id": "Ft3GJrhS4bXY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "imageid_list = list()\n",
        "for root, dirs, files in os.walk(source_path):\n",
        "  for file in files:\n",
        "    classid = file.split(\"-\")[1]\n",
        "    img = cv2.imread(os.path.join(root,file))\n",
        "    gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
        "    gray_img = cv2.resize(gray_img, dsize=(64,64), interpolation=cv2.INTER_AREA)\n",
        "    gray_img = gray_img.astype('float32') / 255.0\n",
        "    gray_img = cv2.normalize(gray_img, None, alpha=0,beta=200, norm_type=cv2.NORM_MINMAX)\n",
        "    imageid_list.append([gray_img,classid])\n",
        "\n",
        "print(f\"Total images: {len(imageid_list)}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zNzucK0g4QQJ",
        "outputId": "7087f214-1ae9-43da-8b7f-ace312b4182c"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total images: 8732\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Turn **imageid_list** into **DataFrame**"
      ],
      "metadata": {
        "id": "c9wE9qcr4oN5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.DataFrame(imageid_list, columns=['img','label'])"
      ],
      "metadata": {
        "id": "u-WgEjl-4QXB"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Save **DataFrame** to **save_path** using **pickle**"
      ],
      "metadata": {
        "id": "g7qLdwWv5bSj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pickle\n",
        "\n",
        "with open(f'{save_path}/dataset.pickle', 'wb') as output:\n",
        "    pickle.dump(df, output)"
      ],
      "metadata": {
        "id": "cOM_hQYt5qzZ"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Split **DataFrame** into **X_train, y_train, X_val, y_val, X_test** and \n",
        "**y_test** datasets"
      ],
      "metadata": {
        "id": "zwP0PTkV5xmZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split"
      ],
      "metadata": {
        "id": "z92t0kKw5GrZ"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X=df['img']\n",
        "y=df['label'].astype(np.float32)\n",
        "X = np.array([np.array(val) for val in X])\n",
        "X = X.reshape(list(X.shape) + [1])\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X,y ,\n",
        "                                   random_state=42, \n",
        "                                   test_size=0.2, \n",
        "                                   shuffle=True)\n",
        "\n",
        "X_train, X_val, y_train, y_val = train_test_split(X_train, y_train,\n",
        "                                  random_state=42,\n",
        "                                  test_size=0.25,\n",
        "                                  shuffle=True)"
      ],
      "metadata": {
        "id": "QYPXrWji5GuJ"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(X_train))\n",
        "print(len(X_val))\n",
        "print(len(X_test))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y7aN7vej5Gx7",
        "outputId": "7acb49a0-ba2a-4da4-eb05-db36d8b82a21"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "5238\n",
            "1747\n",
            "1747\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Save **X_train, y_train, X_val, y_val, X_test** and **y_test** to **save_path** using **numpy**"
      ],
      "metadata": {
        "id": "JBf04emt6KO5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "np.save(f\"{save_path}/X_train\", X_train)\n",
        "np.save(f\"{save_path}/X_test\", X_test)\n",
        "np.save(f\"{save_path}/X_val\", X_val)\n",
        "np.save(f\"{save_path}/y_train\", y_train)\n",
        "np.save(f\"{save_path}/y_test\", y_test)\n",
        "np.save(f\"{save_path}/y_val\", y_val)"
      ],
      "metadata": {
        "id": "9yPGpeQ36r9I"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ZgMI1YSIfPy1"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}