Final_Grid_data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TnbzNWxIduJx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4fa43089-6e90-44a6-a867-37e5d9930125"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "First 5 rows:\n",
            "               date        p1        p2        p3        c1        c2  \\\n",
            "0  01-01-2019 01:00  0.859578  0.887445  0.958034 -0.782604 -1.257395   \n",
            "1  01-01-2019 02:00  0.862414  0.562139  0.781760 -1.940058 -1.872742   \n",
            "2  01-01-2019 03:00  0.766689  0.839444  0.109853 -1.207456 -1.277210   \n",
            "3  01-01-2019 04:00  0.976744  0.929381  0.362718 -1.027473 -1.938944   \n",
            "4  01-01-2019 05:00  0.455450  0.656947  0.820923 -1.125531 -1.845975   \n",
            "\n",
            "         c3 stability  \n",
            "0 -1.723086  unstable  \n",
            "1 -1.255012    stable  \n",
            "2 -0.920492  unstable  \n",
            "3 -0.997374  unstable  \n",
            "4 -0.554305  unstable  \n",
            "\n",
            "Last 5 rows:\n",
            "                   date        p1        p2        p3        c1        c2  \\\n",
            "43818  31-12-2023 19:00  0.257940  0.895296  0.868929 -1.954289 -0.981347   \n",
            "43819  31-12-2023 20:00  0.848075  0.909264  0.266201 -1.428185 -0.525543   \n",
            "43820  31-12-2023 21:00  0.393902  0.441923  0.697164 -0.701491 -1.279522   \n",
            "43821  31-12-2023 22:00  0.280877  0.532758  0.368188 -1.076426 -1.231602   \n",
            "43822  31-12-2023 23:00  0.237036  0.891977  0.656815 -1.579196 -1.526573   \n",
            "\n",
            "             c3 stability  \n",
            "43818 -1.654103    stable  \n",
            "43819 -1.545585    stable  \n",
            "43820 -1.019624  unstable  \n",
            "43821 -1.164986    stable  \n",
            "43822 -0.571834  unstable  \n"
          ]
        }
      ],
      "source": [
        "# Import necessary libraries\n",
        "import pandas as pd\n",
        "\n",
        "# Load the CSV file\n",
        "file_path = '/content/final_merged_data_grid.csv'\n",
        "data = pd.read_csv(file_path)\n",
        "\n",
        "# Print the first 5 rows\n",
        "print(\"First 5 rows:\")\n",
        "print(data.head())\n",
        "\n",
        "# Print the last 5 rows\n",
        "print(\"\\nLast 5 rows:\")\n",
        "print(data.tail())\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Import necessary libraries\n",
        "import pandas as pd\n",
        "\n",
        "# Load the CSV file\n",
        "file_path = '/content/grid_stability_3months_validation_data.csv'\n",
        "data = pd.read_csv(file_path)\n",
        "\n",
        "# Print the first 5 rows\n",
        "print(\"First 5 rows:\")\n",
        "print(data.head())\n",
        "\n",
        "# Print the last 5 rows\n",
        "print(\"\\nLast 5 rows:\")\n",
        "print(data.tail())\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DkPrDsZl1rO-",
        "outputId": "91f272de-de1a-46e6-d413-da896c0c3f8b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "First 5 rows:\n",
            "               date        c1        c2        c3        p1        p2  \\\n",
            "0  01-01-2024 00:00 -1.940140 -0.798906 -1.171796  0.209084  0.125430   \n",
            "1  01-01-2024 01:00 -1.431471 -0.523455 -1.481922  0.659985  0.565942   \n",
            "2  01-01-2024 02:00 -0.791358 -1.134176 -1.023553  0.083137  0.411446   \n",
            "3  01-01-2024 03:00 -1.291775 -1.943314 -1.536861  0.805780  0.431950   \n",
            "4  01-01-2024 04:00 -0.532246 -1.734586 -0.845011  0.067430  0.766504   \n",
            "\n",
            "         p3 stability  \n",
            "0  0.056619    stable  \n",
            "1  0.252764    stable  \n",
            "2  0.376176    stable  \n",
            "3  0.222319  unstable  \n",
            "4  0.349372    stable  \n",
            "\n",
            "Last 5 rows:\n",
            "                  date        c1        c2        c3        p1        p2  \\\n",
            "2179  31-03-2024 19:00 -1.593110 -1.096450 -1.287426  0.549650  0.107777   \n",
            "2180  31-03-2024 20:00 -1.987940 -1.172603 -1.136143  0.202879  0.613149   \n",
            "2181  31-03-2024 21:00 -0.974514 -1.665259 -1.477446  0.772918  0.565030   \n",
            "2182  31-03-2024 22:00 -0.787608 -1.179217 -0.716963  0.579977  0.898658   \n",
            "2183  31-03-2024 23:00 -1.399564 -0.925921 -1.463104  0.999409  0.526355   \n",
            "\n",
            "            p3 stability  \n",
            "2179  0.327430    stable  \n",
            "2180  0.487372  unstable  \n",
            "2181  0.422756    stable  \n",
            "2182  0.214563  unstable  \n",
            "2183  0.274401  unstable  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "\n",
        "# Mount Google Drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "# Path to the folder containing the CSV files\n",
        "folder_path = '/content/drive/My Drive/Grid_data'\n",
        "\n",
        "# List to hold DataFrames from each CSV file\n",
        "dfs = []\n",
        "\n",
        "# Iterate through each file in the folder\n",
        "for filename in os.listdir(folder_path):\n",
        "    if filename.endswith(\".csv\"):\n",
        "        # Read the CSV file into a DataFrame\n",
        "        year = filename.split('_')[2].split('.')[0]\n",
        "        df = pd.read_csv(os.path.join(folder_path, filename))\n",
        "        # Add a column indicating the year\n",
        "        df['year'] = year\n",
        "        dfs.append(df)\n",
        "\n",
        "# Concatenate all DataFrames into a single DataFrame\n",
        "combined_df = pd.concat(dfs, ignore_index=True)\n",
        "\n",
        "# Sort the combined DataFrame by date\n",
        "combined_df.sort_values(by='date', inplace=True)\n",
        "\n",
        "# Display the sorted combined DataFrame\n",
        "print(combined_df)\n",
        "\n",
        "# Save the sorted combined DataFrame to a CSV file\n",
        "combined_file_path = '/content/drive/My Drive/Grid_data/combined_price_per_unit_sorted.csv'\n",
        "combined_df.to_csv(combined_file_path, index=False)\n"
      ],
      "metadata": {
        "id": "-ZUbkjl05XaH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Load the combined file with the year column\n",
        "combined_file_path = '/content/Grid_data/combined_price_per_unit_sorted.csv'\n",
        "combined_df = pd.read_csv(combined_file_path)\n",
        "\n",
        "# Display the DataFrame before deleting the 'year' column\n",
        "print(\"Before deleting 'year' column:\")\n",
        "print(combined_df)\n",
        "\n",
        "# Delete the 'year' column\n",
        "combined_df.drop(columns=['year'], inplace=True)\n",
        "\n",
        "# Display the DataFrame after deleting the 'year' column\n",
        "print(\"\\nAfter deleting 'year' column:\")\n",
        "print(combined_df)\n",
        "\n",
        "# Save the modified DataFrame to a new CSV file\n",
        "modified_combined_file_path = '/content/Grid_data/modified_combined_price_per_unit.csv'\n",
        "combined_df.to_csv(modified_combined_file_path, index=False)\n",
        "print(\"\\nModified DataFrame has been saved to:\", modified_combined_file_path)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pp1Ahw88e33G",
        "outputId": "26bae142-b3dd-43f6-d6e6-a66bcc0883ad"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Before deleting 'year' column:\n",
            "                   date        p1        p2        p3  year\n",
            "0      01-01-2019 01:00  0.859578  0.887445  0.958034  unit\n",
            "1      01-01-2019 02:00  0.862414  0.562139  0.781760  unit\n",
            "2      01-01-2019 03:00  0.766689  0.839444  0.109853  unit\n",
            "3      01-01-2019 04:00  0.976744  0.929381  0.362718  unit\n",
            "4      01-01-2019 05:00  0.455450  0.656947  0.820923  unit\n",
            "...                 ...       ...       ...       ...   ...\n",
            "43818  31-12-2023 19:00  0.257940  0.895296  0.868929  unit\n",
            "43819  31-12-2023 20:00  0.848075  0.909264  0.266201  unit\n",
            "43820  31-12-2023 21:00  0.393902  0.441923  0.697164  unit\n",
            "43821  31-12-2023 22:00  0.280877  0.532758  0.368188  unit\n",
            "43822  31-12-2023 23:00  0.237036  0.891977  0.656815  unit\n",
            "\n",
            "[43823 rows x 5 columns]\n",
            "\n",
            "After deleting 'year' column:\n",
            "                   date        p1        p2        p3\n",
            "0      01-01-2019 01:00  0.859578  0.887445  0.958034\n",
            "1      01-01-2019 02:00  0.862414  0.562139  0.781760\n",
            "2      01-01-2019 03:00  0.766689  0.839444  0.109853\n",
            "3      01-01-2019 04:00  0.976744  0.929381  0.362718\n",
            "4      01-01-2019 05:00  0.455450  0.656947  0.820923\n",
            "...                 ...       ...       ...       ...\n",
            "43818  31-12-2023 19:00  0.257940  0.895296  0.868929\n",
            "43819  31-12-2023 20:00  0.848075  0.909264  0.266201\n",
            "43820  31-12-2023 21:00  0.393902  0.441923  0.697164\n",
            "43821  31-12-2023 22:00  0.280877  0.532758  0.368188\n",
            "43822  31-12-2023 23:00  0.237036  0.891977  0.656815\n",
            "\n",
            "[43823 rows x 4 columns]\n",
            "\n",
            "Modified DataFrame has been saved to: /content/Grid_data/modified_combined_price_per_unit.csv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "\n",
        "# Mount Google Drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "# Path to the folder containing the CSV files\n",
        "folder_path = '/content/Grid_data'\n",
        "\n",
        "# List to hold DataFrames from each CSV file\n",
        "dfs = []\n",
        "\n",
        "# Iterate through each file in the folder\n",
        "for filename in os.listdir(folder_path):\n",
        "    if filename.endswith(\".csv\"):\n",
        "        # Read the CSV file into a DataFrame\n",
        "        year = filename.split('_')[2].split('.')[0]\n",
        "        df = pd.read_csv(os.path.join(folder_path, filename))\n",
        "        # Add a column indicating the year\n",
        "        df['year'] = year\n",
        "        dfs.append(df)\n",
        "\n",
        "# Concatenate all DataFrames into a single DataFrame\n",
        "combined_df = pd.concat(dfs, ignore_index=True)\n",
        "\n",
        "# Sort the combined DataFrame by date\n",
        "combined_df.sort_values(by='date', inplace=True)\n",
        "\n",
        "# Display the sorted combined DataFrame\n",
        "print(combined_df)\n",
        "\n",
        "# Save the sorted combined DataFrame to a CSV file\n",
        "combined_file_path = '/content/Grid_data/combined_unit_consumption_sorted.csv'\n",
        "combined_df.to_csv(combined_file_path, index=False)\n"
      ],
      "metadata": {
        "id": "YG96TMdYfQoW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e76dc1d0-0ec8-452b-ad07-becaf46a8cb3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
            "                   date        c1        c2        c3  year\n",
            "8760   01-01-2019 01:00 -0.782604 -1.257395 -1.723086  2019\n",
            "8761   01-01-2019 02:00 -1.940058 -1.872742 -1.255012  2019\n",
            "8762   01-01-2019 03:00 -1.207456 -1.277210 -0.920492  2019\n",
            "8763   01-01-2019 04:00 -1.027473 -1.938944 -0.997374  2019\n",
            "8764   01-01-2019 05:00 -1.125531 -1.845975 -0.554305  2019\n",
            "...                 ...       ...       ...       ...   ...\n",
            "35034  31-12-2023 19:00 -1.954289 -0.981347 -1.654103  2023\n",
            "35035  31-12-2023 20:00 -1.428185 -0.525543 -1.545585  2023\n",
            "35036  31-12-2023 21:00 -0.701491 -1.279522 -1.019624  2023\n",
            "35037  31-12-2023 22:00 -1.076426 -1.231602 -1.164986  2023\n",
            "35038  31-12-2023 23:00 -1.579196 -1.526573 -0.571834  2023\n",
            "\n",
            "[43823 rows x 5 columns]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Load the combined file with the year column\n",
        "combined_file_path = '/content/Grid_data/combined_unit_consumption_sorted.csv'\n",
        "combined_df = pd.read_csv(combined_file_path)\n",
        "\n",
        "# Display the DataFrame before deleting the 'year' column\n",
        "print(\"Before deleting 'year' column:\")\n",
        "print(combined_df)\n",
        "\n",
        "# Delete the 'year' column\n",
        "combined_df.drop(columns=['year'], inplace=True)\n",
        "\n",
        "# Display the DataFrame after deleting the 'year' column\n",
        "print(\"\\nAfter deleting 'year' column:\")\n",
        "print(combined_df)\n",
        "\n",
        "# Save the modified DataFrame to a new CSV file\n",
        "modified_combined_file_path = '/content/Grid_data/modified_combined_unit_consumption.csv'\n",
        "combined_df.to_csv(modified_combined_file_path, index=False)\n",
        "print(\"\\nModified DataFrame has been saved to:\", modified_combined_file_path)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G6hE044Y5e33",
        "outputId": "5fc5d2d8-93ad-4518-f5ce-c47e863ec489"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Before deleting 'year' column:\n",
            "                   date        c1        c2        c3  year\n",
            "0      01-01-2019 01:00 -0.782604 -1.257395 -1.723086  2019\n",
            "1      01-01-2019 02:00 -1.940058 -1.872742 -1.255012  2019\n",
            "2      01-01-2019 03:00 -1.207456 -1.277210 -0.920492  2019\n",
            "3      01-01-2019 04:00 -1.027473 -1.938944 -0.997374  2019\n",
            "4      01-01-2019 05:00 -1.125531 -1.845975 -0.554305  2019\n",
            "...                 ...       ...       ...       ...   ...\n",
            "43818  31-12-2023 19:00 -1.954289 -0.981347 -1.654103  2023\n",
            "43819  31-12-2023 20:00 -1.428185 -0.525543 -1.545585  2023\n",
            "43820  31-12-2023 21:00 -0.701491 -1.279522 -1.019624  2023\n",
            "43821  31-12-2023 22:00 -1.076426 -1.231602 -1.164986  2023\n",
            "43822  31-12-2023 23:00 -1.579196 -1.526573 -0.571834  2023\n",
            "\n",
            "[43823 rows x 5 columns]\n",
            "\n",
            "After deleting 'year' column:\n",
            "                   date        c1        c2        c3\n",
            "0      01-01-2019 01:00 -0.782604 -1.257395 -1.723086\n",
            "1      01-01-2019 02:00 -1.940058 -1.872742 -1.255012\n",
            "2      01-01-2019 03:00 -1.207456 -1.277210 -0.920492\n",
            "3      01-01-2019 04:00 -1.027473 -1.938944 -0.997374\n",
            "4      01-01-2019 05:00 -1.125531 -1.845975 -0.554305\n",
            "...                 ...       ...       ...       ...\n",
            "43818  31-12-2023 19:00 -1.954289 -0.981347 -1.654103\n",
            "43819  31-12-2023 20:00 -1.428185 -0.525543 -1.545585\n",
            "43820  31-12-2023 21:00 -0.701491 -1.279522 -1.019624\n",
            "43821  31-12-2023 22:00 -1.076426 -1.231602 -1.164986\n",
            "43822  31-12-2023 23:00 -1.579196 -1.526573 -0.571834\n",
            "\n",
            "[43823 rows x 4 columns]\n",
            "\n",
            "Modified DataFrame has been saved to: /content/Grid_data/modified_combined_unit_consumption.csv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Load the files\n",
        "unit_consumption_file = '/content/modified_combined_unit_consumption.csv'\n",
        "price_per_unit_file = '/content/modified_combined_price_per_unit.csv'\n",
        "grid_stability_file = '/content/grid_stability_2019_2023.csv'\n",
        "\n",
        "unit_consumption_df = pd.read_csv(unit_consumption_file)\n",
        "price_per_unit_df = pd.read_csv(price_per_unit_file)\n",
        "grid_stability_df = pd.read_csv(grid_stability_file)\n",
        "\n",
        "# Merge unit_consumption_df with price_per_unit_df on 'date'\n",
        "merged_df = pd.merge(unit_consumption_df, price_per_unit_df, on='date', how='outer')\n",
        "\n",
        "# Merge merged_df with grid_stability_df on 'date'\n",
        "final_merged_df = pd.merge(merged_df, grid_stability_df, on='date', how='outer')\n",
        "\n",
        "# Display the final merged DataFrame\n",
        "print(final_merged_df)\n",
        "\n",
        "# Save the final merged DataFrame to a new CSV file\n",
        "final_merged_file_path = '/content/Grid_data/final_merged_data_2.csv'\n",
        "final_merged_df.to_csv(final_merged_file_path, index=False)\n",
        "print(\"\\nFinal merged DataFrame has been saved to:\", final_merged_file_path)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XEvFNfs755Ez",
        "outputId": "cfb70c94-c117-4fc7-8539-dae0dc8eebca"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                   date        c1        c2        c3        p1        p2  \\\n",
            "0      01-01-2019 01:00 -0.782604 -1.257395 -1.723086  0.859578  0.887445   \n",
            "1      01-01-2019 02:00 -1.940058 -1.872742 -1.255012  0.862414  0.562139   \n",
            "2      01-01-2019 03:00 -1.207456 -1.277210 -0.920492  0.766689  0.839444   \n",
            "3      01-01-2019 04:00 -1.027473 -1.938944 -0.997374  0.976744  0.929381   \n",
            "4      01-01-2019 05:00 -1.125531 -1.845975 -0.554305  0.455450  0.656947   \n",
            "...                 ...       ...       ...       ...       ...       ...   \n",
            "46002  31-03-2024 19:00       NaN       NaN       NaN       NaN       NaN   \n",
            "46003  31-03-2024 20:00       NaN       NaN       NaN       NaN       NaN   \n",
            "46004  31-03-2024 21:00       NaN       NaN       NaN       NaN       NaN   \n",
            "46005  31-03-2024 22:00       NaN       NaN       NaN       NaN       NaN   \n",
            "46006  31-03-2024 23:00       NaN       NaN       NaN       NaN       NaN   \n",
            "\n",
            "             p3 stability  \n",
            "0      0.958034  unstable  \n",
            "1      0.781760    stable  \n",
            "2      0.109853  unstable  \n",
            "3      0.362718  unstable  \n",
            "4      0.820923  unstable  \n",
            "...         ...       ...  \n",
            "46002       NaN    stable  \n",
            "46003       NaN  unstable  \n",
            "46004       NaN    stable  \n",
            "46005       NaN  unstable  \n",
            "46006       NaN  unstable  \n",
            "\n",
            "[46007 rows x 8 columns]\n",
            "\n",
            "Final merged DataFrame has been saved to: /content/Grid_data/final_merged_data_2.csv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "s10nqSIU8C3b"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Load the final merged DataFrame\n",
        "final_merged_file_path = '/content/Grid_data/final_merged_data_2.csv'\n",
        "final_merged_df = pd.read_csv(final_merged_file_path)\n",
        "\n",
        "# Display the count of missing values in each column\n",
        "print(\"Count of missing values in each column:\")\n",
        "print(final_merged_df.isnull().sum())\n",
        "\n",
        "# Drop rows with any missing values\n",
        "final_merged_df.dropna(inplace=True)\n",
        "\n",
        "# Save the DataFrame after handling missing values to a new CSV file\n",
        "missing_values_removed_file_path = '/content/Grid_data/final_merged_data_missing_values_removed.csv'\n",
        "final_merged_df.to_csv(missing_values_removed_file_path, index=False)\n",
        "print(\"\\nDataFrame after handling missing values has been saved to:\", missing_values_removed_file_path)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U_3xy4mO628g",
        "outputId": "e3fef7c9-806a-47b9-f726-67592236f872"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Count of missing values in each column:\n",
            "date            0\n",
            "c1           2184\n",
            "c2           2184\n",
            "c3           2184\n",
            "p1           2184\n",
            "p2           2184\n",
            "p3           2184\n",
            "stability       0\n",
            "dtype: int64\n",
            "\n",
            "DataFrame after handling missing values has been saved to: /content/Grid_data/final_merged_data_missing_values_removed.csv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from scipy.stats import zscore\n",
        "\n",
        "# Load the final merged DataFrame\n",
        "final_merged_file_path = '/content/Grid_data/final_merged_data_missing_values_removed.csv'\n",
        "final_merged_df = pd.read_csv(final_merged_file_path)\n",
        "\n",
        "# Calculate Z-scores for numerical columns (excluding 'date' and 'stability' columns)\n",
        "numerical_cols = final_merged_df.select_dtypes(include=['float64', 'int64']).columns\n",
        "final_merged_df[numerical_cols] = final_merged_df[numerical_cols].apply(zscore)\n",
        "\n",
        "# Define the threshold for Z-score to identify outliers\n",
        "zscore_threshold = 3\n",
        "\n",
        "# Remove rows with Z-scores exceeding the threshold\n",
        "outlier_removed_df = final_merged_df[(final_merged_df[numerical_cols] < zscore_threshold).all(axis=1)]\n",
        "\n",
        "\n",
        "\n",
        "# Save the outlier removed DataFrame to a new CSV file\n",
        "outlier_removed_file_path = '/content/Grid_data/final_merged_data_outlier_removed.csv'\n",
        "outlier_removed_df.to_csv(outlier_removed_file_path, index=False)\n",
        "print(\"\\nOutliers removed DataFrame has been saved to:\", outlier_removed_file_path)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nmn-BzKA8Dvq",
        "outputId": "1a50c466-cff0-46f0-ccad-5480077c33a1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Outliers removed DataFrame has been saved to: /content/Grid_data/final_merged_data_outlier_removed.csv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "fiezpRBy8UD2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.metrics import classification_report, confusion_matrix\n",
        "\n",
        "# Load the existing and newly prepared datasets\n",
        "unit_consumption_file = '/content/final_merged_data_outlier_removed.csv'\n",
        "validation_file = '/content/grid_stability_3months_validation_data.csv'\n",
        "\n",
        "unit_consumption_df = pd.read_csv(unit_consumption_file)\n",
        "validation_df = pd.read_csv(validation_file)\n",
        "\n",
        "# Prepare a database distributing power to three consumption nodes\n",
        "total_power = 44922.666632137916  # Total power generated\n",
        "node1_power = total_power * 0.20\n",
        "node2_power = total_power * 0.45\n",
        "node3_power = total_power * 0.35\n",
        "\n",
        "# Add power generated and stored at each node\n",
        "unit_consumption_df['p1'] = node1_power\n",
        "unit_consumption_df['p2'] = node2_power\n",
        "unit_consumption_df['p3'] = node3_power\n",
        "\n",
        "# Prepare features (X) and target variable (y)\n",
        "X = unit_consumption_df.drop(columns=['date', 'stability'])\n",
        "y = unit_consumption_df['stability']\n",
        "\n",
        "# Split the data into training and testing sets\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
        "\n",
        "# Train a Random Forest classifier\n",
        "clf = RandomForestClassifier(random_state=42)\n",
        "clf.fit(X_train, y_train)\n",
        "\n",
        "# Predict stability on the testing set\n",
        "y_pred = clf.predict(X_test)\n",
        "\n",
        "# Calculate classification metrics\n",
        "classification_metrics = classification_report(y_test, y_pred)\n",
        "confusion_mat = confusion_matrix(y_test, y_pred)\n",
        "\n",
        "# Print classification metrics\n",
        "print(\"Classification Metrics:\")\n",
        "print(classification_metrics)\n",
        "\n",
        "# Print confusion matrix\n",
        "print(\"\\nConfusion Matrix:\")\n",
        "print(confusion_mat)\n",
        "\n",
        "# Validate the model using provided validation data\n",
        "X_validation = validation_df.drop(columns=['date', 'stability'])\n",
        "y_validation = validation_df['stability']\n",
        "\n",
        "# Predict stability on validation data\n",
        "y_validation_pred = clf.predict(X_validation)\n",
        "\n",
        "# Calculate F1 score, recall, and confusion matrix for validation data\n",
        "classification_metrics_validation = classification_report(y_validation, y_validation_pred)\n",
        "confusion_mat_validation = confusion_matrix(y_validation, y_validation_pred)\n",
        "\n",
        "# Print F1 score, recall, and confusion matrix for validation data\n",
        "print(\"\\nValidation Metrics:\")\n",
        "print(classification_metrics_validation)\n",
        "print(\"\\nConfusion Matrix (Validation Data):\")\n",
        "print(confusion_mat_validation)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2jJIX9Nr-QZ0",
        "outputId": "e61abd68-c266-4cb3-c68e-184e2e092fef"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Classification Metrics:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "      stable       0.42      0.22      0.29      3187\n",
            "    unstable       0.65      0.83      0.73      5578\n",
            "\n",
            "    accuracy                           0.61      8765\n",
            "   macro avg       0.54      0.52      0.51      8765\n",
            "weighted avg       0.57      0.61      0.57      8765\n",
            "\n",
            "\n",
            "Confusion Matrix:\n",
            "[[ 709 2478]\n",
            " [ 965 4613]]\n",
            "\n",
            "Validation Metrics:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "      stable       0.35      0.37      0.36       763\n",
            "    unstable       0.65      0.63      0.64      1421\n",
            "\n",
            "    accuracy                           0.54      2184\n",
            "   macro avg       0.50      0.50      0.50      2184\n",
            "weighted avg       0.55      0.54      0.54      2184\n",
            "\n",
            "\n",
            "Confusion Matrix (Validation Data):\n",
            "[[286 477]\n",
            " [529 892]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Load the existing dataset\n",
        "dataset_file = '/content/final_merged_data_outlier_removed.csv'\n",
        "dataset = pd.read_csv(dataset_file)\n",
        "\n",
        "# Calculate total power generated\n",
        "total_power = dataset[['p1', 'p2', 'p3']].sum(axis=1)\n",
        "\n",
        "# Calculate power stored at each node based on percentages\n",
        "node1_power = total_power * 0.20\n",
        "node2_power = total_power * 0.45\n",
        "node3_power = total_power * 0.35\n",
        "\n",
        "# Add power generated and stored at each node to the dataset\n",
        "dataset['total_power_generated'] = total_power\n",
        "dataset['node1_power'] = node1_power\n",
        "dataset['node2_power'] = node2_power\n",
        "dataset['node3_power'] = node3_power\n",
        "\n",
        "# Determine the percentage of 'Stable' and 'Unstable' grid conditions over the span of three months\n",
        "stable_count = dataset[dataset['stability'] == 'stable'].shape[0]\n",
        "unstable_count = dataset[dataset['stability'] == 'unstable'].shape[0]\n",
        "total_samples = dataset.shape[0]\n",
        "\n",
        "stable_percentage = (stable_count / total_samples) * 100\n",
        "unstable_percentage = (unstable_count / total_samples) * 100\n",
        "\n",
        "print(\"Percentage of Stable Grid Conditions:\", stable_percentage)\n",
        "print(\"Percentage of Unstable Grid Conditions:\", unstable_percentage)\n",
        "\n",
        "# Additional analysis can be performed to identify patterns in instability\n",
        "# For example, analyzing instability by hour of the day or day of the week\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LccDSokrAi6d",
        "outputId": "b4381e0d-9ada-4b59-adb4-20be08cba857"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Percentage of Stable Grid Conditions: 36.25265271661\n",
            "Percentage of Unstable Grid Conditions: 63.74734728339\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "z0kvBrH3B9Or"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}