Climate Connect Case Study.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing basic Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing dataset : power_actual"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(70080, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>datetime</th>\n",
       "      <th>ghi</th>\n",
       "      <th>gti</th>\n",
       "      <th>power</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2017-10-01 00:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2017-10-01 00:15:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2017-10-01 00:30:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0             datetime  ghi  gti  power\n",
       "0           0  2017-10-01 00:00:00  0.0  0.0    0.0\n",
       "1           1  2017-10-01 00:15:00  0.0  0.0    0.0\n",
       "2           2  2017-10-01 00:30:00  0.0  0.0    0.0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_power_actual = pd.read_csv('power_actual.csv')\n",
    "print (df_power_actual.shape)\n",
    "df_power_actual.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>datetime</th>\n",
       "      <th>ghi</th>\n",
       "      <th>gti</th>\n",
       "      <th>power</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>70077</th>\n",
       "      <td>70077</td>\n",
       "      <td>2019-09-30 23:15:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70078</th>\n",
       "      <td>70078</td>\n",
       "      <td>2019-09-30 23:30:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70079</th>\n",
       "      <td>70079</td>\n",
       "      <td>2019-09-30 23:45:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Unnamed: 0             datetime  ghi  gti  power\n",
       "70077       70077  2019-09-30 23:15:00  0.0  0.0    0.0\n",
       "70078       70078  2019-09-30 23:30:00  0.0  0.0    0.0\n",
       "70079       70079  2019-09-30 23:45:00  0.0  0.0    0.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_power_actual.tail(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing dataset : weather_actuals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13619, 31)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>plant_id</th>\n",
       "      <th>datetime_utc</th>\n",
       "      <th>datetime_local</th>\n",
       "      <th>cloud_cover</th>\n",
       "      <th>apparent_temperature</th>\n",
       "      <th>temperature</th>\n",
       "      <th>humidity</th>\n",
       "      <th>dew_point</th>\n",
       "      <th>wind_bearing</th>\n",
       "      <th>...</th>\n",
       "      <th>precip_accumulation</th>\n",
       "      <th>precip_intensity</th>\n",
       "      <th>precip_probability</th>\n",
       "      <th>precip_type</th>\n",
       "      <th>visibility</th>\n",
       "      <th>sunrise</th>\n",
       "      <th>sunset</th>\n",
       "      <th>icon</th>\n",
       "      <th>summary</th>\n",
       "      <th>updated_at</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2017-09-30 18:30:00</td>\n",
       "      <td>2017-10-01 00:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>26.52</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>19.46</td>\n",
       "      <td>308</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>rain</td>\n",
       "      <td>3.57</td>\n",
       "      <td>2017-10-01 00:52:02</td>\n",
       "      <td>2017-10-01 12:45:07</td>\n",
       "      <td>clear-night</td>\n",
       "      <td>Clear</td>\n",
       "      <td>2019-07-16 16:37:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2017-09-30 19:30:00</td>\n",
       "      <td>2017-10-01 01:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28.78</td>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "      <td>20.06</td>\n",
       "      <td>-9999</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>rain</td>\n",
       "      <td>3.06</td>\n",
       "      <td>2017-10-01 00:52:02</td>\n",
       "      <td>2017-10-01 12:45:07</td>\n",
       "      <td>fog</td>\n",
       "      <td>Foggy</td>\n",
       "      <td>2019-07-16 16:37:16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  plant_id         datetime_utc       datetime_local  \\\n",
       "0           0         1  2017-09-30 18:30:00  2017-10-01 00:00:00   \n",
       "1           1         1  2017-09-30 19:30:00  2017-10-01 01:00:00   \n",
       "\n",
       "   cloud_cover  apparent_temperature  temperature  humidity  dew_point  \\\n",
       "0          0.0                 26.52           26         1      19.46   \n",
       "1          0.0                 28.78           27         1      20.06   \n",
       "\n",
       "   wind_bearing  ...  precip_accumulation  precip_intensity  \\\n",
       "0           308  ...              -9999.0           -9999.0   \n",
       "1         -9999  ...              -9999.0           -9999.0   \n",
       "\n",
       "   precip_probability  precip_type  visibility              sunrise  \\\n",
       "0             -9999.0         rain        3.57  2017-10-01 00:52:02   \n",
       "1             -9999.0         rain        3.06  2017-10-01 00:52:02   \n",
       "\n",
       "                sunset         icon  summary           updated_at  \n",
       "0  2017-10-01 12:45:07  clear-night    Clear  2019-07-16 16:37:16  \n",
       "1  2017-10-01 12:45:07          fog    Foggy  2019-07-16 16:37:16  \n",
       "\n",
       "[2 rows x 31 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_weather_actuals = pd.read_csv('weather_actuals.csv')\n",
    "print (df_weather_actuals.shape)\n",
    "df_weather_actuals.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>plant_id</th>\n",
       "      <th>datetime_utc</th>\n",
       "      <th>datetime_local</th>\n",
       "      <th>cloud_cover</th>\n",
       "      <th>apparent_temperature</th>\n",
       "      <th>temperature</th>\n",
       "      <th>humidity</th>\n",
       "      <th>dew_point</th>\n",
       "      <th>wind_bearing</th>\n",
       "      <th>...</th>\n",
       "      <th>precip_accumulation</th>\n",
       "      <th>precip_intensity</th>\n",
       "      <th>precip_probability</th>\n",
       "      <th>precip_type</th>\n",
       "      <th>visibility</th>\n",
       "      <th>sunrise</th>\n",
       "      <th>sunset</th>\n",
       "      <th>icon</th>\n",
       "      <th>summary</th>\n",
       "      <th>updated_at</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13617</th>\n",
       "      <td>13617</td>\n",
       "      <td>1</td>\n",
       "      <td>2019-09-30 16:30:00</td>\n",
       "      <td>2019-09-30 22:00:00</td>\n",
       "      <td>0.30</td>\n",
       "      <td>28.97</td>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "      <td>21.99</td>\n",
       "      <td>88</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16.09</td>\n",
       "      <td>2019-09-30 00:51:16</td>\n",
       "      <td>2019-09-30 12:46:54</td>\n",
       "      <td>partly-cloudy-night</td>\n",
       "      <td>Partly Cloudy</td>\n",
       "      <td>2019-10-01 23:32:46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13618</th>\n",
       "      <td>13618</td>\n",
       "      <td>1</td>\n",
       "      <td>2019-09-30 17:30:00</td>\n",
       "      <td>2019-09-30 23:00:00</td>\n",
       "      <td>0.35</td>\n",
       "      <td>28.15</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>22.09</td>\n",
       "      <td>95</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16.09</td>\n",
       "      <td>2019-09-30 00:51:16</td>\n",
       "      <td>2019-09-30 12:46:54</td>\n",
       "      <td>partly-cloudy-night</td>\n",
       "      <td>Partly Cloudy</td>\n",
       "      <td>2019-10-01 23:32:46</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Unnamed: 0  plant_id         datetime_utc       datetime_local  \\\n",
       "13617       13617         1  2019-09-30 16:30:00  2019-09-30 22:00:00   \n",
       "13618       13618         1  2019-09-30 17:30:00  2019-09-30 23:00:00   \n",
       "\n",
       "       cloud_cover  apparent_temperature  temperature  humidity  dew_point  \\\n",
       "13617         0.30                 28.97           27         1      21.99   \n",
       "13618         0.35                 28.15           26         1      22.09   \n",
       "\n",
       "       wind_bearing  ...  precip_accumulation  precip_intensity  \\\n",
       "13617            88  ...                  NaN               0.0   \n",
       "13618            95  ...                  NaN               0.0   \n",
       "\n",
       "       precip_probability  precip_type  visibility              sunrise  \\\n",
       "13617                 0.0          NaN       16.09  2019-09-30 00:51:16   \n",
       "13618                 0.0          NaN       16.09  2019-09-30 00:51:16   \n",
       "\n",
       "                    sunset                 icon        summary  \\\n",
       "13617  2019-09-30 12:46:54  partly-cloudy-night  Partly Cloudy   \n",
       "13618  2019-09-30 12:46:54  partly-cloudy-night  Partly Cloudy   \n",
       "\n",
       "                updated_at  \n",
       "13617  2019-10-01 23:32:46  \n",
       "13618  2019-10-01 23:32:46  \n",
       "\n",
       "[2 rows x 31 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_weather_actuals.tail(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'plant_id', 'datetime_utc', 'datetime_local',\n",
       "       'cloud_cover', 'apparent_temperature', 'temperature', 'humidity',\n",
       "       'dew_point', 'wind_bearing', 'wind_speed', 'wind_chill', 'wind_gust',\n",
       "       'heat_index', 'pressure', 'qpf', 'uv_index', 'snow', 'pop', 'fctcode',\n",
       "       'ozone', 'precip_accumulation', 'precip_intensity',\n",
       "       'precip_probability', 'precip_type', 'visibility', 'sunrise', 'sunset',\n",
       "       'icon', 'summary', 'updated_at'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_weather_actuals.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing dataset for forecasting purpose : weather_forecast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(648, 31)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>plant_id</th>\n",
       "      <th>datetime_utc</th>\n",
       "      <th>datetime_local</th>\n",
       "      <th>cloud_cover</th>\n",
       "      <th>apparent_temperature</th>\n",
       "      <th>temperature</th>\n",
       "      <th>humidity</th>\n",
       "      <th>dew_point</th>\n",
       "      <th>wind_bearing</th>\n",
       "      <th>...</th>\n",
       "      <th>precip_accumulation</th>\n",
       "      <th>precip_intensity</th>\n",
       "      <th>precip_probability</th>\n",
       "      <th>precip_type</th>\n",
       "      <th>visibility</th>\n",
       "      <th>sunrise</th>\n",
       "      <th>sunset</th>\n",
       "      <th>icon</th>\n",
       "      <th>summary</th>\n",
       "      <th>updated_at</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2019-09-30 18:30:00</td>\n",
       "      <td>2019-10-01 00:00:00</td>\n",
       "      <td>0.35</td>\n",
       "      <td>26.41</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>22.01</td>\n",
       "      <td>101</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16.09</td>\n",
       "      <td>2019-10-01 00:51:47</td>\n",
       "      <td>2019-10-01 12:45:42</td>\n",
       "      <td>partly-cloudy-night</td>\n",
       "      <td>Partly Cloudy</td>\n",
       "      <td>2019-10-02 23:32:38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2019-09-30 19:30:00</td>\n",
       "      <td>2019-10-01 01:00:00</td>\n",
       "      <td>0.43</td>\n",
       "      <td>25.95</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>22.06</td>\n",
       "      <td>105</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16.09</td>\n",
       "      <td>2019-10-01 00:51:47</td>\n",
       "      <td>2019-10-01 12:45:42</td>\n",
       "      <td>partly-cloudy-night</td>\n",
       "      <td>Partly Cloudy</td>\n",
       "      <td>2019-10-02 23:32:38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  plant_id         datetime_utc       datetime_local  \\\n",
       "0           0         1  2019-09-30 18:30:00  2019-10-01 00:00:00   \n",
       "1           1         1  2019-09-30 19:30:00  2019-10-01 01:00:00   \n",
       "\n",
       "   cloud_cover  apparent_temperature  temperature  humidity  dew_point  \\\n",
       "0         0.35                 26.41           26         1      22.01   \n",
       "1         0.43                 25.95           25         1      22.06   \n",
       "\n",
       "   wind_bearing  ...  precip_accumulation  precip_intensity  \\\n",
       "0           101  ...                  NaN               0.0   \n",
       "1           105  ...                  NaN               0.0   \n",
       "\n",
       "   precip_probability  precip_type  visibility              sunrise  \\\n",
       "0                 0.0          NaN       16.09  2019-10-01 00:51:47   \n",
       "1                 0.0          NaN       16.09  2019-10-01 00:51:47   \n",
       "\n",
       "                sunset                 icon        summary  \\\n",
       "0  2019-10-01 12:45:42  partly-cloudy-night  Partly Cloudy   \n",
       "1  2019-10-01 12:45:42  partly-cloudy-night  Partly Cloudy   \n",
       "\n",
       "            updated_at  \n",
       "0  2019-10-02 23:32:38  \n",
       "1  2019-10-02 23:32:38  \n",
       "\n",
       "[2 rows x 31 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_weather_forecast = pd.read_csv('weather_forecast.csv')\n",
    "print (df_weather_forecast.shape)\n",
    "df_weather_forecast.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>plant_id</th>\n",
       "      <th>datetime_utc</th>\n",
       "      <th>datetime_local</th>\n",
       "      <th>cloud_cover</th>\n",
       "      <th>apparent_temperature</th>\n",
       "      <th>temperature</th>\n",
       "      <th>humidity</th>\n",
       "      <th>dew_point</th>\n",
       "      <th>wind_bearing</th>\n",
       "      <th>...</th>\n",
       "      <th>precip_accumulation</th>\n",
       "      <th>precip_intensity</th>\n",
       "      <th>precip_probability</th>\n",
       "      <th>precip_type</th>\n",
       "      <th>visibility</th>\n",
       "      <th>sunrise</th>\n",
       "      <th>sunset</th>\n",
       "      <th>icon</th>\n",
       "      <th>summary</th>\n",
       "      <th>updated_at</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>646</th>\n",
       "      <td>646</td>\n",
       "      <td>1</td>\n",
       "      <td>2019-10-27 16:30:00</td>\n",
       "      <td>2019-10-27 22:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>22.86</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "      <td>12.50</td>\n",
       "      <td>92</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16.09</td>\n",
       "      <td>2019-10-27 01:07:31</td>\n",
       "      <td>2019-10-27 12:18:06</td>\n",
       "      <td>clear-night</td>\n",
       "      <td>Clear</td>\n",
       "      <td>2019-10-28 12:33:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>647</th>\n",
       "      <td>647</td>\n",
       "      <td>1</td>\n",
       "      <td>2019-10-27 17:30:00</td>\n",
       "      <td>2019-10-27 23:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>22.21</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>12.67</td>\n",
       "      <td>146</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16.09</td>\n",
       "      <td>2019-10-27 01:07:31</td>\n",
       "      <td>2019-10-27 12:18:06</td>\n",
       "      <td>clear-night</td>\n",
       "      <td>Clear</td>\n",
       "      <td>2019-10-28 12:33:28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Unnamed: 0  plant_id         datetime_utc       datetime_local  \\\n",
       "646         646         1  2019-10-27 16:30:00  2019-10-27 22:00:00   \n",
       "647         647         1  2019-10-27 17:30:00  2019-10-27 23:00:00   \n",
       "\n",
       "     cloud_cover  apparent_temperature  temperature  humidity  dew_point  \\\n",
       "646          0.0                 22.86           23         1      12.50   \n",
       "647          0.0                 22.21           22         1      12.67   \n",
       "\n",
       "     wind_bearing  ...  precip_accumulation  precip_intensity  \\\n",
       "646            92  ...                  NaN               0.0   \n",
       "647           146  ...                  NaN               0.0   \n",
       "\n",
       "     precip_probability  precip_type  visibility              sunrise  \\\n",
       "646                 0.0          NaN       16.09  2019-10-27 01:07:31   \n",
       "647                 0.0          NaN       16.09  2019-10-27 01:07:31   \n",
       "\n",
       "                  sunset         icon  summary           updated_at  \n",
       "646  2019-10-27 12:18:06  clear-night    Clear  2019-10-28 12:33:28  \n",
       "647  2019-10-27 12:18:06  clear-night    Clear  2019-10-28 12:33:28  \n",
       "\n",
       "[2 rows x 31 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_weather_forecast.tail(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'plant_id', 'datetime_utc', 'datetime_local',\n",
       "       'cloud_cover', 'apparent_temperature', 'temperature', 'humidity',\n",
       "       'dew_point', 'wind_bearing', 'wind_speed', 'wind_chill', 'wind_gust',\n",
       "       'heat_index', 'pressure', 'qpf', 'uv_index', 'snow', 'pop', 'fctcode',\n",
       "       'ozone', 'precip_accumulation', 'precip_intensity',\n",
       "       'precip_probability', 'precip_type', 'visibility', 'sunrise', 'sunset',\n",
       "       'icon', 'summary', 'updated_at'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_weather_forecast.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Merging two datasets : power_actual + weather_actuals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_power_actual = df_power_actual.drop(['Unnamed: 0'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_power_actual.columns = ['datetime_local', 'ghi', 'gti', 'power']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df =pd.merge(df_power_actual,df_weather_actuals,on='datetime_local')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datetime_local</th>\n",
       "      <th>ghi</th>\n",
       "      <th>gti</th>\n",
       "      <th>power</th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>plant_id</th>\n",
       "      <th>datetime_utc</th>\n",
       "      <th>cloud_cover</th>\n",
       "      <th>apparent_temperature</th>\n",
       "      <th>temperature</th>\n",
       "      <th>...</th>\n",
       "      <th>precip_accumulation</th>\n",
       "      <th>precip_intensity</th>\n",
       "      <th>precip_probability</th>\n",
       "      <th>precip_type</th>\n",
       "      <th>visibility</th>\n",
       "      <th>sunrise</th>\n",
       "      <th>sunset</th>\n",
       "      <th>icon</th>\n",
       "      <th>summary</th>\n",
       "      <th>updated_at</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017-10-01 00:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2017-09-30 18:30:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>26.52</td>\n",
       "      <td>26</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>rain</td>\n",
       "      <td>3.57</td>\n",
       "      <td>2017-10-01 00:52:02</td>\n",
       "      <td>2017-10-01 12:45:07</td>\n",
       "      <td>clear-night</td>\n",
       "      <td>Clear</td>\n",
       "      <td>2019-07-16 16:37:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017-10-01 01:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2017-09-30 19:30:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28.78</td>\n",
       "      <td>27</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>rain</td>\n",
       "      <td>3.06</td>\n",
       "      <td>2017-10-01 00:52:02</td>\n",
       "      <td>2017-10-01 12:45:07</td>\n",
       "      <td>fog</td>\n",
       "      <td>Foggy</td>\n",
       "      <td>2019-07-16 16:37:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2017-10-01 02:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2017-09-30 20:30:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>26.58</td>\n",
       "      <td>26</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>rain</td>\n",
       "      <td>3.06</td>\n",
       "      <td>2017-10-01 00:52:02</td>\n",
       "      <td>2017-10-01 12:45:07</td>\n",
       "      <td>fog</td>\n",
       "      <td>Foggy</td>\n",
       "      <td>2019-07-16 16:37:16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        datetime_local  ghi  gti  power  Unnamed: 0  plant_id  \\\n",
       "0  2017-10-01 00:00:00  0.0  0.0    0.0           0         1   \n",
       "1  2017-10-01 01:00:00  0.0  0.0    0.0           1         1   \n",
       "2  2017-10-01 02:00:00  0.0  0.0    0.0           2         1   \n",
       "\n",
       "          datetime_utc  cloud_cover  apparent_temperature  temperature  ...  \\\n",
       "0  2017-09-30 18:30:00          0.0                 26.52           26  ...   \n",
       "1  2017-09-30 19:30:00          0.0                 28.78           27  ...   \n",
       "2  2017-09-30 20:30:00          0.0                 26.58           26  ...   \n",
       "\n",
       "   precip_accumulation  precip_intensity  precip_probability  precip_type  \\\n",
       "0              -9999.0           -9999.0             -9999.0         rain   \n",
       "1              -9999.0           -9999.0             -9999.0         rain   \n",
       "2              -9999.0           -9999.0             -9999.0         rain   \n",
       "\n",
       "   visibility              sunrise               sunset         icon  summary  \\\n",
       "0        3.57  2017-10-01 00:52:02  2017-10-01 12:45:07  clear-night    Clear   \n",
       "1        3.06  2017-10-01 00:52:02  2017-10-01 12:45:07          fog    Foggy   \n",
       "2        3.06  2017-10-01 00:52:02  2017-10-01 12:45:07          fog    Foggy   \n",
       "\n",
       "            updated_at  \n",
       "0  2019-07-16 16:37:16  \n",
       "1  2019-07-16 16:37:16  \n",
       "2  2019-07-16 16:37:16  \n",
       "\n",
       "[3 rows x 34 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datetime_local</th>\n",
       "      <th>ghi</th>\n",
       "      <th>gti</th>\n",
       "      <th>power</th>\n",
       "      <th>plant_id</th>\n",
       "      <th>datetime_utc</th>\n",
       "      <th>cloud_cover</th>\n",
       "      <th>apparent_temperature</th>\n",
       "      <th>temperature</th>\n",
       "      <th>humidity</th>\n",
       "      <th>...</th>\n",
       "      <th>precip_accumulation</th>\n",
       "      <th>precip_intensity</th>\n",
       "      <th>precip_probability</th>\n",
       "      <th>precip_type</th>\n",
       "      <th>visibility</th>\n",
       "      <th>sunrise</th>\n",
       "      <th>sunset</th>\n",
       "      <th>icon</th>\n",
       "      <th>summary</th>\n",
       "      <th>updated_at</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017-10-01 00:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2017-09-30 18:30:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>26.52</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>rain</td>\n",
       "      <td>3.57</td>\n",
       "      <td>2017-10-01 00:52:02</td>\n",
       "      <td>2017-10-01 12:45:07</td>\n",
       "      <td>clear-night</td>\n",
       "      <td>Clear</td>\n",
       "      <td>2019-07-16 16:37:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017-10-01 01:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2017-09-30 19:30:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28.78</td>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>rain</td>\n",
       "      <td>3.06</td>\n",
       "      <td>2017-10-01 00:52:02</td>\n",
       "      <td>2017-10-01 12:45:07</td>\n",
       "      <td>fog</td>\n",
       "      <td>Foggy</td>\n",
       "      <td>2019-07-16 16:37:16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        datetime_local  ghi  gti  power  plant_id         datetime_utc  \\\n",
       "0  2017-10-01 00:00:00  0.0  0.0    0.0         1  2017-09-30 18:30:00   \n",
       "1  2017-10-01 01:00:00  0.0  0.0    0.0         1  2017-09-30 19:30:00   \n",
       "\n",
       "   cloud_cover  apparent_temperature  temperature  humidity  ...  \\\n",
       "0          0.0                 26.52           26         1  ...   \n",
       "1          0.0                 28.78           27         1  ...   \n",
       "\n",
       "   precip_accumulation  precip_intensity  precip_probability  precip_type  \\\n",
       "0              -9999.0           -9999.0             -9999.0         rain   \n",
       "1              -9999.0           -9999.0             -9999.0         rain   \n",
       "\n",
       "   visibility              sunrise               sunset         icon  summary  \\\n",
       "0        3.57  2017-10-01 00:52:02  2017-10-01 12:45:07  clear-night    Clear   \n",
       "1        3.06  2017-10-01 00:52:02  2017-10-01 12:45:07          fog    Foggy   \n",
       "\n",
       "            updated_at  \n",
       "0  2019-07-16 16:37:16  \n",
       "1  2019-07-16 16:37:16  \n",
       "\n",
       "[2 rows x 33 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.drop(['Unnamed: 0'],axis=1)\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ghi</th>\n",
       "      <th>gti</th>\n",
       "      <th>power</th>\n",
       "      <th>plant_id</th>\n",
       "      <th>cloud_cover</th>\n",
       "      <th>apparent_temperature</th>\n",
       "      <th>temperature</th>\n",
       "      <th>humidity</th>\n",
       "      <th>dew_point</th>\n",
       "      <th>wind_bearing</th>\n",
       "      <th>...</th>\n",
       "      <th>qpf</th>\n",
       "      <th>uv_index</th>\n",
       "      <th>snow</th>\n",
       "      <th>pop</th>\n",
       "      <th>fctcode</th>\n",
       "      <th>ozone</th>\n",
       "      <th>precip_accumulation</th>\n",
       "      <th>precip_intensity</th>\n",
       "      <th>precip_probability</th>\n",
       "      <th>visibility</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>6810.0</td>\n",
       "      <td>6810.0</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.0</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>3744.0</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>3744.0</td>\n",
       "      <td>3744.0</td>\n",
       "      <td>3744.0</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>3744.0</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>35.255627</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-314.005310</td>\n",
       "      <td>25.944639</td>\n",
       "      <td>24.498054</td>\n",
       "      <td>0.537411</td>\n",
       "      <td>11.924626</td>\n",
       "      <td>-498.677363</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-2394.989280</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-2187.730797</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-2396.367775</td>\n",
       "      <td>-2396.398081</td>\n",
       "      <td>-117.086149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>557.059759</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1744.610028</td>\n",
       "      <td>10.469808</td>\n",
       "      <td>8.840016</td>\n",
       "      <td>0.498617</td>\n",
       "      <td>9.129459</td>\n",
       "      <td>2557.943872</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4269.320576</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4385.706548</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4268.545854</td>\n",
       "      <td>4268.528833</td>\n",
       "      <td>1130.763316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-9999.000000</td>\n",
       "      <td>3.140000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-20.560000</td>\n",
       "      <td>-9999.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.000000</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.000000</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.000000</td>\n",
       "      <td>-9999.000000</td>\n",
       "      <td>-9999.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>17.310000</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.920000</td>\n",
       "      <td>79.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>236.900000</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.540000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>25.540000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>10.710000</td>\n",
       "      <td>149.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>270.800000</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>16.090000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.810000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.410000</td>\n",
       "      <td>34.580000</td>\n",
       "      <td>31.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>19.670000</td>\n",
       "      <td>293.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>280.800000</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>16.090000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15504.750000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>49.750000</td>\n",
       "      <td>47.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>28.350000</td>\n",
       "      <td>359.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>351.700000</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>6.220000</td>\n",
       "      <td>0.730000</td>\n",
       "      <td>16.090000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          ghi     gti         power  plant_id   cloud_cover  \\\n",
       "count  6810.0  6810.0  13619.000000   13619.0  13619.000000   \n",
       "mean      0.0     0.0     35.255627       1.0   -314.005310   \n",
       "std       0.0     0.0    557.059759       0.0   1744.610028   \n",
       "min       0.0     0.0      0.000000       1.0  -9999.000000   \n",
       "25%       0.0     0.0      0.000000       1.0      0.000000   \n",
       "50%       0.0     0.0      0.000000       1.0      0.000000   \n",
       "75%       0.0     0.0      5.810000       1.0      0.410000   \n",
       "max       0.0     0.0  15504.750000       1.0      1.000000   \n",
       "\n",
       "       apparent_temperature   temperature      humidity     dew_point  \\\n",
       "count          13619.000000  13619.000000  13619.000000  13619.000000   \n",
       "mean              25.944639     24.498054      0.537411     11.924626   \n",
       "std               10.469808      8.840016      0.498617      9.129459   \n",
       "min                3.140000      5.000000      0.000000    -20.560000   \n",
       "25%               17.310000     17.000000      0.000000      5.920000   \n",
       "50%               25.540000     25.000000      1.000000     10.710000   \n",
       "75%               34.580000     31.000000      1.000000     19.670000   \n",
       "max               49.750000     47.000000      1.000000     28.350000   \n",
       "\n",
       "       wind_bearing  ...     qpf      uv_index    snow     pop  fctcode  \\\n",
       "count  13619.000000  ...  3744.0  13619.000000  3744.0  3744.0   3744.0   \n",
       "mean    -498.677363  ... -9999.0  -2394.989280 -9999.0 -9999.0  -9999.0   \n",
       "std     2557.943872  ...     0.0   4269.320576     0.0     0.0      0.0   \n",
       "min    -9999.000000  ... -9999.0  -9999.000000 -9999.0 -9999.0  -9999.0   \n",
       "25%       79.000000  ... -9999.0      0.000000 -9999.0 -9999.0  -9999.0   \n",
       "50%      149.000000  ... -9999.0      0.000000 -9999.0 -9999.0  -9999.0   \n",
       "75%      293.000000  ... -9999.0      2.000000 -9999.0 -9999.0  -9999.0   \n",
       "max      359.000000  ... -9999.0     12.000000 -9999.0 -9999.0  -9999.0   \n",
       "\n",
       "              ozone  precip_accumulation  precip_intensity  \\\n",
       "count  13619.000000               3744.0      13619.000000   \n",
       "mean   -2187.730797              -9999.0      -2396.367775   \n",
       "std     4385.706548                  0.0       4268.545854   \n",
       "min    -9999.000000              -9999.0      -9999.000000   \n",
       "25%      236.900000              -9999.0          0.000000   \n",
       "50%      270.800000              -9999.0          0.000000   \n",
       "75%      280.800000              -9999.0          0.000000   \n",
       "max      351.700000              -9999.0          6.220000   \n",
       "\n",
       "       precip_probability    visibility  \n",
       "count        13619.000000  13619.000000  \n",
       "mean         -2396.398081   -117.086149  \n",
       "std           4268.528833   1130.763316  \n",
       "min          -9999.000000  -9999.000000  \n",
       "25%              0.000000      3.540000  \n",
       "50%              0.000000     16.090000  \n",
       "75%              0.000000     16.090000  \n",
       "max              0.730000     16.090000  \n",
       "\n",
       "[8 rows x 25 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datetime_local             0\n",
       "ghi                     6809\n",
       "gti                     6809\n",
       "power                      0\n",
       "plant_id                   0\n",
       "datetime_utc               0\n",
       "cloud_cover                0\n",
       "apparent_temperature       0\n",
       "temperature                0\n",
       "humidity                   0\n",
       "dew_point                  0\n",
       "wind_bearing               0\n",
       "wind_speed                 0\n",
       "wind_chill              9875\n",
       "wind_gust                  0\n",
       "heat_index              9875\n",
       "pressure                   0\n",
       "qpf                     9875\n",
       "uv_index                   0\n",
       "snow                    9875\n",
       "pop                     9875\n",
       "fctcode                 9875\n",
       "ozone                      0\n",
       "precip_accumulation     9875\n",
       "precip_intensity           0\n",
       "precip_probability         0\n",
       "precip_type             7129\n",
       "visibility                 0\n",
       "sunrise                    0\n",
       "sunset                     0\n",
       "icon                       0\n",
       "summary                    0\n",
       "updated_at                 0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Filtering numeric data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "numeric = []\n",
    "category = []\n",
    "for i in df :\n",
    "    if (df[i].dtypes=='float64' or df[i].dtypes=='int64'):\n",
    "        numeric.append(i)     \n",
    "    else :\n",
    "        category.append(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checking the min and max value from the data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ghi --> 0.0 --> 0.0\n",
      "gti --> 0.0 --> 0.0\n",
      "power --> 0.0 --> 15504.75\n",
      "plant_id --> 1 --> 1\n",
      "cloud_cover --> -9999.0 --> 1.0\n",
      "apparent_temperature --> 3.14 --> 49.75\n",
      "temperature --> 5 --> 47\n",
      "humidity --> 0 --> 1\n",
      "dew_point --> -20.56 --> 28.35\n",
      "wind_bearing --> -9999 --> 359\n",
      "wind_speed --> -9999.0 --> 9.73\n",
      "wind_chill --> -9999.0 --> -9999.0\n",
      "wind_gust --> -9999.0 --> 15.15\n",
      "heat_index --> -9999.0 --> -9999.0\n",
      "pressure --> -9999.0 --> 1025.69\n",
      "qpf --> -9999.0 --> -9999.0\n",
      "uv_index --> -9999 --> 12\n",
      "snow --> -9999.0 --> -9999.0\n",
      "pop --> -9999.0 --> -9999.0\n",
      "fctcode --> -9999.0 --> -9999.0\n",
      "ozone --> -9999.0 --> 351.7\n",
      "precip_accumulation --> -9999.0 --> -9999.0\n",
      "precip_intensity --> -9999.0 --> 6.22\n",
      "precip_probability --> -9999.0 --> 0.73\n",
      "visibility --> -9999.0 --> 16.09\n"
     ]
    }
   ],
   "source": [
    "for i in numeric:\n",
    "    print (i, \"-->\",min(df[i]), \"-->\", max(df[i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checking the unique values out of the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0. nan]\n",
      "[ 0. nan]\n",
      "[1]\n",
      "[-9999.    nan]\n",
      "[-9999.    nan]\n",
      "[-9999.    nan]\n",
      "[-9999.    nan]\n",
      "[-9999.    nan]\n",
      "[-9999.    nan]\n",
      "[-9999.    nan]\n",
      "[-9999.     335.82   333.09 ...   302.2    303.1    301.6 ]\n"
     ]
    }
   ],
   "source": [
    "print (df.ghi.unique())\n",
    "print (df.gti.unique())\n",
    "print (df.plant_id.unique())\n",
    "print (df.wind_chill.unique())\n",
    "print (df.heat_index.unique())\n",
    "print (df.qpf.unique())\n",
    "print (df.snow.unique())\n",
    "print (df['pop'].unique())\n",
    "print (df.fctcode.unique())\n",
    "print (df.precip_accumulation.unique())\n",
    "print(df.ozone.unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dropping out the columns whoes minimum and maximum values are the same"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13619, 23)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_drop_numeric = df.drop(['ghi','gti','plant_id','wind_chill','heat_index','qpf','snow','pop','fctcode','precip_accumulation'],axis=1)\n",
    "df_drop_numeric.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datetime_local</th>\n",
       "      <th>power</th>\n",
       "      <th>datetime_utc</th>\n",
       "      <th>cloud_cover</th>\n",
       "      <th>apparent_temperature</th>\n",
       "      <th>temperature</th>\n",
       "      <th>humidity</th>\n",
       "      <th>dew_point</th>\n",
       "      <th>wind_bearing</th>\n",
       "      <th>wind_speed</th>\n",
       "      <th>...</th>\n",
       "      <th>ozone</th>\n",
       "      <th>precip_intensity</th>\n",
       "      <th>precip_probability</th>\n",
       "      <th>precip_type</th>\n",
       "      <th>visibility</th>\n",
       "      <th>sunrise</th>\n",
       "      <th>sunset</th>\n",
       "      <th>icon</th>\n",
       "      <th>summary</th>\n",
       "      <th>updated_at</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017-10-01 00:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2017-09-30 18:30:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>26.52</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>19.46</td>\n",
       "      <td>308</td>\n",
       "      <td>0.83</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>rain</td>\n",
       "      <td>3.57</td>\n",
       "      <td>2017-10-01 00:52:02</td>\n",
       "      <td>2017-10-01 12:45:07</td>\n",
       "      <td>clear-night</td>\n",
       "      <td>Clear</td>\n",
       "      <td>2019-07-16 16:37:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017-10-01 01:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2017-09-30 19:30:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28.78</td>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "      <td>20.06</td>\n",
       "      <td>-9999</td>\n",
       "      <td>-9999.00</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>rain</td>\n",
       "      <td>3.06</td>\n",
       "      <td>2017-10-01 00:52:02</td>\n",
       "      <td>2017-10-01 12:45:07</td>\n",
       "      <td>fog</td>\n",
       "      <td>Foggy</td>\n",
       "      <td>2019-07-16 16:37:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2017-10-01 02:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2017-09-30 20:30:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>26.58</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>19.97</td>\n",
       "      <td>-9999</td>\n",
       "      <td>-9999.00</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>rain</td>\n",
       "      <td>3.06</td>\n",
       "      <td>2017-10-01 00:52:02</td>\n",
       "      <td>2017-10-01 12:45:07</td>\n",
       "      <td>fog</td>\n",
       "      <td>Foggy</td>\n",
       "      <td>2019-07-16 16:37:16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        datetime_local  power         datetime_utc  cloud_cover  \\\n",
       "0  2017-10-01 00:00:00    0.0  2017-09-30 18:30:00          0.0   \n",
       "1  2017-10-01 01:00:00    0.0  2017-09-30 19:30:00          0.0   \n",
       "2  2017-10-01 02:00:00    0.0  2017-09-30 20:30:00          0.0   \n",
       "\n",
       "   apparent_temperature  temperature  humidity  dew_point  wind_bearing  \\\n",
       "0                 26.52           26         1      19.46           308   \n",
       "1                 28.78           27         1      20.06         -9999   \n",
       "2                 26.58           26         1      19.97         -9999   \n",
       "\n",
       "   wind_speed  ...   ozone  precip_intensity  precip_probability  precip_type  \\\n",
       "0        0.83  ... -9999.0           -9999.0             -9999.0         rain   \n",
       "1    -9999.00  ... -9999.0           -9999.0             -9999.0         rain   \n",
       "2    -9999.00  ... -9999.0           -9999.0             -9999.0         rain   \n",
       "\n",
       "   visibility              sunrise               sunset         icon summary  \\\n",
       "0        3.57  2017-10-01 00:52:02  2017-10-01 12:45:07  clear-night   Clear   \n",
       "1        3.06  2017-10-01 00:52:02  2017-10-01 12:45:07          fog   Foggy   \n",
       "2        3.06  2017-10-01 00:52:02  2017-10-01 12:45:07          fog   Foggy   \n",
       "\n",
       "            updated_at  \n",
       "0  2019-07-16 16:37:16  \n",
       "1  2019-07-16 16:37:16  \n",
       "2  2019-07-16 16:37:16  \n",
       "\n",
       "[3 rows x 23 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_drop_numeric.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['datetime_local',\n",
       " 'datetime_utc',\n",
       " 'precip_type',\n",
       " 'sunrise',\n",
       " 'sunset',\n",
       " 'icon',\n",
       " 'summary',\n",
       " 'updated_at']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "category"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dropping all the date apart from 'datetime_local"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datetime_local</th>\n",
       "      <th>power</th>\n",
       "      <th>cloud_cover</th>\n",
       "      <th>apparent_temperature</th>\n",
       "      <th>temperature</th>\n",
       "      <th>humidity</th>\n",
       "      <th>dew_point</th>\n",
       "      <th>wind_bearing</th>\n",
       "      <th>wind_speed</th>\n",
       "      <th>wind_gust</th>\n",
       "      <th>pressure</th>\n",
       "      <th>uv_index</th>\n",
       "      <th>ozone</th>\n",
       "      <th>precip_intensity</th>\n",
       "      <th>precip_probability</th>\n",
       "      <th>precip_type</th>\n",
       "      <th>visibility</th>\n",
       "      <th>icon</th>\n",
       "      <th>summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017-10-01 00:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>26.52</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>19.46</td>\n",
       "      <td>308</td>\n",
       "      <td>0.83</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>1007.9</td>\n",
       "      <td>-9999</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>rain</td>\n",
       "      <td>3.57</td>\n",
       "      <td>clear-night</td>\n",
       "      <td>Clear</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017-10-01 01:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28.78</td>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "      <td>20.06</td>\n",
       "      <td>-9999</td>\n",
       "      <td>-9999.00</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>rain</td>\n",
       "      <td>3.06</td>\n",
       "      <td>fog</td>\n",
       "      <td>Foggy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2017-10-01 02:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>26.58</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>19.97</td>\n",
       "      <td>-9999</td>\n",
       "      <td>-9999.00</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>rain</td>\n",
       "      <td>3.06</td>\n",
       "      <td>fog</td>\n",
       "      <td>Foggy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        datetime_local  power  cloud_cover  apparent_temperature  temperature  \\\n",
       "0  2017-10-01 00:00:00    0.0          0.0                 26.52           26   \n",
       "1  2017-10-01 01:00:00    0.0          0.0                 28.78           27   \n",
       "2  2017-10-01 02:00:00    0.0          0.0                 26.58           26   \n",
       "\n",
       "   humidity  dew_point  wind_bearing  wind_speed  wind_gust  pressure  \\\n",
       "0         1      19.46           308        0.83    -9999.0    1007.9   \n",
       "1         1      20.06         -9999    -9999.00    -9999.0   -9999.0   \n",
       "2         1      19.97         -9999    -9999.00    -9999.0   -9999.0   \n",
       "\n",
       "   uv_index   ozone  precip_intensity  precip_probability precip_type  \\\n",
       "0     -9999 -9999.0           -9999.0             -9999.0        rain   \n",
       "1     -9999 -9999.0           -9999.0             -9999.0        rain   \n",
       "2     -9999 -9999.0           -9999.0             -9999.0        rain   \n",
       "\n",
       "   visibility         icon summary  \n",
       "0        3.57  clear-night   Clear  \n",
       "1        3.06          fog   Foggy  \n",
       "2        3.06          fog   Foggy  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_drop_category = df_drop_numeric.drop([ 'datetime_utc', 'sunrise', 'sunset','updated_at'],axis = 1)\n",
    "df_drop_category.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13619, 19)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_drop_category.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Converting all int64 data type into float64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13619, 19)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1 = df_drop_category.copy()\n",
    "df1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in df1 :\n",
    "    if (df1[i].dtypes=='int64'):\n",
    "        df1[i] = df1[i].astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datetime_local           object\n",
       "power                   float64\n",
       "cloud_cover             float64\n",
       "apparent_temperature    float64\n",
       "temperature             float64\n",
       "humidity                float64\n",
       "dew_point               float64\n",
       "wind_bearing            float64\n",
       "wind_speed              float64\n",
       "wind_gust               float64\n",
       "pressure                float64\n",
       "uv_index                float64\n",
       "ozone                   float64\n",
       "precip_intensity        float64\n",
       "precip_probability      float64\n",
       "precip_type              object\n",
       "visibility              float64\n",
       "icon                     object\n",
       "summary                  object\n",
       "dtype: object"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# We can see so many values contain -9999.0 values, we are checking the count of it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "datetime_local --> 0\n",
      "power --> 0\n",
      "cloud_cover --> 428\n",
      "apparent_temperature --> 0\n",
      "temperature --> 0\n",
      "humidity --> 0\n",
      "dew_point --> 0\n",
      "wind_bearing --> 919\n",
      "wind_speed --> 740\n",
      "wind_gust --> 3264\n",
      "pressure --> 2156\n",
      "uv_index --> 3264\n",
      "ozone --> 3264\n",
      "precip_intensity --> 3264\n",
      "precip_probability --> 3264\n",
      "precip_type --> 0\n",
      "visibility --> 176\n",
      "icon --> 0\n",
      "summary --> 0\n"
     ]
    }
   ],
   "source": [
    "for i in df1:\n",
    "    print(i,\"-->\",len(df1[i][df1[i] == (-9999.0)]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Considering -9999.0 data as an wrong data imputation we will replace that with mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "numeric = []\n",
    "category = []\n",
    "for i in df1 :\n",
    "    if (df1[i].dtypes=='float64' or df1[i].dtypes=='int64'):\n",
    "        numeric.append(i)     \n",
    "    else :\n",
    "        category.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in numeric:\n",
    "    df1.replace(-9999.0, np.nan,inplace=True)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datetime_local             0\n",
       "power                      0\n",
       "cloud_cover              428\n",
       "apparent_temperature       0\n",
       "temperature                0\n",
       "humidity                   0\n",
       "dew_point                  0\n",
       "wind_bearing             919\n",
       "wind_speed               740\n",
       "wind_gust               3264\n",
       "pressure                2156\n",
       "uv_index                3264\n",
       "ozone                   3264\n",
       "precip_intensity        3264\n",
       "precip_probability      3264\n",
       "precip_type             7129\n",
       "visibility               176\n",
       "icon                       0\n",
       "summary                    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean = ['cloud_cover','wind_speed','wind_gust','uv_index','precip_intensity','precip_probability','visibility','ozone']\n",
    "fwd = ['wind_bearing','pressure',]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = df1.drop(['precip_type'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in df1:\n",
    "    for j in mean:\n",
    "        if i==j:\n",
    "            df1[i].fillna(df1[i].mean(),inplace=True)\n",
    "        else :\n",
    "            df1[i].fillna(method='ffill',inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checking for null values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datetime_local          0\n",
       "power                   0\n",
       "cloud_cover             0\n",
       "apparent_temperature    0\n",
       "temperature             0\n",
       "humidity                0\n",
       "dew_point               0\n",
       "wind_bearing            0\n",
       "wind_speed              0\n",
       "wind_gust               0\n",
       "pressure                0\n",
       "uv_index                0\n",
       "ozone                   0\n",
       "precip_intensity        0\n",
       "precip_probability      0\n",
       "visibility              0\n",
       "icon                    0\n",
       "summary                 0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>power</th>\n",
       "      <th>cloud_cover</th>\n",
       "      <th>apparent_temperature</th>\n",
       "      <th>temperature</th>\n",
       "      <th>humidity</th>\n",
       "      <th>dew_point</th>\n",
       "      <th>wind_bearing</th>\n",
       "      <th>wind_speed</th>\n",
       "      <th>wind_gust</th>\n",
       "      <th>pressure</th>\n",
       "      <th>uv_index</th>\n",
       "      <th>ozone</th>\n",
       "      <th>precip_intensity</th>\n",
       "      <th>precip_probability</th>\n",
       "      <th>visibility</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "      <td>13619.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>35.255627</td>\n",
       "      <td>0.237562</td>\n",
       "      <td>25.944639</td>\n",
       "      <td>24.498054</td>\n",
       "      <td>0.537411</td>\n",
       "      <td>11.924626</td>\n",
       "      <td>190.680667</td>\n",
       "      <td>2.514417</td>\n",
       "      <td>4.245261</td>\n",
       "      <td>1009.952931</td>\n",
       "      <td>1.871270</td>\n",
       "      <td>274.459707</td>\n",
       "      <td>0.058259</td>\n",
       "      <td>0.018400</td>\n",
       "      <td>12.165426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>557.059759</td>\n",
       "      <td>0.337459</td>\n",
       "      <td>10.469808</td>\n",
       "      <td>8.840016</td>\n",
       "      <td>0.498617</td>\n",
       "      <td>9.129459</td>\n",
       "      <td>110.783674</td>\n",
       "      <td>1.363799</td>\n",
       "      <td>1.898630</td>\n",
       "      <td>7.597269</td>\n",
       "      <td>2.550984</td>\n",
       "      <td>13.295012</td>\n",
       "      <td>0.218980</td>\n",
       "      <td>0.048505</td>\n",
       "      <td>6.268604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.140000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-20.560000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.630000</td>\n",
       "      <td>991.660000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>225.800000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>17.310000</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.920000</td>\n",
       "      <td>92.000000</td>\n",
       "      <td>1.620000</td>\n",
       "      <td>3.015000</td>\n",
       "      <td>1003.450000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>270.300000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.540000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.010000</td>\n",
       "      <td>25.540000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>10.710000</td>\n",
       "      <td>185.000000</td>\n",
       "      <td>2.410000</td>\n",
       "      <td>4.245261</td>\n",
       "      <td>1011.310000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>274.459707</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>16.090000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>5.810000</td>\n",
       "      <td>0.410000</td>\n",
       "      <td>34.580000</td>\n",
       "      <td>31.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>19.670000</td>\n",
       "      <td>300.000000</td>\n",
       "      <td>3.330000</td>\n",
       "      <td>4.710000</td>\n",
       "      <td>1016.155000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>280.800000</td>\n",
       "      <td>0.058259</td>\n",
       "      <td>0.018400</td>\n",
       "      <td>16.090000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>15504.750000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>49.750000</td>\n",
       "      <td>47.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>28.350000</td>\n",
       "      <td>359.000000</td>\n",
       "      <td>9.730000</td>\n",
       "      <td>15.150000</td>\n",
       "      <td>1025.690000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>351.700000</td>\n",
       "      <td>6.220000</td>\n",
       "      <td>0.730000</td>\n",
       "      <td>16.090000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              power   cloud_cover  apparent_temperature   temperature  \\\n",
       "count  13619.000000  13619.000000          13619.000000  13619.000000   \n",
       "mean      35.255627      0.237562             25.944639     24.498054   \n",
       "std      557.059759      0.337459             10.469808      8.840016   \n",
       "min        0.000000      0.000000              3.140000      5.000000   \n",
       "25%        0.000000      0.000000             17.310000     17.000000   \n",
       "50%        0.000000      0.010000             25.540000     25.000000   \n",
       "75%        5.810000      0.410000             34.580000     31.000000   \n",
       "max    15504.750000      1.000000             49.750000     47.000000   \n",
       "\n",
       "           humidity     dew_point  wind_bearing    wind_speed     wind_gust  \\\n",
       "count  13619.000000  13619.000000  13619.000000  13619.000000  13619.000000   \n",
       "mean       0.537411     11.924626    190.680667      2.514417      4.245261   \n",
       "std        0.498617      9.129459    110.783674      1.363799      1.898630   \n",
       "min        0.000000    -20.560000      0.000000      0.000000      0.630000   \n",
       "25%        0.000000      5.920000     92.000000      1.620000      3.015000   \n",
       "50%        1.000000     10.710000    185.000000      2.410000      4.245261   \n",
       "75%        1.000000     19.670000    300.000000      3.330000      4.710000   \n",
       "max        1.000000     28.350000    359.000000      9.730000     15.150000   \n",
       "\n",
       "           pressure      uv_index         ozone  precip_intensity  \\\n",
       "count  13619.000000  13619.000000  13619.000000      13619.000000   \n",
       "mean    1009.952931      1.871270    274.459707          0.058259   \n",
       "std        7.597269      2.550984     13.295012          0.218980   \n",
       "min      991.660000      0.000000    225.800000          0.000000   \n",
       "25%     1003.450000      0.000000    270.300000          0.000000   \n",
       "50%     1011.310000      1.000000    274.459707          0.000000   \n",
       "75%     1016.155000      2.000000    280.800000          0.058259   \n",
       "max     1025.690000     12.000000    351.700000          6.220000   \n",
       "\n",
       "       precip_probability    visibility  \n",
       "count        13619.000000  13619.000000  \n",
       "mean             0.018400     12.165426  \n",
       "std              0.048505      6.268604  \n",
       "min              0.000000      0.000000  \n",
       "25%              0.000000      3.540000  \n",
       "50%              0.000000     16.090000  \n",
       "75%              0.018400     16.090000  \n",
       "max              0.730000     16.090000  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_label = df1.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datetime_local</th>\n",
       "      <th>power</th>\n",
       "      <th>cloud_cover</th>\n",
       "      <th>apparent_temperature</th>\n",
       "      <th>temperature</th>\n",
       "      <th>humidity</th>\n",
       "      <th>dew_point</th>\n",
       "      <th>wind_bearing</th>\n",
       "      <th>wind_speed</th>\n",
       "      <th>wind_gust</th>\n",
       "      <th>pressure</th>\n",
       "      <th>uv_index</th>\n",
       "      <th>ozone</th>\n",
       "      <th>precip_intensity</th>\n",
       "      <th>precip_probability</th>\n",
       "      <th>visibility</th>\n",
       "      <th>icon</th>\n",
       "      <th>summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017-10-01 00:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>26.52</td>\n",
       "      <td>26.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.46</td>\n",
       "      <td>308.0</td>\n",
       "      <td>0.83</td>\n",
       "      <td>4.245261</td>\n",
       "      <td>1007.90</td>\n",
       "      <td>1.87127</td>\n",
       "      <td>274.459707</td>\n",
       "      <td>0.058259</td>\n",
       "      <td>0.0184</td>\n",
       "      <td>3.57</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017-10-01 01:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>28.78</td>\n",
       "      <td>27.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>20.06</td>\n",
       "      <td>308.0</td>\n",
       "      <td>0.83</td>\n",
       "      <td>4.245261</td>\n",
       "      <td>1007.90</td>\n",
       "      <td>1.87127</td>\n",
       "      <td>274.459707</td>\n",
       "      <td>0.058259</td>\n",
       "      <td>0.0184</td>\n",
       "      <td>3.06</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2017-10-01 02:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>26.58</td>\n",
       "      <td>26.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.97</td>\n",
       "      <td>308.0</td>\n",
       "      <td>0.83</td>\n",
       "      <td>4.245261</td>\n",
       "      <td>1007.90</td>\n",
       "      <td>1.87127</td>\n",
       "      <td>274.459707</td>\n",
       "      <td>0.058259</td>\n",
       "      <td>0.0184</td>\n",
       "      <td>3.06</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2017-10-01 03:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>23.84</td>\n",
       "      <td>23.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>18.66</td>\n",
       "      <td>270.0</td>\n",
       "      <td>0.45</td>\n",
       "      <td>4.245261</td>\n",
       "      <td>1008.12</td>\n",
       "      <td>1.87127</td>\n",
       "      <td>274.459707</td>\n",
       "      <td>0.058259</td>\n",
       "      <td>0.0184</td>\n",
       "      <td>3.56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2017-10-01 04:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>24.13</td>\n",
       "      <td>24.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>17.84</td>\n",
       "      <td>270.0</td>\n",
       "      <td>0.45</td>\n",
       "      <td>4.245261</td>\n",
       "      <td>1008.12</td>\n",
       "      <td>1.87127</td>\n",
       "      <td>274.459707</td>\n",
       "      <td>0.058259</td>\n",
       "      <td>0.0184</td>\n",
       "      <td>3.06</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13614</th>\n",
       "      <td>2019-09-30 19:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.16</td>\n",
       "      <td>31.66</td>\n",
       "      <td>29.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>21.19</td>\n",
       "      <td>40.0</td>\n",
       "      <td>2.72</td>\n",
       "      <td>4.590000</td>\n",
       "      <td>1007.44</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>269.600000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>16.09</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13615</th>\n",
       "      <td>2019-09-30 20:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.16</td>\n",
       "      <td>30.42</td>\n",
       "      <td>28.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>21.35</td>\n",
       "      <td>59.0</td>\n",
       "      <td>3.24</td>\n",
       "      <td>5.450000</td>\n",
       "      <td>1008.25</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>268.700000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>16.09</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13616</th>\n",
       "      <td>2019-09-30 21:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.18</td>\n",
       "      <td>29.56</td>\n",
       "      <td>27.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>21.61</td>\n",
       "      <td>76.0</td>\n",
       "      <td>3.34</td>\n",
       "      <td>5.830000</td>\n",
       "      <td>1008.65</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>268.400000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>16.09</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13617</th>\n",
       "      <td>2019-09-30 22:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.30</td>\n",
       "      <td>28.97</td>\n",
       "      <td>27.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>21.99</td>\n",
       "      <td>88.0</td>\n",
       "      <td>3.22</td>\n",
       "      <td>5.730000</td>\n",
       "      <td>1008.76</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>268.700000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>16.09</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13618</th>\n",
       "      <td>2019-09-30 23:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.35</td>\n",
       "      <td>28.15</td>\n",
       "      <td>26.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>22.09</td>\n",
       "      <td>95.0</td>\n",
       "      <td>3.14</td>\n",
       "      <td>6.030000</td>\n",
       "      <td>1008.61</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>268.700000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>16.09</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13619 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            datetime_local  power  cloud_cover  apparent_temperature  \\\n",
       "0      2017-10-01 00:00:00    0.0         0.00                 26.52   \n",
       "1      2017-10-01 01:00:00    0.0         0.00                 28.78   \n",
       "2      2017-10-01 02:00:00    0.0         0.00                 26.58   \n",
       "3      2017-10-01 03:00:00    0.0         0.00                 23.84   \n",
       "4      2017-10-01 04:00:00    0.0         0.00                 24.13   \n",
       "...                    ...    ...          ...                   ...   \n",
       "13614  2019-09-30 19:00:00    0.0         0.16                 31.66   \n",
       "13615  2019-09-30 20:00:00    0.0         0.16                 30.42   \n",
       "13616  2019-09-30 21:00:00    0.0         0.18                 29.56   \n",
       "13617  2019-09-30 22:00:00    0.0         0.30                 28.97   \n",
       "13618  2019-09-30 23:00:00    0.0         0.35                 28.15   \n",
       "\n",
       "       temperature  humidity  dew_point  wind_bearing  wind_speed  wind_gust  \\\n",
       "0             26.0       1.0      19.46         308.0        0.83   4.245261   \n",
       "1             27.0       1.0      20.06         308.0        0.83   4.245261   \n",
       "2             26.0       1.0      19.97         308.0        0.83   4.245261   \n",
       "3             23.0       1.0      18.66         270.0        0.45   4.245261   \n",
       "4             24.0       1.0      17.84         270.0        0.45   4.245261   \n",
       "...            ...       ...        ...           ...         ...        ...   \n",
       "13614         29.0       1.0      21.19          40.0        2.72   4.590000   \n",
       "13615         28.0       1.0      21.35          59.0        3.24   5.450000   \n",
       "13616         27.0       1.0      21.61          76.0        3.34   5.830000   \n",
       "13617         27.0       1.0      21.99          88.0        3.22   5.730000   \n",
       "13618         26.0       1.0      22.09          95.0        3.14   6.030000   \n",
       "\n",
       "       pressure  uv_index       ozone  precip_intensity  precip_probability  \\\n",
       "0       1007.90   1.87127  274.459707          0.058259              0.0184   \n",
       "1       1007.90   1.87127  274.459707          0.058259              0.0184   \n",
       "2       1007.90   1.87127  274.459707          0.058259              0.0184   \n",
       "3       1008.12   1.87127  274.459707          0.058259              0.0184   \n",
       "4       1008.12   1.87127  274.459707          0.058259              0.0184   \n",
       "...         ...       ...         ...               ...                 ...   \n",
       "13614   1007.44   0.00000  269.600000          0.000000              0.0000   \n",
       "13615   1008.25   0.00000  268.700000          0.000000              0.0000   \n",
       "13616   1008.65   0.00000  268.400000          0.000000              0.0000   \n",
       "13617   1008.76   0.00000  268.700000          0.000000              0.0000   \n",
       "13618   1008.61   0.00000  268.700000          0.000000              0.0000   \n",
       "\n",
       "       visibility  icon  summary  \n",
       "0            3.57     1        1  \n",
       "1            3.06     3        2  \n",
       "2            3.06     3        2  \n",
       "3            3.56     1        1  \n",
       "4            3.06     3        2  \n",
       "...           ...   ...      ...  \n",
       "13614       16.09     5       12  \n",
       "13615       16.09     5       12  \n",
       "13616       16.09     5       12  \n",
       "13617       16.09     5       12  \n",
       "13618       16.09     5       12  \n",
       "\n",
       "[13619 rows x 18 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "labelencoder = LabelEncoder()\n",
    "df_label['icon'] = labelencoder.fit_transform(df_label['icon'])\n",
    "df_label['summary'] = labelencoder.fit_transform(df_label['summary'])\n",
    "df_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " VIF score for cloud_cover is --> 1.47\n",
      " VIF score for apparent_temperature is --> 42.25\n",
      " VIF score for temperature is --> 33.0\n",
      " VIF score for humidity is --> 2.73\n",
      " VIF score for dew_point is --> 4.02\n",
      " VIF score for wind_bearing is --> 1.17\n",
      " VIF score for wind_speed is --> 2.65\n",
      " VIF score for wind_gust is --> 2.25\n",
      " VIF score for pressure is --> 5.62\n",
      " VIF score for uv_index is --> 1.4\n",
      " VIF score for ozone is --> 1.34\n",
      " VIF score for precip_intensity is --> 2.19\n",
      " VIF score for precip_probability is --> 2.51\n",
      " VIF score for visibility is --> 1.91\n"
     ]
    }
   ],
   "source": [
    "import statsmodels.api as sm\n",
    "from statsmodels.stats.outliers_influence import variance_inflation_factor\n",
    "import statsmodels.formula.api as smf\n",
    "from statsmodels.tools.tools import add_constant\n",
    "\n",
    "def vif_cal(input_data, dependent_col):\n",
    "    x_vars=input_data.drop([dependent_col], axis=1)\n",
    "    xvar_names=x_vars.columns\n",
    "    for i in range(0, xvar_names.shape[0]):\n",
    "        y=x_vars[xvar_names[i]] \n",
    "        x=x_vars[xvar_names.drop(xvar_names[i])]\n",
    "        rsq=smf.ols(formula='y~x', data=x_vars).fit().rsquared  \n",
    "        vif=round(1/(1-rsq),2)\n",
    "        print( \" VIF score for\", xvar_names[i], \"is -->\" , vif)\n",
    "        \n",
    "X = df1.copy()\n",
    "X = X.drop(['datetime_local','icon','summary'], axis = 1)\n",
    "# calling previously defined function\n",
    "vif_cal(X,'power')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Since ,apparent temperature and temperature had high multivollinearity so removing those columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_with =df_label.drop(['apparent_temperature','temperature'],axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "power                  5.81\n",
      "cloud_cover            0.41\n",
      "humidity               1.00\n",
      "dew_point             13.75\n",
      "wind_bearing         208.00\n",
      "wind_speed             1.71\n",
      "wind_gust              1.70\n",
      "pressure              12.70\n",
      "uv_index               2.00\n",
      "ozone                 10.50\n",
      "precip_intensity       0.06\n",
      "precip_probability     0.02\n",
      "visibility            12.55\n",
      "icon                   2.00\n",
      "summary                5.00\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "df_l =df_with\n",
    "Q1 = df_l.quantile(0.25)\n",
    "Q3 = df_l.quantile(0.75)\n",
    "IQR = Q3 - Q1\n",
    "print(IQR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datetime_local</th>\n",
       "      <th>power</th>\n",
       "      <th>cloud_cover</th>\n",
       "      <th>humidity</th>\n",
       "      <th>dew_point</th>\n",
       "      <th>wind_bearing</th>\n",
       "      <th>wind_speed</th>\n",
       "      <th>wind_gust</th>\n",
       "      <th>pressure</th>\n",
       "      <th>uv_index</th>\n",
       "      <th>ozone</th>\n",
       "      <th>precip_intensity</th>\n",
       "      <th>precip_probability</th>\n",
       "      <th>visibility</th>\n",
       "      <th>icon</th>\n",
       "      <th>summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017-10-01 00:00:00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>19.46</td>\n",
       "      <td>308.00</td>\n",
       "      <td>0.83</td>\n",
       "      <td>4.25</td>\n",
       "      <td>1007.90</td>\n",
       "      <td>1.87</td>\n",
       "      <td>274.46</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.02</td>\n",
       "      <td>3.57</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017-10-01 01:00:00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>20.06</td>\n",
       "      <td>308.00</td>\n",
       "      <td>0.83</td>\n",
       "      <td>4.25</td>\n",
       "      <td>1007.90</td>\n",
       "      <td>1.87</td>\n",
       "      <td>274.46</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.02</td>\n",
       "      <td>3.06</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2017-10-01 02:00:00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>19.97</td>\n",
       "      <td>308.00</td>\n",
       "      <td>0.83</td>\n",
       "      <td>4.25</td>\n",
       "      <td>1007.90</td>\n",
       "      <td>1.87</td>\n",
       "      <td>274.46</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.02</td>\n",
       "      <td>3.06</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2017-10-01 03:00:00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>18.66</td>\n",
       "      <td>270.00</td>\n",
       "      <td>0.45</td>\n",
       "      <td>4.25</td>\n",
       "      <td>1008.12</td>\n",
       "      <td>1.87</td>\n",
       "      <td>274.46</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.02</td>\n",
       "      <td>3.56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2017-10-01 04:00:00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>17.84</td>\n",
       "      <td>270.00</td>\n",
       "      <td>0.45</td>\n",
       "      <td>4.25</td>\n",
       "      <td>1008.12</td>\n",
       "      <td>1.87</td>\n",
       "      <td>274.46</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.02</td>\n",
       "      <td>3.06</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13614</th>\n",
       "      <td>2019-09-30 19:00:00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.16</td>\n",
       "      <td>1.00</td>\n",
       "      <td>21.19</td>\n",
       "      <td>40.00</td>\n",
       "      <td>2.72</td>\n",
       "      <td>4.59</td>\n",
       "      <td>1007.44</td>\n",
       "      <td>0.00</td>\n",
       "      <td>269.60</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13615</th>\n",
       "      <td>2019-09-30 20:00:00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.16</td>\n",
       "      <td>1.00</td>\n",
       "      <td>21.35</td>\n",
       "      <td>59.00</td>\n",
       "      <td>3.24</td>\n",
       "      <td>5.45</td>\n",
       "      <td>1008.25</td>\n",
       "      <td>0.00</td>\n",
       "      <td>268.70</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13616</th>\n",
       "      <td>2019-09-30 21:00:00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.18</td>\n",
       "      <td>1.00</td>\n",
       "      <td>21.61</td>\n",
       "      <td>76.00</td>\n",
       "      <td>3.34</td>\n",
       "      <td>5.83</td>\n",
       "      <td>1008.65</td>\n",
       "      <td>0.00</td>\n",
       "      <td>268.40</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13617</th>\n",
       "      <td>2019-09-30 22:00:00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.30</td>\n",
       "      <td>1.00</td>\n",
       "      <td>21.99</td>\n",
       "      <td>88.00</td>\n",
       "      <td>3.22</td>\n",
       "      <td>5.73</td>\n",
       "      <td>1008.76</td>\n",
       "      <td>0.00</td>\n",
       "      <td>268.70</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13618</th>\n",
       "      <td>2019-09-30 23:00:00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.35</td>\n",
       "      <td>1.00</td>\n",
       "      <td>22.09</td>\n",
       "      <td>95.00</td>\n",
       "      <td>3.14</td>\n",
       "      <td>6.03</td>\n",
       "      <td>1008.61</td>\n",
       "      <td>0.00</td>\n",
       "      <td>268.70</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11735 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            datetime_local  power  cloud_cover  humidity  dew_point  \\\n",
       "0      2017-10-01 00:00:00   0.00         0.00      1.00      19.46   \n",
       "1      2017-10-01 01:00:00   0.00         0.00      1.00      20.06   \n",
       "2      2017-10-01 02:00:00   0.00         0.00      1.00      19.97   \n",
       "3      2017-10-01 03:00:00   0.00         0.00      1.00      18.66   \n",
       "4      2017-10-01 04:00:00   0.00         0.00      1.00      17.84   \n",
       "...                    ...    ...          ...       ...        ...   \n",
       "13614  2019-09-30 19:00:00   0.00         0.16      1.00      21.19   \n",
       "13615  2019-09-30 20:00:00   0.00         0.16      1.00      21.35   \n",
       "13616  2019-09-30 21:00:00   0.00         0.18      1.00      21.61   \n",
       "13617  2019-09-30 22:00:00   0.00         0.30      1.00      21.99   \n",
       "13618  2019-09-30 23:00:00   0.00         0.35      1.00      22.09   \n",
       "\n",
       "       wind_bearing  wind_speed  wind_gust  pressure  uv_index  ozone  \\\n",
       "0            308.00        0.83       4.25   1007.90      1.87 274.46   \n",
       "1            308.00        0.83       4.25   1007.90      1.87 274.46   \n",
       "2            308.00        0.83       4.25   1007.90      1.87 274.46   \n",
       "3            270.00        0.45       4.25   1008.12      1.87 274.46   \n",
       "4            270.00        0.45       4.25   1008.12      1.87 274.46   \n",
       "...             ...         ...        ...       ...       ...    ...   \n",
       "13614         40.00        2.72       4.59   1007.44      0.00 269.60   \n",
       "13615         59.00        3.24       5.45   1008.25      0.00 268.70   \n",
       "13616         76.00        3.34       5.83   1008.65      0.00 268.40   \n",
       "13617         88.00        3.22       5.73   1008.76      0.00 268.70   \n",
       "13618         95.00        3.14       6.03   1008.61      0.00 268.70   \n",
       "\n",
       "       precip_intensity  precip_probability  visibility  icon  summary  \n",
       "0                  0.06                0.02        3.57     1        1  \n",
       "1                  0.06                0.02        3.06     3        2  \n",
       "2                  0.06                0.02        3.06     3        2  \n",
       "3                  0.06                0.02        3.56     1        1  \n",
       "4                  0.06                0.02        3.06     3        2  \n",
       "...                 ...                 ...         ...   ...      ...  \n",
       "13614              0.00                0.00       16.09     5       12  \n",
       "13615              0.00                0.00       16.09     5       12  \n",
       "13616              0.00                0.00       16.09     5       12  \n",
       "13617              0.00                0.00       16.09     5       12  \n",
       "13618              0.00                0.00       16.09     5       12  \n",
       "\n",
       "[11735 rows x 16 columns]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_l1 = df_l[~((df_l < (Q1 - 3 * IQR)) |(df_l > (Q3 + 3 * IQR))).any(axis=1)]\n",
    "df_l1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_l1.set_index(['datetime_local'])\n",
    "X = df_l1.iloc[:, 2:]\n",
    "y = df_l1.iloc[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Absolute Error: 1.2949536961128425\n",
      "Mean Squared Error: 5.703331091708171\n",
      "Root Mean Squared Error: 2.388164795760161\n",
      "R score of test data ---> 0.7482466615049699\n",
      "R score of train data---> 0.9615221521783136\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)\n",
    "\n",
    "# Feature Scaling\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "sc = StandardScaler()\n",
    "X_train = sc.fit_transform(X_train)\n",
    "X_test = sc.transform(X_test)\n",
    "\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "regressor = RandomForestRegressor(n_estimators=50, random_state=0)\n",
    "regressor.fit(X_train, y_train)\n",
    "y_pred = regressor.predict(X_test)\n",
    "y_pred_train = regressor.predict(X_train)\n",
    "\n",
    "\n",
    "from sklearn import metrics\n",
    "\n",
    "print('Mean Absolute Error:', metrics.mean_absolute_error(y_test, y_pred))\n",
    "print('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred))\n",
    "print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))\n",
    "print(\"R score of test data --->\",metrics.r2_score(y_test, y_pred))\n",
    "print(\"R score of train data--->\",metrics.r2_score(y_train, y_pred_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trying Lazypredict library to get more idea about how the model is performing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████| 39/39 [02:02<00:00,  3.15s/it]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>R-Squared</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>Time Taken</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ExtraTreesRegressor</th>\n",
       "      <td>0.76</td>\n",
       "      <td>2.35</td>\n",
       "      <td>4.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGBRegressor</th>\n",
       "      <td>0.75</td>\n",
       "      <td>2.38</td>\n",
       "      <td>0.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RandomForestRegressor</th>\n",
       "      <td>0.75</td>\n",
       "      <td>2.39</td>\n",
       "      <td>8.71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LGBMRegressor</th>\n",
       "      <td>0.74</td>\n",
       "      <td>2.43</td>\n",
       "      <td>0.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HistGradientBoostingRegressor</th>\n",
       "      <td>0.74</td>\n",
       "      <td>2.45</td>\n",
       "      <td>1.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BaggingRegressor</th>\n",
       "      <td>0.72</td>\n",
       "      <td>2.52</td>\n",
       "      <td>0.89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MLPRegressor</th>\n",
       "      <td>0.68</td>\n",
       "      <td>2.69</td>\n",
       "      <td>24.66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KNeighborsRegressor</th>\n",
       "      <td>0.68</td>\n",
       "      <td>2.69</td>\n",
       "      <td>0.48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GradientBoostingRegressor</th>\n",
       "      <td>0.68</td>\n",
       "      <td>2.69</td>\n",
       "      <td>3.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NuSVR</th>\n",
       "      <td>0.63</td>\n",
       "      <td>2.88</td>\n",
       "      <td>7.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SVR</th>\n",
       "      <td>0.63</td>\n",
       "      <td>2.90</td>\n",
       "      <td>6.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SGDRegressor</th>\n",
       "      <td>0.56</td>\n",
       "      <td>3.17</td>\n",
       "      <td>0.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LinearRegression</th>\n",
       "      <td>0.56</td>\n",
       "      <td>3.17</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TransformedTargetRegressor</th>\n",
       "      <td>0.56</td>\n",
       "      <td>3.17</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lars</th>\n",
       "      <td>0.56</td>\n",
       "      <td>3.17</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ridge</th>\n",
       "      <td>0.56</td>\n",
       "      <td>3.17</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RidgeCV</th>\n",
       "      <td>0.56</td>\n",
       "      <td>3.17</td>\n",
       "      <td>0.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BayesianRidge</th>\n",
       "      <td>0.56</td>\n",
       "      <td>3.17</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LassoCV</th>\n",
       "      <td>0.56</td>\n",
       "      <td>3.17</td>\n",
       "      <td>0.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LarsCV</th>\n",
       "      <td>0.56</td>\n",
       "      <td>3.17</td>\n",
       "      <td>0.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LassoLarsCV</th>\n",
       "      <td>0.56</td>\n",
       "      <td>3.17</td>\n",
       "      <td>0.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ElasticNetCV</th>\n",
       "      <td>0.56</td>\n",
       "      <td>3.17</td>\n",
       "      <td>0.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LassoLarsIC</th>\n",
       "      <td>0.55</td>\n",
       "      <td>3.18</td>\n",
       "      <td>0.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>OrthogonalMatchingPursuitCV</th>\n",
       "      <td>0.55</td>\n",
       "      <td>3.21</td>\n",
       "      <td>0.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AdaBoostRegressor</th>\n",
       "      <td>0.53</td>\n",
       "      <td>3.25</td>\n",
       "      <td>0.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DecisionTreeRegressor</th>\n",
       "      <td>0.50</td>\n",
       "      <td>3.35</td>\n",
       "      <td>0.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ExtraTreeRegressor</th>\n",
       "      <td>0.50</td>\n",
       "      <td>3.36</td>\n",
       "      <td>0.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HuberRegressor</th>\n",
       "      <td>0.48</td>\n",
       "      <td>3.44</td>\n",
       "      <td>0.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>OrthogonalMatchingPursuit</th>\n",
       "      <td>0.47</td>\n",
       "      <td>3.45</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lasso</th>\n",
       "      <td>0.42</td>\n",
       "      <td>3.63</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ElasticNet</th>\n",
       "      <td>0.40</td>\n",
       "      <td>3.69</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LinearSVR</th>\n",
       "      <td>0.39</td>\n",
       "      <td>3.70</td>\n",
       "      <td>0.81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PassiveAggressiveRegressor</th>\n",
       "      <td>0.30</td>\n",
       "      <td>3.97</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KernelRidge</th>\n",
       "      <td>0.16</td>\n",
       "      <td>4.36</td>\n",
       "      <td>18.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DummyRegressor</th>\n",
       "      <td>-0.00</td>\n",
       "      <td>4.76</td>\n",
       "      <td>0.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LassoLars</th>\n",
       "      <td>-0.00</td>\n",
       "      <td>4.76</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GaussianProcessRegressor</th>\n",
       "      <td>-483.34</td>\n",
       "      <td>104.75</td>\n",
       "      <td>42.69</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               R-Squared   RMSE  Time Taken\n",
       "Model                                                      \n",
       "ExtraTreesRegressor                 0.76   2.35        4.03\n",
       "XGBRegressor                        0.75   2.38        0.78\n",
       "RandomForestRegressor               0.75   2.39        8.71\n",
       "LGBMRegressor                       0.74   2.43        0.30\n",
       "HistGradientBoostingRegressor       0.74   2.45        1.64\n",
       "BaggingRegressor                    0.72   2.52        0.89\n",
       "MLPRegressor                        0.68   2.69       24.66\n",
       "KNeighborsRegressor                 0.68   2.69        0.48\n",
       "GradientBoostingRegressor           0.68   2.69        3.35\n",
       "NuSVR                               0.63   2.88        7.01\n",
       "SVR                                 0.63   2.90        6.01\n",
       "SGDRegressor                        0.56   3.17        0.07\n",
       "LinearRegression                    0.56   3.17        0.04\n",
       "TransformedTargetRegressor          0.56   3.17        0.04\n",
       "Lars                                0.56   3.17        0.05\n",
       "Ridge                               0.56   3.17        0.05\n",
       "RidgeCV                             0.56   3.17        0.06\n",
       "BayesianRidge                       0.56   3.17        0.04\n",
       "LassoCV                             0.56   3.17        0.47\n",
       "LarsCV                              0.56   3.17        0.17\n",
       "LassoLarsCV                         0.56   3.17        0.19\n",
       "ElasticNetCV                        0.56   3.17        0.38\n",
       "LassoLarsIC                         0.55   3.18        0.06\n",
       "OrthogonalMatchingPursuitCV         0.55   3.21        0.08\n",
       "AdaBoostRegressor                   0.53   3.25        0.47\n",
       "DecisionTreeRegressor               0.50   3.35        0.17\n",
       "ExtraTreeRegressor                  0.50   3.36        0.08\n",
       "HuberRegressor                      0.48   3.44        0.13\n",
       "OrthogonalMatchingPursuit           0.47   3.45        0.04\n",
       "Lasso                               0.42   3.63        0.05\n",
       "ElasticNet                          0.40   3.69        0.04\n",
       "LinearSVR                           0.39   3.70        0.81\n",
       "PassiveAggressiveRegressor          0.30   3.97        0.05\n",
       "KernelRidge                         0.16   4.36       18.43\n",
       "DummyRegressor                     -0.00   4.76        0.03\n",
       "LassoLars                          -0.00   4.76        0.05\n",
       "GaussianProcessRegressor         -483.34 104.75       42.69"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from lazypredict.Supervised import LazyRegressor\n",
    "\n",
    "\n",
    "reg = LazyRegressor(verbose=0,ignore_warnings=True, custom_metric=None)\n",
    "models,predictions = reg.fit(X_train, X_test, y_train, y_test)\n",
    "models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now as it seems that the model is working fine so apllying the model on conplete weather_actual data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['datetime_local', 'power', 'cloud_cover', 'humidity', 'dew_point',\n",
       "       'wind_bearing', 'wind_speed', 'wind_gust', 'pressure', 'uv_index',\n",
       "       'ozone', 'precip_intensity', 'precip_probability', 'visibility', 'icon',\n",
       "       'summary'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df_l1.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = df_weather_forecast.drop(['plant_id','wind_chill','heat_index','qpf','snow','pop','fctcode','precip_accumulation',\n",
    "                         'datetime_utc', 'sunrise', 'sunset','updated_at','apparent_temperature','temperature','precip_type','Unnamed: 0'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cloud_cover</th>\n",
       "      <th>humidity</th>\n",
       "      <th>dew_point</th>\n",
       "      <th>wind_bearing</th>\n",
       "      <th>wind_speed</th>\n",
       "      <th>wind_gust</th>\n",
       "      <th>pressure</th>\n",
       "      <th>uv_index</th>\n",
       "      <th>ozone</th>\n",
       "      <th>precip_intensity</th>\n",
       "      <th>precip_probability</th>\n",
       "      <th>visibility</th>\n",
       "      <th>icon</th>\n",
       "      <th>summary</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>datetime_local</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-10-01 00:00:00</th>\n",
       "      <td>0.35</td>\n",
       "      <td>1</td>\n",
       "      <td>22.01</td>\n",
       "      <td>101</td>\n",
       "      <td>3.08</td>\n",
       "      <td>6.42</td>\n",
       "      <td>1008.45</td>\n",
       "      <td>0</td>\n",
       "      <td>268.60</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-01 01:00:00</th>\n",
       "      <td>0.43</td>\n",
       "      <td>1</td>\n",
       "      <td>22.06</td>\n",
       "      <td>105</td>\n",
       "      <td>2.94</td>\n",
       "      <td>6.28</td>\n",
       "      <td>1008.24</td>\n",
       "      <td>0</td>\n",
       "      <td>269.10</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-01 02:00:00</th>\n",
       "      <td>0.48</td>\n",
       "      <td>1</td>\n",
       "      <td>22.06</td>\n",
       "      <td>106</td>\n",
       "      <td>2.82</td>\n",
       "      <td>6.08</td>\n",
       "      <td>1007.90</td>\n",
       "      <td>0</td>\n",
       "      <td>269.70</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-01 03:00:00</th>\n",
       "      <td>0.49</td>\n",
       "      <td>1</td>\n",
       "      <td>21.87</td>\n",
       "      <td>106</td>\n",
       "      <td>2.63</td>\n",
       "      <td>5.74</td>\n",
       "      <td>1007.61</td>\n",
       "      <td>0</td>\n",
       "      <td>270.40</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-01 04:00:00</th>\n",
       "      <td>0.49</td>\n",
       "      <td>1</td>\n",
       "      <td>21.53</td>\n",
       "      <td>100</td>\n",
       "      <td>2.39</td>\n",
       "      <td>5.30</td>\n",
       "      <td>1007.51</td>\n",
       "      <td>0</td>\n",
       "      <td>271.30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-27 19:00:00</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>12.20</td>\n",
       "      <td>345</td>\n",
       "      <td>1.74</td>\n",
       "      <td>2.85</td>\n",
       "      <td>1011.70</td>\n",
       "      <td>0</td>\n",
       "      <td>264.90</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-27 20:00:00</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>12.06</td>\n",
       "      <td>353</td>\n",
       "      <td>1.39</td>\n",
       "      <td>2.73</td>\n",
       "      <td>1012.28</td>\n",
       "      <td>0</td>\n",
       "      <td>264.90</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-27 21:00:00</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>12.19</td>\n",
       "      <td>29</td>\n",
       "      <td>1.15</td>\n",
       "      <td>2.55</td>\n",
       "      <td>1012.67</td>\n",
       "      <td>0</td>\n",
       "      <td>264.60</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-27 22:00:00</th>\n",
       "      <td>0.00</td>\n",
       "      <td>1</td>\n",
       "      <td>12.50</td>\n",
       "      <td>92</td>\n",
       "      <td>1.03</td>\n",
       "      <td>2.33</td>\n",
       "      <td>1012.79</td>\n",
       "      <td>0</td>\n",
       "      <td>264.30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-27 23:00:00</th>\n",
       "      <td>0.00</td>\n",
       "      <td>1</td>\n",
       "      <td>12.67</td>\n",
       "      <td>146</td>\n",
       "      <td>0.91</td>\n",
       "      <td>2.01</td>\n",
       "      <td>1012.85</td>\n",
       "      <td>0</td>\n",
       "      <td>262.80</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>648 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     cloud_cover  humidity  dew_point  wind_bearing  \\\n",
       "datetime_local                                                        \n",
       "2019-10-01 00:00:00         0.35         1      22.01           101   \n",
       "2019-10-01 01:00:00         0.43         1      22.06           105   \n",
       "2019-10-01 02:00:00         0.48         1      22.06           106   \n",
       "2019-10-01 03:00:00         0.49         1      21.87           106   \n",
       "2019-10-01 04:00:00         0.49         1      21.53           100   \n",
       "...                          ...       ...        ...           ...   \n",
       "2019-10-27 19:00:00         0.00         0      12.20           345   \n",
       "2019-10-27 20:00:00         0.00         0      12.06           353   \n",
       "2019-10-27 21:00:00         0.00         0      12.19            29   \n",
       "2019-10-27 22:00:00         0.00         1      12.50            92   \n",
       "2019-10-27 23:00:00         0.00         1      12.67           146   \n",
       "\n",
       "                     wind_speed  wind_gust  pressure  uv_index  ozone  \\\n",
       "datetime_local                                                          \n",
       "2019-10-01 00:00:00        3.08       6.42   1008.45         0 268.60   \n",
       "2019-10-01 01:00:00        2.94       6.28   1008.24         0 269.10   \n",
       "2019-10-01 02:00:00        2.82       6.08   1007.90         0 269.70   \n",
       "2019-10-01 03:00:00        2.63       5.74   1007.61         0 270.40   \n",
       "2019-10-01 04:00:00        2.39       5.30   1007.51         0 271.30   \n",
       "...                         ...        ...       ...       ...    ...   \n",
       "2019-10-27 19:00:00        1.74       2.85   1011.70         0 264.90   \n",
       "2019-10-27 20:00:00        1.39       2.73   1012.28         0 264.90   \n",
       "2019-10-27 21:00:00        1.15       2.55   1012.67         0 264.60   \n",
       "2019-10-27 22:00:00        1.03       2.33   1012.79         0 264.30   \n",
       "2019-10-27 23:00:00        0.91       2.01   1012.85         0 262.80   \n",
       "\n",
       "                     precip_intensity  precip_probability  visibility  icon  \\\n",
       "datetime_local                                                                \n",
       "2019-10-01 00:00:00              0.00                0.00       16.09     4   \n",
       "2019-10-01 01:00:00              0.00                0.00       16.09     4   \n",
       "2019-10-01 02:00:00              0.00                0.00       16.09     4   \n",
       "2019-10-01 03:00:00              0.00                0.00       16.09     4   \n",
       "2019-10-01 04:00:00              0.00                0.00       16.09     4   \n",
       "...                               ...                 ...         ...   ...   \n",
       "2019-10-27 19:00:00              0.00                0.00       16.09     1   \n",
       "2019-10-27 20:00:00              0.00                0.00       16.09     1   \n",
       "2019-10-27 21:00:00              0.00                0.00       16.09     1   \n",
       "2019-10-27 22:00:00              0.00                0.00       16.09     1   \n",
       "2019-10-27 23:00:00              0.00                0.00       16.09     1   \n",
       "\n",
       "                     summary  \n",
       "datetime_local                \n",
       "2019-10-01 00:00:00        4  \n",
       "2019-10-01 01:00:00        4  \n",
       "2019-10-01 02:00:00        4  \n",
       "2019-10-01 03:00:00        4  \n",
       "2019-10-01 04:00:00        4  \n",
       "...                      ...  \n",
       "2019-10-27 19:00:00        0  \n",
       "2019-10-27 20:00:00        0  \n",
       "2019-10-27 21:00:00        0  \n",
       "2019-10-27 22:00:00        0  \n",
       "2019-10-27 23:00:00        0  \n",
       "\n",
       "[648 rows x 14 columns]"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = test.set_index(['datetime_local'])\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "labelencoder = LabelEncoder()\n",
    "test['icon'] = labelencoder.fit_transform(test['icon'])\n",
    "test['summary'] = labelencoder.fit_transform(test['summary'])\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11735, 14) (11735,) (648, 14)\n"
     ]
    }
   ],
   "source": [
    "X_train = df_l1.iloc[:, 2:]\n",
    "y_train = df_l1.iloc[:, 1]\n",
    "X_test = test.iloc[:, 0:]\n",
    "print(X_train.shape,y_train.shape,X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cloud_cover</th>\n",
       "      <th>humidity</th>\n",
       "      <th>dew_point</th>\n",
       "      <th>wind_bearing</th>\n",
       "      <th>wind_speed</th>\n",
       "      <th>wind_gust</th>\n",
       "      <th>pressure</th>\n",
       "      <th>uv_index</th>\n",
       "      <th>ozone</th>\n",
       "      <th>precip_intensity</th>\n",
       "      <th>precip_probability</th>\n",
       "      <th>visibility</th>\n",
       "      <th>icon</th>\n",
       "      <th>summary</th>\n",
       "      <th>Output</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>datetime_local</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-10-01 00:00:00</th>\n",
       "      <td>0.35</td>\n",
       "      <td>1</td>\n",
       "      <td>22.01</td>\n",
       "      <td>101</td>\n",
       "      <td>3.08</td>\n",
       "      <td>6.42</td>\n",
       "      <td>1008.45</td>\n",
       "      <td>0</td>\n",
       "      <td>268.60</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-01 01:00:00</th>\n",
       "      <td>0.43</td>\n",
       "      <td>1</td>\n",
       "      <td>22.06</td>\n",
       "      <td>105</td>\n",
       "      <td>2.94</td>\n",
       "      <td>6.28</td>\n",
       "      <td>1008.24</td>\n",
       "      <td>0</td>\n",
       "      <td>269.10</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-01 02:00:00</th>\n",
       "      <td>0.48</td>\n",
       "      <td>1</td>\n",
       "      <td>22.06</td>\n",
       "      <td>106</td>\n",
       "      <td>2.82</td>\n",
       "      <td>6.08</td>\n",
       "      <td>1007.90</td>\n",
       "      <td>0</td>\n",
       "      <td>269.70</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-01 03:00:00</th>\n",
       "      <td>0.49</td>\n",
       "      <td>1</td>\n",
       "      <td>21.87</td>\n",
       "      <td>106</td>\n",
       "      <td>2.63</td>\n",
       "      <td>5.74</td>\n",
       "      <td>1007.61</td>\n",
       "      <td>0</td>\n",
       "      <td>270.40</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-01 04:00:00</th>\n",
       "      <td>0.49</td>\n",
       "      <td>1</td>\n",
       "      <td>21.53</td>\n",
       "      <td>100</td>\n",
       "      <td>2.39</td>\n",
       "      <td>5.30</td>\n",
       "      <td>1007.51</td>\n",
       "      <td>0</td>\n",
       "      <td>271.30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-27 19:00:00</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>12.20</td>\n",
       "      <td>345</td>\n",
       "      <td>1.74</td>\n",
       "      <td>2.85</td>\n",
       "      <td>1011.70</td>\n",
       "      <td>0</td>\n",
       "      <td>264.90</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-27 20:00:00</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>12.06</td>\n",
       "      <td>353</td>\n",
       "      <td>1.39</td>\n",
       "      <td>2.73</td>\n",
       "      <td>1012.28</td>\n",
       "      <td>0</td>\n",
       "      <td>264.90</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-27 21:00:00</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>12.19</td>\n",
       "      <td>29</td>\n",
       "      <td>1.15</td>\n",
       "      <td>2.55</td>\n",
       "      <td>1012.67</td>\n",
       "      <td>0</td>\n",
       "      <td>264.60</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-27 22:00:00</th>\n",
       "      <td>0.00</td>\n",
       "      <td>1</td>\n",
       "      <td>12.50</td>\n",
       "      <td>92</td>\n",
       "      <td>1.03</td>\n",
       "      <td>2.33</td>\n",
       "      <td>1012.79</td>\n",
       "      <td>0</td>\n",
       "      <td>264.30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-27 23:00:00</th>\n",
       "      <td>0.00</td>\n",
       "      <td>1</td>\n",
       "      <td>12.67</td>\n",
       "      <td>146</td>\n",
       "      <td>0.91</td>\n",
       "      <td>2.01</td>\n",
       "      <td>1012.85</td>\n",
       "      <td>0</td>\n",
       "      <td>262.80</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>16.09</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>648 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     cloud_cover  humidity  dew_point  wind_bearing  \\\n",
       "datetime_local                                                        \n",
       "2019-10-01 00:00:00         0.35         1      22.01           101   \n",
       "2019-10-01 01:00:00         0.43         1      22.06           105   \n",
       "2019-10-01 02:00:00         0.48         1      22.06           106   \n",
       "2019-10-01 03:00:00         0.49         1      21.87           106   \n",
       "2019-10-01 04:00:00         0.49         1      21.53           100   \n",
       "...                          ...       ...        ...           ...   \n",
       "2019-10-27 19:00:00         0.00         0      12.20           345   \n",
       "2019-10-27 20:00:00         0.00         0      12.06           353   \n",
       "2019-10-27 21:00:00         0.00         0      12.19            29   \n",
       "2019-10-27 22:00:00         0.00         1      12.50            92   \n",
       "2019-10-27 23:00:00         0.00         1      12.67           146   \n",
       "\n",
       "                     wind_speed  wind_gust  pressure  uv_index  ozone  \\\n",
       "datetime_local                                                          \n",
       "2019-10-01 00:00:00        3.08       6.42   1008.45         0 268.60   \n",
       "2019-10-01 01:00:00        2.94       6.28   1008.24         0 269.10   \n",
       "2019-10-01 02:00:00        2.82       6.08   1007.90         0 269.70   \n",
       "2019-10-01 03:00:00        2.63       5.74   1007.61         0 270.40   \n",
       "2019-10-01 04:00:00        2.39       5.30   1007.51         0 271.30   \n",
       "...                         ...        ...       ...       ...    ...   \n",
       "2019-10-27 19:00:00        1.74       2.85   1011.70         0 264.90   \n",
       "2019-10-27 20:00:00        1.39       2.73   1012.28         0 264.90   \n",
       "2019-10-27 21:00:00        1.15       2.55   1012.67         0 264.60   \n",
       "2019-10-27 22:00:00        1.03       2.33   1012.79         0 264.30   \n",
       "2019-10-27 23:00:00        0.91       2.01   1012.85         0 262.80   \n",
       "\n",
       "                     precip_intensity  precip_probability  visibility  icon  \\\n",
       "datetime_local                                                                \n",
       "2019-10-01 00:00:00              0.00                0.00       16.09     4   \n",
       "2019-10-01 01:00:00              0.00                0.00       16.09     4   \n",
       "2019-10-01 02:00:00              0.00                0.00       16.09     4   \n",
       "2019-10-01 03:00:00              0.00                0.00       16.09     4   \n",
       "2019-10-01 04:00:00              0.00                0.00       16.09     4   \n",
       "...                               ...                 ...         ...   ...   \n",
       "2019-10-27 19:00:00              0.00                0.00       16.09     1   \n",
       "2019-10-27 20:00:00              0.00                0.00       16.09     1   \n",
       "2019-10-27 21:00:00              0.00                0.00       16.09     1   \n",
       "2019-10-27 22:00:00              0.00                0.00       16.09     1   \n",
       "2019-10-27 23:00:00              0.00                0.00       16.09     1   \n",
       "\n",
       "                     summary  Output  \n",
       "datetime_local                        \n",
       "2019-10-01 00:00:00        4    0.67  \n",
       "2019-10-01 01:00:00        4    0.68  \n",
       "2019-10-01 02:00:00        4    0.70  \n",
       "2019-10-01 03:00:00        4    0.93  \n",
       "2019-10-01 04:00:00        4    0.70  \n",
       "...                      ...     ...  \n",
       "2019-10-27 19:00:00        0    0.00  \n",
       "2019-10-27 20:00:00        0    0.00  \n",
       "2019-10-27 21:00:00        0    0.00  \n",
       "2019-10-27 22:00:00        0    0.00  \n",
       "2019-10-27 23:00:00        0    0.00  \n",
       "\n",
       "[648 rows x 15 columns]"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "# Feature Scaling\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "sc = StandardScaler()\n",
    "X_train = sc.fit_transform(X_train)\n",
    "X_test = sc.transform(X_test)\n",
    "y_train = df_l1.iloc[:, 1]\n",
    "\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "regressor = RandomForestRegressor(n_estimators=50, random_state=0)\n",
    "regressor.fit(X_train, y_train)\n",
    "y_pred_test = regressor.predict(X_test)\n",
    "\n",
    "test['Output'] = y_pred_test\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "test.to_csv('Submission_output.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R score of train data---> 0.9636861291030345\n"
     ]
    }
   ],
   "source": [
    "y_pred_train = regressor.predict(X_train)\n",
    "print(\"R score of train data--->\",metrics.r2_score(y_train, y_pred_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}