time_series_basics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pylab as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Essential Components of Time-Series Analysis\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the data\n",
    "\n",
    "car_sales_data = pd.read_csv(r\"D:\\time_series_session\\sales_cars.csv\")\n",
    "sample_pm_data = pd.read_csv(r'D:/time_series_session/sample_ctl.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Month    object\n",
       "Sales     int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check for the Columns & Also their Data type\n",
    "#car_sales_data[:10]\n",
    "car_sales_data.columns\n",
    "car_sales_data.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Month</th>\n",
       "      <th>Sales</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2016-01</td>\n",
       "      <td>266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2016-02</td>\n",
       "      <td>146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2016-03</td>\n",
       "      <td>183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2016-04</td>\n",
       "      <td>119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2016-05</td>\n",
       "      <td>180</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Month  Sales\n",
       "0  2016-01    266\n",
       "1  2016-02    146\n",
       "2  2016-03    183\n",
       "3  2016-04    119\n",
       "4  2016-05    180"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the Sample Data\n",
    "\n",
    "car_sales_data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RangeIndex(start=0, stop=36, step=1)"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check what is the Current Index (By default sequential numbers from 0 - [length-1] )\n",
    "\n",
    "car_sales_data.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Simply remove data present in the rows 1, 2, 3\n",
    "\n",
    "car_sales_data.drop([1,2,3], axis=0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert a specific Column to DateTime\n",
    "\n",
    "#car_sales_data[:10]\n",
    "#car_sales_data.drop(columns=['level_0'], inplace=True)\n",
    "car_sales_data['Month'] = pd.to_datetime(car_sales_data['Month'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Month    datetime64[ns]\n",
       "Sales             int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check for the Column Type again\n",
    "\n",
    "car_sales_data.dtypes\n",
    "#car_sales_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the Index -- Converted Datetime column\n",
    "\n",
    "car_sales_data.set_index('Month', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatetimeIndex(['2016-01-01', '2016-05-01', '2016-06-01', '2016-07-01',\n",
       "               '2016-08-01'],\n",
       "              dtype='datetime64[ns]', name='Month', freq=None)"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Check for the Initial 5 rows of Data\n",
    "\n",
    "car_sales_data.index[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Int64Index([ 1,  5,  6,  7,  8,  9, 10, 11, 12,  1,  2,  3,  4,  5,  6,  7,  8,\n",
       "             9, 10, 11, 12,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12],\n",
       "           dtype='int64', name='Month')"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Datetime Index has options for Month, Week, Day of the Week, Hour of the day\n",
    "\n",
    "car_sales_data.index.month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Evaluate the Length of the TIMESTAMP INDEX to check if that is the expected length\n",
    "\n",
    "#max(car_sales_data.index)\n",
    "#idx = pd.date_range(min(car_sales_data.index), max(car_sales_data.index), freq='MS')\n",
    "#len(idx)\n",
    "\n",
    "len(car_sales_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sales</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Month</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2016-01-01</th>\n",
       "      <td>266.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-02-01</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-03-01</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-04-01</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-05-01</th>\n",
       "      <td>180.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-06-01</th>\n",
       "      <td>169.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-07-01</th>\n",
       "      <td>232.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-08-01</th>\n",
       "      <td>225.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-09-01</th>\n",
       "      <td>193.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-10-01</th>\n",
       "      <td>123.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Sales\n",
       "Month            \n",
       "2016-01-01  266.0\n",
       "2016-02-01    NaN\n",
       "2016-03-01    NaN\n",
       "2016-04-01    NaN\n",
       "2016-05-01  180.0\n",
       "2016-06-01  169.0\n",
       "2016-07-01  232.0\n",
       "2016-08-01  225.0\n",
       "2016-09-01  193.0\n",
       "2016-10-01  123.0"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Try filling the missing values (in the TIMESTAMP COLUMN) for Car Sales Data\n",
    "\n",
    "car_sales_data = car_sales_data.resample('MS').mean()\n",
    "car_sales_data[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sales</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Month</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2016-01-01</th>\n",
       "      <td>266.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-02-01</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-03-01</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-04-01</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-05-01</th>\n",
       "      <td>180.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-06-01</th>\n",
       "      <td>169.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-07-01</th>\n",
       "      <td>232.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-08-01</th>\n",
       "      <td>225.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-09-01</th>\n",
       "      <td>193.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-10-01</th>\n",
       "      <td>123.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Sales\n",
       "Month            \n",
       "2016-01-01  266.0\n",
       "2016-02-01    0.0\n",
       "2016-03-01    0.0\n",
       "2016-04-01    0.0\n",
       "2016-05-01  180.0\n",
       "2016-06-01  169.0\n",
       "2016-07-01  232.0\n",
       "2016-08-01  225.0\n",
       "2016-09-01  193.0\n",
       "2016-10-01  123.0"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Efficient Methods to fill the Missing Values\n",
    "\n",
    "#car_sales_data.fillna(0, inplace=True)\n",
    "#car_sales_data.fillna(method='bfill')[:10]\n",
    "sample_data = car_sales_data.interpolate(method='linear', order=2 )\n",
    "#car_sales_data[:10]\n",
    "sample_data[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check for Sample PM data\n",
    "\n",
    "sample_pm_data['pm_collectiontime'] = pd.to_datetime(sample_pm_data['pm_collectiontime'])\n",
    "sample_pm_data.set_index('pm_collectiontime', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Checking for the missing length & filling with the approx value\n",
    "\n",
    "test_idx = pd.date_range(min(sample_pm_data.index), max(sample_pm_data.index), freq='5T')\n",
    "len_diff = len(test_idx) - len(sample_pm_data)\n",
    "len_diff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sampling the Data for '5 min interval'\n",
    "\n",
    "sample_pm_data = sample_pm_data.resample('5T').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Int64Index([17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,\n",
       "            18, 18, 19, 19, 19],\n",
       "           dtype='int64', name='pm_collectiontime')"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get New Features from the Timestamped Index\n",
    "\n",
    "#sample_pm_data.index.month\n",
    "sample_pm_data.index.hour\n",
    "#sample_pm_data.index.day\n",
    "#sample_pm_data.index.dayofweek\n",
    "#sample_pm_data.index.dayofyear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Resampling the Data to 1-min Interval\n",
    "\n",
    "sample_pm_data_1min = sample_pm_data.resample('1T').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pm_jitteratoz</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pm_collectiontime</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:25:00+00:00</th>\n",
       "      <td>114.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:26:00+00:00</th>\n",
       "      <td>123.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:27:00+00:00</th>\n",
       "      <td>133.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:28:00+00:00</th>\n",
       "      <td>143.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:29:00+00:00</th>\n",
       "      <td>153.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:30:00+00:00</th>\n",
       "      <td>163.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:31:00+00:00</th>\n",
       "      <td>150.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:32:00+00:00</th>\n",
       "      <td>138.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:33:00+00:00</th>\n",
       "      <td>125.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:34:00+00:00</th>\n",
       "      <td>113.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:35:00+00:00</th>\n",
       "      <td>101.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:36:00+00:00</th>\n",
       "      <td>111.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:37:00+00:00</th>\n",
       "      <td>121.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:38:00+00:00</th>\n",
       "      <td>132.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:39:00+00:00</th>\n",
       "      <td>142.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:40:00+00:00</th>\n",
       "      <td>153.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:41:00+00:00</th>\n",
       "      <td>150.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:42:00+00:00</th>\n",
       "      <td>147.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:43:00+00:00</th>\n",
       "      <td>144.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:44:00+00:00</th>\n",
       "      <td>141.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:45:00+00:00</th>\n",
       "      <td>139.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:46:00+00:00</th>\n",
       "      <td>136.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:47:00+00:00</th>\n",
       "      <td>133.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:48:00+00:00</th>\n",
       "      <td>130.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:49:00+00:00</th>\n",
       "      <td>127.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:50:00+00:00</th>\n",
       "      <td>125.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:51:00+00:00</th>\n",
       "      <td>122.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:52:00+00:00</th>\n",
       "      <td>119.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:53:00+00:00</th>\n",
       "      <td>116.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:54:00+00:00</th>\n",
       "      <td>113.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 18:41:00+00:00</th>\n",
       "      <td>101.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 18:42:00+00:00</th>\n",
       "      <td>97.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 18:43:00+00:00</th>\n",
       "      <td>93.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 18:44:00+00:00</th>\n",
       "      <td>89.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 18:45:00+00:00</th>\n",
       "      <td>85.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 18:46:00+00:00</th>\n",
       "      <td>90.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 18:47:00+00:00</th>\n",
       "      <td>95.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 18:48:00+00:00</th>\n",
       "      <td>101.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 18:49:00+00:00</th>\n",
       "      <td>106.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 18:50:00+00:00</th>\n",
       "      <td>112.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 18:51:00+00:00</th>\n",
       "      <td>121.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 18:52:00+00:00</th>\n",
       "      <td>131.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 18:53:00+00:00</th>\n",
       "      <td>141.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 18:54:00+00:00</th>\n",
       "      <td>151.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 18:55:00+00:00</th>\n",
       "      <td>161.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 18:56:00+00:00</th>\n",
       "      <td>151.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 18:57:00+00:00</th>\n",
       "      <td>141.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 18:58:00+00:00</th>\n",
       "      <td>132.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 18:59:00+00:00</th>\n",
       "      <td>122.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 19:00:00+00:00</th>\n",
       "      <td>113.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 19:01:00+00:00</th>\n",
       "      <td>117.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 19:02:00+00:00</th>\n",
       "      <td>121.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 19:03:00+00:00</th>\n",
       "      <td>125.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 19:04:00+00:00</th>\n",
       "      <td>129.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 19:05:00+00:00</th>\n",
       "      <td>133.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 19:06:00+00:00</th>\n",
       "      <td>127.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 19:07:00+00:00</th>\n",
       "      <td>121.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 19:08:00+00:00</th>\n",
       "      <td>116.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 19:09:00+00:00</th>\n",
       "      <td>110.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 19:10:00+00:00</th>\n",
       "      <td>105.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>106 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                           pm_jitteratoz\n",
       "pm_collectiontime                       \n",
       "2019-01-03 17:25:00+00:00          114.0\n",
       "2019-01-03 17:26:00+00:00          123.8\n",
       "2019-01-03 17:27:00+00:00          133.6\n",
       "2019-01-03 17:28:00+00:00          143.4\n",
       "2019-01-03 17:29:00+00:00          153.2\n",
       "2019-01-03 17:30:00+00:00          163.0\n",
       "2019-01-03 17:31:00+00:00          150.6\n",
       "2019-01-03 17:32:00+00:00          138.2\n",
       "2019-01-03 17:33:00+00:00          125.8\n",
       "2019-01-03 17:34:00+00:00          113.4\n",
       "2019-01-03 17:35:00+00:00          101.0\n",
       "2019-01-03 17:36:00+00:00          111.4\n",
       "2019-01-03 17:37:00+00:00          121.8\n",
       "2019-01-03 17:38:00+00:00          132.2\n",
       "2019-01-03 17:39:00+00:00          142.6\n",
       "2019-01-03 17:40:00+00:00          153.0\n",
       "2019-01-03 17:41:00+00:00          150.2\n",
       "2019-01-03 17:42:00+00:00          147.4\n",
       "2019-01-03 17:43:00+00:00          144.6\n",
       "2019-01-03 17:44:00+00:00          141.8\n",
       "2019-01-03 17:45:00+00:00          139.0\n",
       "2019-01-03 17:46:00+00:00          136.2\n",
       "2019-01-03 17:47:00+00:00          133.4\n",
       "2019-01-03 17:48:00+00:00          130.6\n",
       "2019-01-03 17:49:00+00:00          127.8\n",
       "2019-01-03 17:50:00+00:00          125.0\n",
       "2019-01-03 17:51:00+00:00          122.2\n",
       "2019-01-03 17:52:00+00:00          119.4\n",
       "2019-01-03 17:53:00+00:00          116.6\n",
       "2019-01-03 17:54:00+00:00          113.8\n",
       "...                                  ...\n",
       "2019-01-03 18:41:00+00:00          101.0\n",
       "2019-01-03 18:42:00+00:00           97.0\n",
       "2019-01-03 18:43:00+00:00           93.0\n",
       "2019-01-03 18:44:00+00:00           89.0\n",
       "2019-01-03 18:45:00+00:00           85.0\n",
       "2019-01-03 18:46:00+00:00           90.4\n",
       "2019-01-03 18:47:00+00:00           95.8\n",
       "2019-01-03 18:48:00+00:00          101.2\n",
       "2019-01-03 18:49:00+00:00          106.6\n",
       "2019-01-03 18:50:00+00:00          112.0\n",
       "2019-01-03 18:51:00+00:00          121.8\n",
       "2019-01-03 18:52:00+00:00          131.6\n",
       "2019-01-03 18:53:00+00:00          141.4\n",
       "2019-01-03 18:54:00+00:00          151.2\n",
       "2019-01-03 18:55:00+00:00          161.0\n",
       "2019-01-03 18:56:00+00:00          151.4\n",
       "2019-01-03 18:57:00+00:00          141.8\n",
       "2019-01-03 18:58:00+00:00          132.2\n",
       "2019-01-03 18:59:00+00:00          122.6\n",
       "2019-01-03 19:00:00+00:00          113.0\n",
       "2019-01-03 19:01:00+00:00          117.0\n",
       "2019-01-03 19:02:00+00:00          121.0\n",
       "2019-01-03 19:03:00+00:00          125.0\n",
       "2019-01-03 19:04:00+00:00          129.0\n",
       "2019-01-03 19:05:00+00:00          133.0\n",
       "2019-01-03 19:06:00+00:00          127.4\n",
       "2019-01-03 19:07:00+00:00          121.8\n",
       "2019-01-03 19:08:00+00:00          116.2\n",
       "2019-01-03 19:09:00+00:00          110.6\n",
       "2019-01-03 19:10:00+00:00          105.0\n",
       "\n",
       "[106 rows x 1 columns]"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_pm_data_1min.interpolate(method='linear', order=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Shifting of Data\n",
    "\n",
    "shifted_version = sample_pm_data.shift(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pm_jitteratoz</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pm_collectiontime</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:25:00+00:00</th>\n",
       "      <td>163.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:30:00+00:00</th>\n",
       "      <td>101.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:35:00+00:00</th>\n",
       "      <td>153.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:40:00+00:00</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:45:00+00:00</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           pm_jitteratoz\n",
       "pm_collectiontime                       \n",
       "2019-01-03 17:25:00+00:00          163.0\n",
       "2019-01-03 17:30:00+00:00          101.0\n",
       "2019-01-03 17:35:00+00:00          153.0\n",
       "2019-01-03 17:40:00+00:00            NaN\n",
       "2019-01-03 17:45:00+00:00            NaN"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shifted_version[:5]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pm_jitteratoz</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pm_collectiontime</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:25:00+00:00</th>\n",
       "      <td>114.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:30:00+00:00</th>\n",
       "      <td>163.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:35:00+00:00</th>\n",
       "      <td>101.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:40:00+00:00</th>\n",
       "      <td>153.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-03 17:45:00+00:00</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           pm_jitteratoz\n",
       "pm_collectiontime                       \n",
       "2019-01-03 17:25:00+00:00          114.0\n",
       "2019-01-03 17:30:00+00:00          163.0\n",
       "2019-01-03 17:35:00+00:00          101.0\n",
       "2019-01-03 17:40:00+00:00          153.0\n",
       "2019-01-03 17:45:00+00:00            NaN"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_pm_data[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}