ch13.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction to Modeling Libraries "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "np.random.seed(12345)\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rc('figure', figsize=(10, 6))\n",
    "PREVIOUS_MAX_ROWS = pd.options.display.max_rows\n",
    "pd.options.display.max_rows = 20\n",
    "np.set_printoptions(precision=4, suppress=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Interfacing Between pandas and Model Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x0</th>\n",
       "      <th>x1</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.01</td>\n",
       "      <td>-1.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>-0.01</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.25</td>\n",
       "      <td>3.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>-4.10</td>\n",
       "      <td>1.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-2.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   x0    x1    y\n",
       "0   1  0.01 -1.5\n",
       "1   2 -0.01  0.0\n",
       "2   3  0.25  3.6\n",
       "3   4 -4.10  1.3\n",
       "4   5  0.00 -2.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "data = pd.DataFrame({\n",
    "    'x0': [1, 2, 3, 4, 5],\n",
    "    'x1': [0.01, -0.01, 0.25, -4.1, 0.],\n",
    "    'y': [-1.5, 0., 3.6, 1.3, -2.]})\n",
    "data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['x0', 'x1', 'y'], dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.  ,  0.01, -1.5 ],\n",
       "       [ 2.  , -0.01,  0.  ],\n",
       "       [ 3.  ,  0.25,  3.6 ],\n",
       "       [ 4.  , -4.1 ,  1.3 ],\n",
       "       [ 5.  ,  0.  , -2.  ]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtype('float64')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.values.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>one</th>\n",
       "      <th>two</th>\n",
       "      <th>three</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.01</td>\n",
       "      <td>-1.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.0</td>\n",
       "      <td>-0.01</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.0</td>\n",
       "      <td>0.25</td>\n",
       "      <td>3.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>-4.10</td>\n",
       "      <td>1.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-2.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   one   two  three\n",
       "0  1.0  0.01   -1.5\n",
       "1  2.0 -0.01    0.0\n",
       "2  3.0  0.25    3.6\n",
       "3  4.0 -4.10    1.3\n",
       "4  5.0  0.00   -2.0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2 = pd.DataFrame(data.values, columns=['one', 'two', 'three'])\n",
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x0</th>\n",
       "      <th>x1</th>\n",
       "      <th>y</th>\n",
       "      <th>strings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.01</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>-0.01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.25</td>\n",
       "      <td>3.6</td>\n",
       "      <td>c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>-4.10</td>\n",
       "      <td>1.3</td>\n",
       "      <td>d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>e</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   x0    x1    y strings\n",
       "0   1  0.01 -1.5       a\n",
       "1   2 -0.01  0.0       b\n",
       "2   3  0.25  3.6       c\n",
       "3   4 -4.10  1.3       d\n",
       "4   5  0.00 -2.0       e"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3 = data.copy()\n",
    "df3['strings'] = ['a', 'b','c','d','e']\n",
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0.01, -1.5, 'a'],\n",
       "       [2, -0.01, 0.0, 'b'],\n",
       "       [3, 0.25, 3.6, 'c'],\n",
       "       [4, -4.1, 1.3, 'd'],\n",
       "       [5, 0.0, -2.0, 'e']], dtype=object)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x0</th>\n",
       "      <th>x1</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.01</td>\n",
       "      <td>-1.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>-0.01</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.25</td>\n",
       "      <td>3.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>-4.10</td>\n",
       "      <td>1.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-2.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   x0    x1    y\n",
       "0   1  0.01 -1.5\n",
       "1   2 -0.01  0.0\n",
       "2   3  0.25  3.6\n",
       "3   4 -4.10  1.3\n",
       "4   5  0.00 -2.0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.  ,  0.01],\n",
       "       [ 2.  , -0.01],\n",
       "       [ 3.  ,  0.25],\n",
       "       [ 4.  , -4.1 ],\n",
       "       [ 5.  ,  0.  ]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_cols = ['x0', 'x1']\n",
    "data.loc[:, model_cols].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x0</th>\n",
       "      <th>x1</th>\n",
       "      <th>y</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.01</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>-0.01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.25</td>\n",
       "      <td>3.6</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>-4.10</td>\n",
       "      <td>1.3</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>b</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   x0    x1    y category\n",
       "0   1  0.01 -1.5        a\n",
       "1   2 -0.01  0.0        b\n",
       "2   3  0.25  3.6        a\n",
       "3   4 -4.10  1.3        a\n",
       "4   5  0.00 -2.0        b"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['category'] = pd.Categorical(['a', 'b', 'a', 'a', 'b'],\n",
    "                                  categories=['a', 'b'])\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x0</th>\n",
       "      <th>x1</th>\n",
       "      <th>y</th>\n",
       "      <th>category_a</th>\n",
       "      <th>category_b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.01</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>-0.01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.25</td>\n",
       "      <td>3.6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>-4.10</td>\n",
       "      <td>1.3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   x0    x1    y  category_a  category_b\n",
       "0   1  0.01 -1.5           1           0\n",
       "1   2 -0.01  0.0           0           1\n",
       "2   3  0.25  3.6           1           0\n",
       "3   4 -4.10  1.3           1           0\n",
       "4   5  0.00 -2.0           0           1"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummies = pd.get_dummies(data.category, prefix='category')\n",
    "data_with_dummies = data.drop('category', axis=1).join(dummies)\n",
    "data_with_dummies"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating Model Descriptions with Patsy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "y ~ x0 + x1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x0</th>\n",
       "      <th>x1</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.01</td>\n",
       "      <td>-1.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>-0.01</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.25</td>\n",
       "      <td>3.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>-4.10</td>\n",
       "      <td>1.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-2.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   x0    x1    y\n",
       "0   1  0.01 -1.5\n",
       "1   2 -0.01  0.0\n",
       "2   3  0.25  3.6\n",
       "3   4 -4.10  1.3\n",
       "4   5  0.00 -2.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.DataFrame({\n",
    "    'x0': [1, 2, 3, 4, 5],\n",
    "    'x1': [0.01, -0.01, 0.25, -4.1, 0.],\n",
    "    'y': [-1.5, 0., 3.6, 1.3, -2.]})\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import patsy\n",
    "y, X = patsy.dmatrices('y ~ x0 + x1', data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DesignMatrix with shape (5, 1)\n",
       "     y\n",
       "  -1.5\n",
       "   0.0\n",
       "   3.6\n",
       "   1.3\n",
       "  -2.0\n",
       "  Terms:\n",
       "    'y' (column 0)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DesignMatrix with shape (5, 3)\n",
       "  Intercept  x0     x1\n",
       "          1   1   0.01\n",
       "          1   2  -0.01\n",
       "          1   3   0.25\n",
       "          1   4  -4.10\n",
       "          1   5   0.00\n",
       "  Terms:\n",
       "    'Intercept' (column 0)\n",
       "    'x0' (column 1)\n",
       "    'x1' (column 2)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.5],\n",
       "       [ 0. ],\n",
       "       [ 3.6],\n",
       "       [ 1.3],\n",
       "       [-2. ]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.asarray(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.  ,  1.  ,  0.01],\n",
       "       [ 1.  ,  2.  , -0.01],\n",
       "       [ 1.  ,  3.  ,  0.25],\n",
       "       [ 1.  ,  4.  , -4.1 ],\n",
       "       [ 1.  ,  5.  ,  0.  ]])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.asarray(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DesignMatrix with shape (5, 2)\n",
       "  x0     x1\n",
       "   1   0.01\n",
       "   2  -0.01\n",
       "   3   0.25\n",
       "   4  -4.10\n",
       "   5   0.00\n",
       "  Terms:\n",
       "    'x0' (column 0)\n",
       "    'x1' (column 1)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "patsy.dmatrices('y ~ x0 + x1 + 0', data)[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\python\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: FutureWarning: `rcond` parameter will change to the default of machine precision times ``max(M, N)`` where M and N are the input matrix dimensions.\n",
      "To use the future default and silence this warning we advise to pass `rcond=None`, to keep using the old, explicitly pass `rcond=-1`.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "coef, resid, _, _ = np.linalg.lstsq(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.3129],\n",
       "       [-0.0791],\n",
       "       [-0.2655]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coef"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Intercept    0.312910\n",
       "x0          -0.079106\n",
       "x1          -0.265464\n",
       "dtype: float64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coef = pd.Series(coef.squeeze(), index=X.design_info.column_names)\n",
    "coef"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Transformations in Patsy Formulas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DesignMatrix with shape (5, 3)\n",
       "  Intercept  x0  np.log(np.abs(x1) + 1)\n",
       "          1   1                 0.00995\n",
       "          1   2                 0.00995\n",
       "          1   3                 0.22314\n",
       "          1   4                 1.62924\n",
       "          1   5                 0.00000\n",
       "  Terms:\n",
       "    'Intercept' (column 0)\n",
       "    'x0' (column 1)\n",
       "    'np.log(np.abs(x1) + 1)' (column 2)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y, X = patsy.dmatrices('y ~ x0 + np.log(np.abs(x1) + 1)', data)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DesignMatrix with shape (5, 3)\n",
       "  Intercept  standardize(x0)  center(x1)\n",
       "          1         -1.41421        0.78\n",
       "          1         -0.70711        0.76\n",
       "          1          0.00000        1.02\n",
       "          1          0.70711       -3.33\n",
       "          1          1.41421        0.77\n",
       "  Terms:\n",
       "    'Intercept' (column 0)\n",
       "    'standardize(x0)' (column 1)\n",
       "    'center(x1)' (column 2)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y, X = patsy.dmatrices('y ~ standardize(x0) + center(x1)', data)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[DesignMatrix with shape (4, 3)\n",
       "   Intercept  standardize(x0)  center(x1)\n",
       "           1          2.12132        3.87\n",
       "           1          2.82843        0.27\n",
       "           1          3.53553        0.77\n",
       "           1          4.24264        3.07\n",
       "   Terms:\n",
       "     'Intercept' (column 0)\n",
       "     'standardize(x0)' (column 1)\n",
       "     'center(x1)' (column 2)]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_data = pd.DataFrame({\n",
    "    'x0': [6, 7, 8, 9],\n",
    "    'x1': [3.1, -0.5, 0, 2.3],\n",
    "    'y': [1, 2, 3, 4]})\n",
    "new_X = patsy.build_design_matrices([X.design_info], new_data)\n",
    "new_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DesignMatrix with shape (5, 2)\n",
       "  Intercept  I(x0 + x1)\n",
       "          1        1.01\n",
       "          1        1.99\n",
       "          1        3.25\n",
       "          1       -0.10\n",
       "          1        5.00\n",
       "  Terms:\n",
       "    'Intercept' (column 0)\n",
       "    'I(x0 + x1)' (column 1)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y, X = patsy.dmatrices('y ~ I(x0 + x1)', data)\n",
    "X"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Categorical Data and Patsy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DesignMatrix with shape (8, 2)\n",
       "  Intercept  key1[T.b]\n",
       "          1          0\n",
       "          1          0\n",
       "          1          1\n",
       "          1          1\n",
       "          1          0\n",
       "          1          1\n",
       "          1          0\n",
       "          1          1\n",
       "  Terms:\n",
       "    'Intercept' (column 0)\n",
       "    'key1' (column 1)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.DataFrame({\n",
    "    'key1': ['a', 'a', 'b', 'b', 'a', 'b', 'a', 'b'],\n",
    "    'key2': [0, 1, 0, 1, 0, 1, 0, 0],\n",
    "    'v1': [1, 2, 3, 4, 5, 6, 7, 8],\n",
    "    'v2': [-1, 0, 2.5, -0.5, 4.0, -1.2, 0.2, -1.7]\n",
    "})\n",
    "y, X = patsy.dmatrices('v2 ~ key1', data)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DesignMatrix with shape (8, 2)\n",
       "  key1[a]  key1[b]\n",
       "        1        0\n",
       "        1        0\n",
       "        0        1\n",
       "        0        1\n",
       "        1        0\n",
       "        0        1\n",
       "        1        0\n",
       "        0        1\n",
       "  Terms:\n",
       "    'key1' (columns 0:2)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y, X = patsy.dmatrices('v2 ~ key1 + 0', data)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DesignMatrix with shape (8, 2)\n",
       "  Intercept  C(key2)[T.1]\n",
       "          1             0\n",
       "          1             1\n",
       "          1             0\n",
       "          1             1\n",
       "          1             0\n",
       "          1             1\n",
       "          1             0\n",
       "          1             0\n",
       "  Terms:\n",
       "    'Intercept' (column 0)\n",
       "    'C(key2)' (column 1)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y, X = patsy.dmatrices('v2 ~ C(key2)', data)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DesignMatrix with shape (8, 4)\n",
       "  Intercept  key1[T.b]  key2[T.zero]  key1[T.b]:key2[T.zero]\n",
       "          1          0             1                       0\n",
       "          1          0             0                       0\n",
       "          1          1             1                       1\n",
       "          1          1             0                       0\n",
       "          1          0             1                       0\n",
       "          1          1             0                       0\n",
       "          1          0             1                       0\n",
       "          1          1             1                       1\n",
       "  Terms:\n",
       "    'Intercept' (column 0)\n",
       "    'key1' (column 1)\n",
       "    'key2' (column 2)\n",
       "    'key1:key2' (column 3)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['key2'] = data['key2'].map({0: 'zero', 1: 'one'})\n",
    "data\n",
    "y, X = patsy.dmatrices('v2 ~ key1 + key2', data)\n",
    "X\n",
    "y, X = patsy.dmatrices('v2 ~ key1 + key2 + key1:key2', data)\n",
    "X"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Introduction to statsmodels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Estimating Linear Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import statsmodels.api as sm\n",
    "import statsmodels.formula.api as smf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dnorm(mean, variance, size=1):\n",
    "    if isinstance(size, int):\n",
    "        size = size,\n",
    "    return mean + np.sqrt(variance) * np.random.randn(*size)\n",
    "\n",
    "# For reproducibility\n",
    "np.random.seed(12345)\n",
    "\n",
    "N = 100\n",
    "X = np.c_[dnorm(0, 0.4, size=N),\n",
    "          dnorm(0, 0.6, size=N),\n",
    "          dnorm(0, 0.2, size=N)]\n",
    "eps = dnorm(0, 0.1, size=N)\n",
    "beta = [0.1, 0.3, 0.5]\n",
    "\n",
    "y = np.dot(X, beta) + eps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.5525, -0.446 , -0.1196, -0.1094,  0.1202,  0.0598,  0.4185,\n",
       "       -0.7161, -0.2893, -0.1515,  0.3311,  0.2922, -0.0361,  0.1283,\n",
       "        0.0912, -0.1375,  0.1134, -0.1228,  0.6732,  0.4458, -0.0333,\n",
       "        0.2215,  0.6618, -0.0433, -0.2942,  0.1036,  0.412 , -0.4457,\n",
       "       -0.0456, -0.2266,  0.0328, -0.473 , -0.3715,  0.8266, -0.218 ,\n",
       "       -0.2377,  0.2012, -0.3661,  0.1944,  0.323 ,  0.2113, -0.256 ,\n",
       "       -0.2872,  0.4782,  0.0301,  0.3746,  0.2014, -0.1705, -0.0174,\n",
       "       -0.3592, -0.0539, -0.3662,  0.3493,  0.2006,  0.3983,  0.3051,\n",
       "       -0.1374, -0.2782, -0.2197,  0.3878,  0.1446,  0.0366,  0.3207,\n",
       "       -0.3589, -0.0833,  0.4131, -0.5094, -0.3246,  0.3926, -0.0496,\n",
       "       -0.7745, -0.327 ,  0.5058,  0.1499,  0.0479, -0.1715, -0.1504,\n",
       "        0.0336, -0.4137,  0.6872,  0.1785, -0.0602, -0.29  , -0.3086,\n",
       "        0.6996,  0.0234,  0.5751, -0.5001, -0.2449,  0.1749,  0.0335,\n",
       "        1.242 , -0.0807,  0.2701, -0.1154,  0.0415, -0.2206,  0.4224,\n",
       "       -0.0478,  0.1401])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.4279, -0.6735, -0.0909, -0.4895, -0.1289, -0.045 ,  0.0876,\n",
       "       -0.5046, -0.5458,  0.2653,  0.5978,  0.4527,  0.087 ,  0.0554,\n",
       "       -0.0912,  0.1447, -0.1513, -0.0563,  1.2168, -0.0223, -0.6906,\n",
       "        0.0852,  0.7344, -0.3527, -0.2547,  0.3078,  0.7038, -0.5332,\n",
       "       -0.2207, -0.0968, -0.4969, -1.3334, -0.3769,  1.26  , -0.2948,\n",
       "       -0.6145,  0.1873, -0.4078,  0.0573,  0.4745, -0.4352,  0.0315,\n",
       "       -0.0564,  0.1213,  0.2235,  0.0596,  0.2581, -0.275 ,  0.3051,\n",
       "       -0.2003,  0.0863, -0.4245,  0.2348, -0.3206,  0.6756, -0.3873,\n",
       "       -0.3786, -0.1638, -0.1701,  0.3924, -0.1369,  0.1887, -0.1399,\n",
       "        0.6137, -0.4083,  0.4687, -0.5963, -0.0771,  0.7082,  0.1474,\n",
       "       -0.2524, -0.4508,  0.1486,  0.0276, -0.1538, -0.3791, -0.6927,\n",
       "       -0.4175, -0.4004,  0.4613,  0.0241, -0.2985, -0.4436, -0.074 ,\n",
       "        1.2439,  0.1765, -0.2986, -0.3386, -0.539 ,  0.2511, -0.2229,\n",
       "        1.2876, -0.1681,  0.5865,  0.0675,  0.253 , -0.1486,  0.3485,\n",
       "        0.2041,  0.402 ])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.1295, -1.2128,  0.5042],\n",
       "       [ 0.3029, -0.4357, -0.2542],\n",
       "       [-0.3285, -0.0253,  0.1384],\n",
       "       [-0.3515, -0.7196, -0.2582],\n",
       "       [ 1.2433, -0.3738, -0.5226]])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.4279, -0.6735, -0.0909, -0.4895, -0.1289])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.    , -0.1295, -1.2128,  0.5042],\n",
       "       [ 1.    ,  0.3029, -0.4357, -0.2542],\n",
       "       [ 1.    , -0.3285, -0.0253,  0.1384],\n",
       "       [ 1.    , -0.3515, -0.7196, -0.2582],\n",
       "       [ 1.    ,  1.2433, -0.3738, -0.5226]])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_model = sm.add_constant(X)\n",
    "X_model[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = sm.OLS(y, X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.1783, 0.223 , 0.501 ])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = model.fit()\n",
    "results.params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                 OLS Regression Results                                \n",
      "=======================================================================================\n",
      "Dep. Variable:                      y   R-squared (uncentered):                   0.430\n",
      "Model:                            OLS   Adj. R-squared (uncentered):              0.413\n",
      "Method:                 Least Squares   F-statistic:                              24.42\n",
      "Date:                Mon, 27 Dec 2021   Prob (F-statistic):                    7.44e-12\n",
      "Time:                        10:11:24   Log-Likelihood:                         -34.305\n",
      "No. Observations:                 100   AIC:                                      74.61\n",
      "Df Residuals:                      97   BIC:                                      82.42\n",
      "Df Model:                           3                                                  \n",
      "Covariance Type:            nonrobust                                                  \n",
      "==============================================================================\n",
      "                 coef    std err          t      P>|t|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "x1             0.1783      0.053      3.364      0.001       0.073       0.283\n",
      "x2             0.2230      0.046      4.818      0.000       0.131       0.315\n",
      "x3             0.5010      0.080      6.237      0.000       0.342       0.660\n",
      "==============================================================================\n",
      "Omnibus:                        4.662   Durbin-Watson:                   2.201\n",
      "Prob(Omnibus):                  0.097   Jarque-Bera (JB):                4.098\n",
      "Skew:                           0.481   Prob(JB):                        0.129\n",
      "Kurtosis:                       3.243   Cond. No.                         1.74\n",
      "==============================================================================\n",
      "\n",
      "Warnings:\n",
      "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n"
     ]
    }
   ],
   "source": [
    "print(results.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col0</th>\n",
       "      <th>col1</th>\n",
       "      <th>col2</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.129468</td>\n",
       "      <td>-1.212753</td>\n",
       "      <td>0.504225</td>\n",
       "      <td>0.427863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.302910</td>\n",
       "      <td>-0.435742</td>\n",
       "      <td>-0.254180</td>\n",
       "      <td>-0.673480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.328522</td>\n",
       "      <td>-0.025302</td>\n",
       "      <td>0.138351</td>\n",
       "      <td>-0.090878</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.351475</td>\n",
       "      <td>-0.719605</td>\n",
       "      <td>-0.258215</td>\n",
       "      <td>-0.489494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.243269</td>\n",
       "      <td>-0.373799</td>\n",
       "      <td>-0.522629</td>\n",
       "      <td>-0.128941</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       col0      col1      col2         y\n",
       "0 -0.129468 -1.212753  0.504225  0.427863\n",
       "1  0.302910 -0.435742 -0.254180 -0.673480\n",
       "2 -0.328522 -0.025302  0.138351 -0.090878\n",
       "3 -0.351475 -0.719605 -0.258215 -0.489494\n",
       "4  1.243269 -0.373799 -0.522629 -0.128941"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.DataFrame(X, columns=['col0', 'col1', 'col2'])\n",
    "data['y'] = y\n",
    "data[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Intercept    0.033559\n",
       "col0         0.176149\n",
       "col1         0.224826\n",
       "col2         0.514808\n",
       "dtype: float64"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = smf.ols('y ~ col0 + col1 + col2', data=data).fit()\n",
    "results.params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Intercept    0.952188\n",
       "col0         3.319754\n",
       "col1         4.850730\n",
       "col2         6.303971\n",
       "dtype: float64"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.tvalues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0   -0.002327\n",
       "1   -0.141904\n",
       "2    0.041226\n",
       "3   -0.323070\n",
       "4   -0.100535\n",
       "dtype: float64"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.predict(data[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Estimating Time Series Processes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[4,\n",
       " 4,\n",
       " 1.5239259186004943,\n",
       " -0.32929765907526476,\n",
       " -1.058529579932304,\n",
       " -0.5839919422634523,\n",
       " 0.09130619199413403,\n",
       " 0.677436916817214,\n",
       " 0.3942501062184828,\n",
       " 0.5044078876666771,\n",
       " 0.8919804688565234,\n",
       " -0.041358874581530425,\n",
       " -0.512333702166864,\n",
       " -0.45912220261937253,\n",
       " -0.5487809902217377,\n",
       " -0.2767084927720815,\n",
       " 0.07056345240070924,\n",
       " -0.015086990166775283,\n",
       " 0.21803003801485396,\n",
       " -0.06372982168353292,\n",
       " -0.559909587726604,\n",
       " -0.5201878266625373,\n",
       " -0.18315553919214744,\n",
       " 0.41876312500969687,\n",
       " 0.2899487200351789,\n",
       " 0.06084837305377688,\n",
       " 0.019005439039556815,\n",
       " -0.19936005112734528,\n",
       " 0.014508205852043654,\n",
       " 0.029965874911542033,\n",
       " 0.08211844486903001,\n",
       " -0.10602585467518115,\n",
       " 0.8166077282898706,\n",
       " -0.13606059063285314,\n",
       " -0.5469469984191534,\n",
       " -0.534229438266625,\n",
       " -0.15750410234586604,\n",
       " 0.6211118678628107,\n",
       " 0.819819232715918,\n",
       " 0.19635842150328683,\n",
       " -0.022691683195220053,\n",
       " 0.20888764514208696,\n",
       " 0.34076525849118094,\n",
       " 0.15145612595892088,\n",
       " -0.004548096672983625,\n",
       " -0.13555704203291033,\n",
       " -0.07318926746635465,\n",
       " 0.09156462023796405,\n",
       " 0.040645938736817426,\n",
       " 0.23778497085869685,\n",
       " 0.03171841020912716,\n",
       " 0.2585444986554568,\n",
       " 0.6916323032899407,\n",
       " 0.8555740639409919,\n",
       " 0.4966570542514953,\n",
       " -0.22699055353008352,\n",
       " -0.21658699924927555,\n",
       " -0.08495641734043555,\n",
       " -0.49956635923932563,\n",
       " 0.07743632994616173,\n",
       " 0.8178745455908967,\n",
       " 0.8222109205453448,\n",
       " 0.19660911983476162,\n",
       " 0.07814063802253471,\n",
       " 0.2803537805791525,\n",
       " -0.519527446068504,\n",
       " -1.0884095358211292,\n",
       " -0.9463929017151143,\n",
       " -0.10758116363406497,\n",
       " 0.30403562152076896,\n",
       " 0.3138049508304622,\n",
       " 0.11467385122017464,\n",
       " 0.2594353648232348,\n",
       " 0.3141904809975369,\n",
       " -0.12663248600884694,\n",
       " -0.07081583489417864,\n",
       " -0.05515226314197013,\n",
       " 0.4260651484672361,\n",
       " 0.8394855332989719,\n",
       " 0.12818992949286423,\n",
       " -0.09309918645106868,\n",
       " 0.31559611206401006,\n",
       " 0.12222386655667952,\n",
       " 0.22052510832261743,\n",
       " 0.2062221668526688,\n",
       " 0.31180055626310976,\n",
       " 0.567848207741972,\n",
       " 0.7414119604977253,\n",
       " 0.27992772078452033,\n",
       " -0.08891839337263588,\n",
       " -0.33101827663272965,\n",
       " -0.558382259611548,\n",
       " -0.17945308723586073,\n",
       " 0.17875241554539206,\n",
       " -0.1380286199302797,\n",
       " 0.20839501935305954,\n",
       " -0.3767214355423645,\n",
       " -0.9101809302155144,\n",
       " -0.5791880760613489,\n",
       " 0.24927008659745709,\n",
       " 0.7515869048733794,\n",
       " 1.2190187559742922,\n",
       " 0.7208525179157028,\n",
       " 0.24984178637234025,\n",
       " -0.15067356400850346,\n",
       " -1.4048610877821903,\n",
       " -1.5443214711727666,\n",
       " -0.7831929464998593,\n",
       " -0.22921064235533056,\n",
       " 0.5433229394786371,\n",
       " 0.31015643979279206,\n",
       " 0.3780102370357359,\n",
       " 0.059266617066583996,\n",
       " 0.2719311267457589,\n",
       " 0.7609691954758444,\n",
       " 0.9104350177575278,\n",
       " 0.6825482956768583,\n",
       " 0.07615097179175158,\n",
       " -0.7070193097822529,\n",
       " -0.14603576344674868,\n",
       " 0.4318583432809311,\n",
       " -0.06353002350796455,\n",
       " 0.043171680566917586,\n",
       " -0.047140015421950066,\n",
       " 0.2291793313023785,\n",
       " 0.29730586759356525,\n",
       " -0.31265579570412344,\n",
       " -0.21483910384765306,\n",
       " 0.12801259819519556,\n",
       " 0.3945152513364656,\n",
       " -0.28789417679490303,\n",
       " -0.12375611914226037,\n",
       " -0.14565549748917356,\n",
       " -0.11830866495660003,\n",
       " 0.33172175301687634,\n",
       " 0.09984002186713417,\n",
       " 0.4086360755699036,\n",
       " 1.0386527149675961,\n",
       " 0.3667327153305424,\n",
       " -0.2062606741437239,\n",
       " -0.5044920066241919,\n",
       " -0.4061029033560951,\n",
       " -0.06123389216645902,\n",
       " 0.38534349507736954,\n",
       " 0.24953521240797324,\n",
       " 0.4064174717502118,\n",
       " 0.19308467544229696,\n",
       " -0.03234129368029209,\n",
       " -0.06489484139379882,\n",
       " 0.301473570795245,\n",
       " 0.4044429409265539,\n",
       " 0.22081033712018616,\n",
       " -0.10494439488872287,\n",
       " -0.15715429622694393,\n",
       " 0.024285940229544778,\n",
       " 0.5781102160875212,\n",
       " 0.43771519481271537,\n",
       " -0.14095575247775083,\n",
       " -0.3445597203689479,\n",
       " -0.2719728324068896,\n",
       " 0.025618826415749377,\n",
       " -0.04293817248877088,\n",
       " -0.2512955444084867,\n",
       " -0.9146717415447005,\n",
       " -0.12137458255831768,\n",
       " 0.013695136128894336,\n",
       " 0.23592444730594325,\n",
       " 0.36673718288989066,\n",
       " -0.1822927403627127,\n",
       " -0.2841230421413876,\n",
       " 0.004591645170381992,\n",
       " 0.6053323185875167,\n",
       " 0.7631194587342517,\n",
       " -0.2778565325870943,\n",
       " -0.7834110746762531,\n",
       " -0.6533616377124916,\n",
       " 0.48157277012108346,\n",
       " 0.3055768887370905,\n",
       " 0.7517274558445652,\n",
       " 0.6205663226061349,\n",
       " 0.1643142064678698,\n",
       " -0.2982873711263919,\n",
       " 0.24203349058202045,\n",
       " 0.20174767609698746,\n",
       " 0.2586985164775655,\n",
       " 0.06591569975105732,\n",
       " -0.3607799835972236,\n",
       " -0.8625590902707625,\n",
       " -0.48769020918345696,\n",
       " -0.16683362918521666,\n",
       " -0.3198713214273191,\n",
       " -0.7964351066092814,\n",
       " -0.5027297700941475,\n",
       " -0.5105327350412042,\n",
       " -0.0773453929494993,\n",
       " 0.6460191288209245,\n",
       " 1.1514744410303432,\n",
       " 1.2178192896008686,\n",
       " 0.4277103606391165,\n",
       " 0.26297681012742574,\n",
       " -0.034143152253364895,\n",
       " 0.003056202620454751,\n",
       " -0.21254273794332634,\n",
       " -0.047713842676633256,\n",
       " 0.57168352505546,\n",
       " 0.7527904677907487,\n",
       " 0.0003431886852909849,\n",
       " -0.3668967515471683,\n",
       " -0.4674758790283804,\n",
       " -0.6234174322497066,\n",
       " -0.7154908181827834,\n",
       " 0.2718481871041699,\n",
       " 0.8541269492159,\n",
       " 0.02053318976327001,\n",
       " -0.6331980840735386,\n",
       " -0.22740557767790426,\n",
       " -0.0896766824927149,\n",
       " -0.30440662243676136,\n",
       " -0.3380268899543364,\n",
       " -0.4879834416351495,\n",
       " -0.4453587330680133,\n",
       " -0.09083465912621598,\n",
       " 0.40569058082588483,\n",
       " 0.5835695564757045,\n",
       " -0.12516261080417684,\n",
       " 0.029621626714367744,\n",
       " 0.0225737090466926,\n",
       " -0.19219918019051146,\n",
       " 0.032517174800194226,\n",
       " -0.35532962676442525,\n",
       " 0.022464244978929737,\n",
       " -0.029570630554543814,\n",
       " 0.24019132223213652,\n",
       " 0.729498985440212,\n",
       " 0.21851137398477288,\n",
       " -0.05126876160998549,\n",
       " -0.16863981945557038,\n",
       " -0.026026083217459015,\n",
       " 0.2739172247913073,\n",
       " 0.06144979923670335,\n",
       " -0.338765727192406,\n",
       " -0.41476995179306547,\n",
       " -0.5624007508872568,\n",
       " -0.3237410854177017,\n",
       " -0.4774583415331999,\n",
       " 0.5142768074024069,\n",
       " -0.016150134991858467,\n",
       " -0.5023943588201921,\n",
       " -0.08251165882286193,\n",
       " 0.48167827588680534,\n",
       " 0.5620154521736268,\n",
       " -0.06506735868486091,\n",
       " 0.43093260008295103,\n",
       " 0.4992416814271608,\n",
       " 0.6368387874421245,\n",
       " 0.45808584956496756,\n",
       " -0.5824897092572745,\n",
       " -1.0355694590904685,\n",
       " -0.2908015679106378,\n",
       " 0.21553085466051813,\n",
       " -0.10826763718462118,\n",
       " -0.34192588525416745,\n",
       " 0.0731887989367084,\n",
       " -0.05993790510086161,\n",
       " -0.07891903068328024,\n",
       " 0.01625642613867795,\n",
       " 0.6265080373762334,\n",
       " -0.08153685729954385,\n",
       " -0.4343328353455359,\n",
       " 0.24349300424208487,\n",
       " 0.3006613792335249,\n",
       " 0.04531775962873079,\n",
       " 0.0295546414419212,\n",
       " 0.2544581351228957,\n",
       " 0.09696529352809742,\n",
       " -0.18293845529113162,\n",
       " -0.10963186853162858,\n",
       " 0.04498049088563444,\n",
       " -0.27380903519946426,\n",
       " -0.11490599066433388,\n",
       " 0.08847261842379436,\n",
       " -0.31307115875720126,\n",
       " 0.060072462494300094,\n",
       " -0.22336396134811626,\n",
       " -1.1277220308743552,\n",
       " -0.6058364270704504,\n",
       " 0.3545086287026154,\n",
       " 0.6528026235766258,\n",
       " 0.2511659261203607,\n",
       " 0.6205147145767436,\n",
       " -0.2276370053793294,\n",
       " -0.614616442602517,\n",
       " -0.61663251885123,\n",
       " -0.3151906686801386,\n",
       " -0.21750075309902517,\n",
       " -0.017468060241283327,\n",
       " -0.199530971216449,\n",
       " 0.2015699836351003,\n",
       " 0.6905670099160713,\n",
       " 0.44669675178428814,\n",
       " 0.27058760513334507,\n",
       " -0.14591452403860958,\n",
       " 0.29239856840825584,\n",
       " 0.865526521737074,\n",
       " 0.33468917952551813,\n",
       " -0.25811706652628486,\n",
       " -0.21562589153705364,\n",
       " 0.2690241572096931,\n",
       " 0.6027683740937732,\n",
       " 0.20464100322363768,\n",
       " 0.3403399161746379,\n",
       " 0.23890639734491173,\n",
       " -0.6041024274969551,\n",
       " -0.5227168680615124,\n",
       " -0.13882546194930956,\n",
       " 0.33462802558921645,\n",
       " 0.47747108017624407,\n",
       " -0.2755059333245322,\n",
       " 0.26392601191352255,\n",
       " 0.09291976718534695,\n",
       " 0.3036995127621124,\n",
       " 0.5955273940409096,\n",
       " 0.4105526205255015,\n",
       " -0.08372877507926405,\n",
       " 0.15908209407999652,\n",
       " 0.08624477440005997,\n",
       " 0.3634306150985577,\n",
       " 0.18407990464355867,\n",
       " -0.4434641606089933,\n",
       " -0.43037923803913897,\n",
       " -0.3434862334376887,\n",
       " -0.2475784129747982,\n",
       " -0.6756259546930349,\n",
       " 0.08689514657755282,\n",
       " -0.2661890899903298,\n",
       " -0.20718128837414906,\n",
       " -0.3197687331367086,\n",
       " 0.0856346860731339,\n",
       " 0.31696731347705065,\n",
       " 0.27249316716820404,\n",
       " 0.021318013063857222,\n",
       " -0.520242693724982,\n",
       " -0.22106862131205993,\n",
       " -0.03242784915047635,\n",
       " 0.3847374239688262,\n",
       " 0.3877995105750566,\n",
       " 0.03895745196971992,\n",
       " 0.14356935387928726,\n",
       " -0.002977645759489833,\n",
       " -0.10359135604372474,\n",
       " 0.14655599487865334,\n",
       " 0.31731829848397475,\n",
       " 0.5388202228543486,\n",
       " 0.2558147939173985,\n",
       " 0.41443229690233546,\n",
       " 0.10144553731634787,\n",
       " -0.12842237482153854,\n",
       " -0.29438800924260417,\n",
       " -0.4357724488414329,\n",
       " -0.22866861585170775,\n",
       " -0.06307119149460275,\n",
       " -0.096779692252188,\n",
       " -0.48817146369879544,\n",
       " -0.44349235279583527,\n",
       " -0.08394126607131489,\n",
       " 0.619195310682721,\n",
       " 0.4391632463002042,\n",
       " -0.3149586781887907,\n",
       " -0.3191079049474622,\n",
       " 0.5329600070112135,\n",
       " 0.5895292563164445,\n",
       " 0.17369931527306268,\n",
       " 0.15097140555081393,\n",
       " 0.18061993932116294,\n",
       " 0.04692221274442371,\n",
       " 0.05475114610370445,\n",
       " -0.007914868998938639,\n",
       " -0.17082712952737208,\n",
       " -0.008197015235186345,\n",
       " 0.6895811641193647,\n",
       " 0.32155282958944986,\n",
       " -0.471086454061444,\n",
       " -0.3585325159745304,\n",
       " 0.5575742021524708,\n",
       " 0.9202394324103665,\n",
       " 0.9548848452230496,\n",
       " -0.004921158624561439,\n",
       " 0.09082594708771607,\n",
       " -0.5780999533121121,\n",
       " -0.3844367575190267,\n",
       " -0.4654963041107001,\n",
       " -0.2885490527512699,\n",
       " 0.3436205810469276,\n",
       " 0.8291523008075492,\n",
       " 0.41679058989769674,\n",
       " 0.14309680956641213,\n",
       " 0.03411588929389102,\n",
       " -0.07722314745199202,\n",
       " -0.2974775046739211,\n",
       " 0.27716679778645326,\n",
       " 0.2630137745355394,\n",
       " 0.391927467975724,\n",
       " -0.10654452153261601,\n",
       " -0.28856461065214,\n",
       " 0.09369768919366703,\n",
       " -0.18266245098324652,\n",
       " 0.1665919524585912,\n",
       " 0.4162542151351517,\n",
       " 0.03225003913811522,\n",
       " -0.005881224084505976,\n",
       " -0.421220280857899,\n",
       " -0.13620802472569107,\n",
       " -0.017558120632222757,\n",
       " 0.14207695594651548,\n",
       " 0.4426181732967748,\n",
       " 0.429746771208242,\n",
       " -0.07528477026845129,\n",
       " -0.4704200395106354,\n",
       " -0.3587508620828608,\n",
       " -0.046074850641493616,\n",
       " 0.5811607148229659,\n",
       " 0.5649422620981714,\n",
       " 0.17855029436230996,\n",
       " -0.18693220374387232,\n",
       " -0.009054960916192845,\n",
       " -0.025384080669666784,\n",
       " 0.1495272911323656,\n",
       " -0.15589025113085547,\n",
       " -0.1262689137361413,\n",
       " -0.36317593796330316,\n",
       " -0.1111728141046871,\n",
       " 0.3131420030770603,\n",
       " 0.5057010940940064,\n",
       " 0.4684507492114843,\n",
       " 0.29401840046158556,\n",
       " 0.020546732541146074,\n",
       " -0.5572031854595738,\n",
       " -0.3274476408731487,\n",
       " 0.031136532444462475,\n",
       " 0.11379954929410147,\n",
       " 0.2040398753043239,\n",
       " 0.46700656366343085,\n",
       " 0.13334271562126074,\n",
       " -0.21874394205683134,\n",
       " -0.070424935735003,\n",
       " -0.05257794255679449,\n",
       " -0.11707361547736043,\n",
       " -0.4341028999817091,\n",
       " -0.5254802725244518,\n",
       " -0.4678594653587496,\n",
       " -0.16411932583605973,\n",
       " 0.31717792800798356,\n",
       " 0.5467544458432037,\n",
       " 0.43900718549498663,\n",
       " 0.3391178731342932,\n",
       " -0.2559206770162227,\n",
       " 0.14920183767668632,\n",
       " 0.9308593879288017,\n",
       " 0.35360051906870993,\n",
       " 0.13094580677174092,\n",
       " -0.48903721882603024,\n",
       " -0.12458663790291102,\n",
       " 0.19959434714603486,\n",
       " 0.5736025880029729,\n",
       " 0.37943725794081873,\n",
       " 0.40863277858166863,\n",
       " -0.3413863115523421,\n",
       " -0.2936966299676911,\n",
       " -0.2308357415638822,\n",
       " -0.644573808350576,\n",
       " -0.20916063737502805,\n",
       " 0.06458053085273732,\n",
       " 0.6290851957023431,\n",
       " 0.7765767446459353,\n",
       " 0.18209355438124786,\n",
       " 0.0012535447973874014,\n",
       " -0.6931919553643754,\n",
       " -0.565397265961951,\n",
       " 0.22811898787528823,\n",
       " 0.41253879380074504,\n",
       " 0.15158680801804367,\n",
       " -0.6647683665910451,\n",
       " -1.1586469749437696,\n",
       " -0.46121977730849284,\n",
       " -0.36344456856027263,\n",
       " 0.47503473198851065,\n",
       " 0.6206911252927005,\n",
       " 0.9720022004522308,\n",
       " 0.2703072883445891,\n",
       " 0.12720938424123754,\n",
       " -0.33326206471152814,\n",
       " 0.16889405757353648,\n",
       " -0.3140938284857856,\n",
       " -0.37481080519333093,\n",
       " -0.21186094775630676,\n",
       " 0.1863225700960763,\n",
       " 0.6352650776855284,\n",
       " 0.009465606066627164,\n",
       " -0.339293548661533,\n",
       " -0.3518933181004176,\n",
       " -1.1125022716569055,\n",
       " -0.14259218963734865,\n",
       " 0.14788576828714373,\n",
       " 0.7595592930799276,\n",
       " 0.6919780840119071,\n",
       " 0.15411558514286705,\n",
       " 0.2900289184871978,\n",
       " 0.553757017053721,\n",
       " -0.18323128607545075,\n",
       " -0.5495919543550911,\n",
       " -0.23201773890653757,\n",
       " 0.4318930678531204,\n",
       " 0.10143065536416163,\n",
       " -0.0042212077552959415,\n",
       " 0.4873559674990759,\n",
       " 0.17674877559373942,\n",
       " -0.09455726166890441,\n",
       " -0.7569593835953842,\n",
       " -0.7598965331320315,\n",
       " -0.36687946682936456,\n",
       " -0.6464774533365378,\n",
       " -0.23677714571419417,\n",
       " 0.7021352829945132,\n",
       " 1.0946836675511906,\n",
       " 0.24556084096447917,\n",
       " -0.027307730774325745,\n",
       " 0.057067783639089056,\n",
       " -0.20126478581625787,\n",
       " 0.13394284421469071,\n",
       " 0.3020716489721653,\n",
       " 0.08040870852156576,\n",
       " 0.08971790843517521,\n",
       " -0.23448911642183573,\n",
       " -0.016896227032715982,\n",
       " -0.5384014644645445,\n",
       " -0.4645897339007209,\n",
       " -0.37625667464407125,\n",
       " -0.5110172929584659,\n",
       " -0.30577977190081707,\n",
       " 0.027665701345575167,\n",
       " -0.4722306099772451,\n",
       " -0.8121719706471251,\n",
       " -0.34345387097416474,\n",
       " 0.5019171168725503,\n",
       " 0.3514497501148536,\n",
       " -0.018411998257352125,\n",
       " -0.13998786920946152,\n",
       " -0.061618096801118113,\n",
       " -0.13182381375247104,\n",
       " -0.12873459893300673,\n",
       " 0.07673494669118316,\n",
       " -0.5902525358279046,\n",
       " -0.6846175948150557,\n",
       " -0.06297168831922514,\n",
       " -0.217295419381974,\n",
       " -0.035039138304154754,\n",
       " 0.4291580435418585,\n",
       " 0.5719535409950854,\n",
       " 0.24608002911855084,\n",
       " 0.015700443964573797,\n",
       " -0.36556711455743435,\n",
       " -0.4949599584608182,\n",
       " -0.17786576776281598,\n",
       " 0.3473370080731726,\n",
       " -0.3319947186200046,\n",
       " -0.520192997535266,\n",
       " -0.40220822688667573,\n",
       " -0.009353640994567272,\n",
       " 0.46393416619530525,\n",
       " 0.724609199039052,\n",
       " -0.11618801073446816,\n",
       " 0.11102315442479949,\n",
       " 0.26384333499046075,\n",
       " 0.2802682754607363,\n",
       " -0.07562433359832224,\n",
       " -0.7075552252972307,\n",
       " -0.29587211425825943,\n",
       " -0.16942939918968308,\n",
       " -0.3456080929194921,\n",
       " -0.007874895013581185,\n",
       " -0.057801416053389426,\n",
       " -0.09341363831216373,\n",
       " -0.18367225314952876,\n",
       " 0.2513973900368972,\n",
       " -0.37147217862879667,\n",
       " -0.3581969252982219,\n",
       " -0.07358218229566282,\n",
       " -0.19536226691677513,\n",
       " -0.13535626932691755,\n",
       " 0.10957456392504437,\n",
       " 0.45929422262242725,\n",
       " 0.5395387393274833,\n",
       " 0.35101774817912607,\n",
       " 0.1995169463813648,\n",
       " -1.0649025381104402,\n",
       " -1.0254383765220578,\n",
       " -0.5335102656221854,\n",
       " -0.29085441314784954,\n",
       " -0.05104138198243452,\n",
       " 0.39753414463272346,\n",
       " -0.06205011228343377,\n",
       " -0.1558947173977356,\n",
       " -0.05768457405695015,\n",
       " 0.22872303222455825,\n",
       " 0.3789356932327666,\n",
       " 0.45800121447366693,\n",
       " -0.04123172456473842,\n",
       " -0.638297779217931,\n",
       " -0.002494427871365945,\n",
       " -0.2511458707960837,\n",
       " -0.5986010550208827,\n",
       " -0.8422592005595788,\n",
       " -0.2470220970993653,\n",
       " 0.3696455325884632,\n",
       " 0.42209089646684095,\n",
       " 0.17172897154050942,\n",
       " -0.5661303601491209,\n",
       " -0.4233057553950334,\n",
       " -0.6647594752729751,\n",
       " 0.05127122442454729,\n",
       " 0.1945002403098572,\n",
       " 0.39049006772856504,\n",
       " 0.26777430289685134,\n",
       " 0.21761157212570015,\n",
       " -0.30704810383112685,\n",
       " -0.3144433247038808,\n",
       " -0.028720772867947164,\n",
       " -0.4559133351776111,\n",
       " 0.1099531295061894,\n",
       " 0.30015962503433313,\n",
       " 0.13450412573911222,\n",
       " 0.05888716705225572,\n",
       " 0.002777285804833527,\n",
       " -0.40071320726078047,\n",
       " -0.16341944680039064,\n",
       " 0.002921113855564768,\n",
       " 0.021850679430165873,\n",
       " -0.10454999335680822,\n",
       " 0.09360941001153,\n",
       " 0.955867272674184,\n",
       " 0.3787953991198918,\n",
       " -0.2717172844494078,\n",
       " -0.47667232742332305,\n",
       " -0.21033182795128616,\n",
       " 0.37972964046189106,\n",
       " 0.04680431640701338,\n",
       " -0.031747705423981115,\n",
       " 0.10563389962385238,\n",
       " -0.19998495347826956,\n",
       " 0.18386549097039478,\n",
       " -0.04262383903905331,\n",
       " 0.1258473332709012,\n",
       " 0.22601734142131413,\n",
       " -0.15369232048270592,\n",
       " -0.1985099071460952,\n",
       " -0.6001046378387707,\n",
       " -0.01488692066264341,\n",
       " 0.07563500018493638,\n",
       " 0.1662854071144774,\n",
       " 0.08764221405867559,\n",
       " -0.14001350524823153,\n",
       " -0.3231350481749194,\n",
       " -0.12239551154676513,\n",
       " 0.11753997311523487,\n",
       " 0.1050082506636518,\n",
       " 0.1507662242280531,\n",
       " -0.363785163662089,\n",
       " -0.22410556883302882,\n",
       " -0.048267224650126894,\n",
       " 0.557536811694929,\n",
       " 0.6153001882068808,\n",
       " 0.5845411509876395,\n",
       " -0.07938861028102026,\n",
       " -0.20728710874915696,\n",
       " -0.31793230553269114,\n",
       " -0.1805132029039194,\n",
       " 0.12146894393398244,\n",
       " 0.0169336116595038,\n",
       " -0.09363151004240713,\n",
       " -0.260593997496255,\n",
       " -0.34897629457212215,\n",
       " -0.27260629852754237,\n",
       " -0.5931575850213111,\n",
       " -0.3163306564971526,\n",
       " -0.15847449665942143,\n",
       " -0.05565381067979238,\n",
       " -0.05272891024903395,\n",
       " 0.11003999692037045,\n",
       " 0.5715655003548187,\n",
       " 0.4604094772171639,\n",
       " -0.10507626733979053,\n",
       " -0.3231258109994294,\n",
       " 0.12583011474989392,\n",
       " 0.0400436797037059,\n",
       " 0.3253933033168329,\n",
       " -0.21362215897362516,\n",
       " -0.00801270838631113,\n",
       " -0.19126163266314633,\n",
       " -0.047272916687634475,\n",
       " -0.13757960983900686,\n",
       " -0.026967874679801412,\n",
       " -0.5808946974601809,\n",
       " -0.5157329687123129,\n",
       " 0.133436002588238,\n",
       " 0.6808846126222807,\n",
       " 0.29230461873114927,\n",
       " -0.05795963533052223,\n",
       " -0.5836164847917176,\n",
       " -0.215046493473214,\n",
       " 0.5197291747761186,\n",
       " 0.8942608405675234,\n",
       " 0.6234296166632812,\n",
       " 0.4013110409227162,\n",
       " 0.23485452228940146,\n",
       " -0.11357089921995167,\n",
       " -0.062846361338102,\n",
       " -0.02659865613088101,\n",
       " -0.09410206250284248,\n",
       " 0.06736866387797526,\n",
       " -0.22754647441240128,\n",
       " -0.37029956890937726,\n",
       " -0.4085947918148475,\n",
       " -0.07261750501061365,\n",
       " 0.05094452935398107,\n",
       " 0.7270493581447236,\n",
       " 0.630295930249067,\n",
       " 0.3305252746361374,\n",
       " -0.2553406979367293,\n",
       " 0.10403739068623197,\n",
       " 0.2603229418231229,\n",
       " 0.6015217419732402,\n",
       " 0.6514567670309146,\n",
       " 0.6492145941340003,\n",
       " 0.3403929057388956,\n",
       " 0.23819853370552044,\n",
       " -0.06258276659539673,\n",
       " -0.4063920348970463,\n",
       " -0.034407146536065636,\n",
       " 0.4479238608383832,\n",
       " 0.6985841345749122,\n",
       " 0.22960604542162702,\n",
       " -0.37326576165711156,\n",
       " -0.4350988465496878,\n",
       " 0.25471729252118214,\n",
       " 0.5191636026010662,\n",
       " -0.02727982842308102,\n",
       " -0.29221752675657375,\n",
       " -0.44799092826318765,\n",
       " -0.27572036185780474,\n",
       " 0.26428085629848075,\n",
       " 0.35381896292287907,\n",
       " 0.47313596911472167,\n",
       " 0.3449447877286659,\n",
       " -0.013506502121467354,\n",
       " -0.0230790580889165,\n",
       " 0.002166746774890569,\n",
       " 0.08362445842739044,\n",
       " -0.10217595131817564,\n",
       " -0.22965320142003423,\n",
       " -1.016443975816469,\n",
       " -0.16720338541934054,\n",
       " 0.29829058333249703,\n",
       " 0.7807620398425784,\n",
       " 0.06226093490945667,\n",
       " 0.08697719579957047,\n",
       " -0.3106062836352077,\n",
       " 0.10396180486251372,\n",
       " 0.6791362906996015,\n",
       " 0.34271690382495684,\n",
       " -0.2960668450231526,\n",
       " -0.49390459307177476,\n",
       " -0.7942804552525596,\n",
       " -0.4938134432610523,\n",
       " 0.4406921127237003,\n",
       " 0.44666083094563186,\n",
       " 0.2533422316330953,\n",
       " 0.5854872324972098,\n",
       " -0.3363681030350385,\n",
       " -0.06235102374825552,\n",
       " 0.6471776982432109,\n",
       " 0.22913331386535551,\n",
       " -0.6748613299063614,\n",
       " -0.5636735815301435,\n",
       " -0.257152936784257,\n",
       " -0.2725636566965183,\n",
       " -0.12171704824439039,\n",
       " -0.24982070420739283,\n",
       " -0.2901334452638413,\n",
       " -0.040728555562726476,\n",
       " 0.18402408133002407,\n",
       " 0.17482762715876218,\n",
       " 0.3933495425075795,\n",
       " 0.08444439856939573,\n",
       " 0.2987517933812384,\n",
       " -0.1361916016568685,\n",
       " -0.7039546409066053,\n",
       " -0.561610882345489,\n",
       " -0.22585761790869283,\n",
       " -0.27324138503595063,\n",
       " -0.22081571705750283,\n",
       " -0.3300163296726813,\n",
       " -0.2615433947971276,\n",
       " 0.18384399880506433,\n",
       " 0.6063605903845423,\n",
       " 0.5671388902283092,\n",
       " 0.45118458080120727,\n",
       " -0.0057323792609499935,\n",
       " 0.2212438284488508,\n",
       " 0.43266879781205736,\n",
       " 0.5818329933953221,\n",
       " 0.327658209518181,\n",
       " -0.07639763927654288,\n",
       " -0.5006291892116452,\n",
       " -0.42128700102704764,\n",
       " -0.358946170212602,\n",
       " -0.4065688848264948,\n",
       " -0.08593884178901755,\n",
       " 0.4375722501838705,\n",
       " 0.5173723371133987,\n",
       " 0.5754176913466129,\n",
       " 0.07082825230989911,\n",
       " -0.25772847615591205,\n",
       " 0.3239953612495814,\n",
       " 0.15239106063426303,\n",
       " -0.05482659531197898,\n",
       " -0.10943769745043609,\n",
       " -0.16174784520484864,\n",
       " 0.009470238094142394,\n",
       " -0.1205743236306782,\n",
       " -0.22095454758029465,\n",
       " -0.057988897115682586,\n",
       " 0.1192954773932784,\n",
       " 0.10721466803679605,\n",
       " 0.7379114719690895,\n",
       " 0.2520345703869184,\n",
       " 0.17893487076838938,\n",
       " 0.3149363206837971,\n",
       " -0.17284581285348932,\n",
       " -0.48116872669404964,\n",
       " -0.6190167447057142,\n",
       " -0.6826529433444861,\n",
       " -0.8093825604055791,\n",
       " -0.7300565673929068,\n",
       " -0.6553070359670805,\n",
       " -0.5154858545829859,\n",
       " -0.17727800125016902,\n",
       " 0.1513799651682577,\n",
       " 0.5649196106321555,\n",
       " -0.6156045867777228,\n",
       " -0.2852538069039932,\n",
       " -0.32185310175894544,\n",
       " -0.17182138432345742,\n",
       " -0.5675769104018876,\n",
       " -0.4566436667730579,\n",
       " 0.043347777740450205,\n",
       " -0.2032366491451511,\n",
       " -0.28435355769603404,\n",
       " -0.3424427197747788,\n",
       " -0.3049216351011099,\n",
       " -0.2961275449657329,\n",
       " -0.2395220103729334,\n",
       " -0.3654651698933734,\n",
       " -0.09902264459489513,\n",
       " -0.21332779314639916,\n",
       " -0.21089826789112603,\n",
       " 0.36640166167400234,\n",
       " 0.41832591047815587,\n",
       " 0.0820848104689015,\n",
       " 0.48229547523468586,\n",
       " 0.583740186050406,\n",
       " 1.040519123725995,\n",
       " 0.5713806620117725,\n",
       " 0.12267781555168505,\n",
       " -0.15807073233508576,\n",
       " 0.011084362400579917,\n",
       " -0.31356913592940433,\n",
       " -0.16902593061729562,\n",
       " -0.8609467075320187,\n",
       " -0.12537150599604413,\n",
       " 0.32789144267838843,\n",
       " 0.018555769343647832,\n",
       " 0.16373536527895632,\n",
       " -0.36885769319685385,\n",
       " -1.072800132295376,\n",
       " -1.043298509723896,\n",
       " -0.3846546219060518,\n",
       " -0.24073877342152006,\n",
       " 0.7561186730171829,\n",
       " 0.6076314355885316,\n",
       " 0.3769580867516493,\n",
       " 0.19314988491995982,\n",
       " -0.06321580898377181,\n",
       " 0.3504171124035285,\n",
       " 0.9497701016765165,\n",
       " 0.7016552105047688,\n",
       " 0.37945135439600813,\n",
       " 0.5065715057242524,\n",
       " 0.20501355564396065,\n",
       " -0.06683714711102708,\n",
       " -0.3342670345157076,\n",
       " 0.4426751327125939,\n",
       " 0.7400254975977034,\n",
       " 0.6410856975530198,\n",
       " 0.6945322129452123,\n",
       " 0.46802991280298056,\n",
       " 0.44247142356825614,\n",
       " -0.003570967985261936,\n",
       " 0.004319321057180858,\n",
       " -0.5256719930242899,\n",
       " -0.43326092004861844,\n",
       " -0.7508317023347091,\n",
       " -0.29901991430365227,\n",
       " 0.2762311456162695,\n",
       " 0.7697837805845258,\n",
       " 0.06678249039966305,\n",
       " -0.3117516821779916,\n",
       " -0.48421944774288705,\n",
       " -0.11191296688094757,\n",
       " 0.0534451883772007,\n",
       " -0.39392672487842695,\n",
       " -0.19797702505174294,\n",
       " 0.3697550847018891,\n",
       " 1.1083479654174904,\n",
       " 1.1146555844602302,\n",
       " 0.8194510222063036,\n",
       " 0.47522444342400605,\n",
       " -0.04099212572914994,\n",
       " 0.26394232746086266,\n",
       " -0.09943267638708667,\n",
       " 0.16355251827396844,\n",
       " 0.14445977376127767,\n",
       " 0.49212067596958603,\n",
       " 0.40484540219572474,\n",
       " 0.046536177712018456,\n",
       " -0.27981416708588014,\n",
       " -0.7503883220735151,\n",
       " -0.5675806021167872,\n",
       " -0.20172856402151443,\n",
       " 0.11827608067912115,\n",
       " -0.05204725757099257,\n",
       " -0.34921368312425605,\n",
       " -0.10743140211320765,\n",
       " -0.12759424393104543,\n",
       " 0.059022256809171655,\n",
       " 0.2899298553141707,\n",
       " 0.36135546290989234,\n",
       " 0.3147350725002815,\n",
       " 0.20903217239461358,\n",
       " -0.3683008807215963,\n",
       " -0.26577612210676804,\n",
       " -0.28576585548716915,\n",
       " 0.10787112352545757,\n",
       " -0.1967746050503233,\n",
       " -0.3112228980869136,\n",
       " 0.09275083893502295,\n",
       " 0.13627994410171365,\n",
       " 0.7580606314878866,\n",
       " 0.6993665024705953,\n",
       " 0.4762970045187993,\n",
       " -0.33966694065818565,\n",
       " -0.7350903831230626,\n",
       " -0.025592321321282363,\n",
       " 0.48121295457783264,\n",
       " 0.13849121635701295,\n",
       " -0.1874952816141705,\n",
       " 0.06604538179652969,\n",
       " -0.16474663439034132,\n",
       " -0.10379983598310039,\n",
       " -0.05776811883848986,\n",
       " 0.24318812647374538,\n",
       " 0.2228818213719609,\n",
       " -0.12841245557409484,\n",
       " -0.24118291735033984,\n",
       " -1.0317812440566407,\n",
       " -0.4290307034944275,\n",
       " 0.8673648425493539,\n",
       " 0.7731854260058625,\n",
       " -0.5160766017417302,\n",
       " -0.5741321993232891,\n",
       " -0.3615571966009093,\n",
       " -0.511231551560621,\n",
       " 0.042189610847622405,\n",
       " -0.08633510207301115,\n",
       " 0.2612413373622907,\n",
       " -0.45750586748020977,\n",
       " -1.0023737841876676,\n",
       " -0.7376391798113158,\n",
       " -0.43079575676264054,\n",
       " 0.42485703745817555,\n",
       " 0.9096866867863342,\n",
       " 1.090880156622764,\n",
       " 0.7373542794206077,\n",
       " 0.451010145176262,\n",
       " 0.253795251724273,\n",
       " 0.08152130102450919,\n",
       " -0.2437674523968344,\n",
       " -0.10758505101864542,\n",
       " -0.08973882176864202,\n",
       " -0.2868932937748967,\n",
       " -0.3832998660924656,\n",
       " ...]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "init_x = 4\n",
    "\n",
    "import random\n",
    "values = [init_x, init_x]\n",
    "N = 1000\n",
    "\n",
    "b0 = 0.8\n",
    "b1 = -0.4\n",
    "noise = dnorm(0, 0.1, N)\n",
    "for i in range(N):\n",
    "    new_x = values[-1] * b0 + values[-2] * b1 + noise[i]\n",
    "    values.append(new_x)\n",
    "values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAXLAGS = 5\n",
    "model = sm.tsa.AR(values)\n",
    "results = model.fit(MAXLAGS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.0102,  0.7318, -0.32  , -0.0672,  0.0067, -0.0143])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ARIMA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1.7373505599869747,\n",
       " 2.5481161158559855,\n",
       " 3.33606405931503,\n",
       " 4.448390104967511,\n",
       " 5.497359792437154,\n",
       " 6.0561714533624,\n",
       " 7.820550906090741,\n",
       " 8.650661573083365,\n",
       " 9.95924356789412,\n",
       " 10.481055366109572,\n",
       " 11.22180278000399,\n",
       " 12.020075918975296,\n",
       " 13.85013567264662,\n",
       " 14.783204427666789,\n",
       " 15.15198950297594,\n",
       " 16.351774112776578,\n",
       " 17.39204834594581,\n",
       " 18.500878783692354,\n",
       " 19.48792821143781,\n",
       " 20.10497071454212,\n",
       " 21.72421013493498,\n",
       " 22.990266129633653,\n",
       " 23.171681004301792,\n",
       " 24.59527095930655,\n",
       " 25.31842219125594,\n",
       " 26.296086741605258,\n",
       " 27.14399794391812,\n",
       " 28.163595082139302,\n",
       " 29.91132727100939,\n",
       " 30.097450276253287,\n",
       " 31.846654883800735,\n",
       " 32.988995790123106,\n",
       " 33.008250901275616,\n",
       " 34.84100338384953,\n",
       " 35.6029967263188,\n",
       " 36.568561171568525,\n",
       " 37.07299420856423,\n",
       " 38.91554435880022,\n",
       " 39.59639870776305,\n",
       " 40.69682325690507,\n",
       " 41.70533695956636,\n",
       " 42.48181339946387,\n",
       " 43.0692905796393,\n",
       " 44.920654779337774,\n",
       " 45.540238640113046,\n",
       " 46.96134434285732,\n",
       " 47.293722444984695,\n",
       " 48.294719551133035,\n",
       " 49.041379487876476,\n",
       " 50.62571025713431,\n",
       " 51.32111256556512,\n",
       " 52.14222069687793,\n",
       " 53.56383122892908,\n",
       " 54.93686542623605,\n",
       " 55.57296616449628,\n",
       " 56.077472190442734,\n",
       " 57.10378900125024,\n",
       " 58.52310621102212,\n",
       " 59.882977426607546,\n",
       " 60.72817088182959,\n",
       " 61.13126490185861,\n",
       " 62.486236889069644,\n",
       " 63.25021346147857,\n",
       " 64.22782537763396,\n",
       " 65.3673525248497,\n",
       " 66.95939875625024,\n",
       " 67.9594456164814,\n",
       " 68.85859908559058,\n",
       " 69.59876493559848,\n",
       " 70.50103330550745,\n",
       " 71.88864339191394,\n",
       " 72.07420310863257,\n",
       " 73.46377275059517,\n",
       " 74.7946977909743,\n",
       " 75.44549899612903,\n",
       " 76.02082325033825,\n",
       " 77.84821788530557,\n",
       " 78.16251237230259,\n",
       " 79.79061834260084,\n",
       " 80.4644334340484,\n",
       " 81.05806059788803,\n",
       " 82.64254973014005,\n",
       " 83.9965131740354,\n",
       " 84.85115505773871,\n",
       " 85.78854651469233,\n",
       " 86.51916577124403,\n",
       " 87.94175389892837,\n",
       " 88.99676158320229,\n",
       " 89.52072200285426,\n",
       " 90.50635152631429,\n",
       " 91.46590729556502,\n",
       " 92.92168398308232,\n",
       " 93.94452764792518,\n",
       " 94.42895530522823,\n",
       " 95.2321960861151,\n",
       " 96.17720518247256,\n",
       " 97.53315825782241,\n",
       " 98.55519261343846,\n",
       " 99.28613244459126]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from statsmodels.tsa.arima.model import ARIMA\n",
    "from random import random\n",
    "# contrived dataset\n",
    "data = [x + random() for x in range(1, 100)]\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\python\\anaconda3\\lib\\site-packages\\statsmodels\\tsa\\statespace\\sarimax.py:963: UserWarning: Non-stationary starting autoregressive parameters found. Using zeros as starting parameters.\n",
      "  warn('Non-stationary starting autoregressive parameters'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[100.2796]\n"
     ]
    }
   ],
   "source": [
    "# fit model\n",
    "model = ARIMA(data, order=(1, 1, 1))\n",
    "model_fit = model.fit()\n",
    "# make prediction\n",
    "yhat = model_fit.predict(len(data), len(data), typ='levels')\n",
    "print(yhat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Introduction to scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "\n",
       "                                                Name     Sex   Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male  22.0      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "2                             Heikkinen, Miss. Laina  female  26.0      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  \n",
       "0      0         A/5 21171   7.2500   NaN        S  \n",
       "1      0          PC 17599  71.2833   C85        C  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3      0            113803  53.1000  C123        S  "
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('datasets/titanic/train.csv')\n",
    "test = pd.read_csv('datasets/titanic/test.csv')\n",
    "train[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PassengerId      0\n",
       "Survived         0\n",
       "Pclass           0\n",
       "Name             0\n",
       "Sex              0\n",
       "Age            177\n",
       "SibSp            0\n",
       "Parch            0\n",
       "Ticket           0\n",
       "Fare             0\n",
       "Cabin          687\n",
       "Embarked         2\n",
       "dtype: int64"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PassengerId      0\n",
       "Pclass           0\n",
       "Name             0\n",
       "Sex              0\n",
       "Age             86\n",
       "SibSp            0\n",
       "Parch            0\n",
       "Ticket           0\n",
       "Fare             1\n",
       "Cabin          327\n",
       "Embarked         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "impute_value = train['Age'].median()\n",
    "train['Age'] = train['Age'].fillna(impute_value)\n",
    "test['Age'] = test['Age'].fillna(impute_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "      <th>IsFemale</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name     Sex   Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male  22.0      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "2                             Heikkinen, Miss. Laina  female  26.0      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1   \n",
       "4                           Allen, Mr. William Henry    male  35.0      0   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  IsFemale  \n",
       "0      0         A/5 21171   7.2500   NaN        S         0  \n",
       "1      0          PC 17599  71.2833   C85        C         1  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S         1  \n",
       "3      0            113803  53.1000  C123        S         1  \n",
       "4      0            373450   8.0500   NaN        S         0  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['IsFemale'] = (train['Sex'] == 'female').astype(int)\n",
    "test['IsFemale'] = (test['Sex'] == 'female').astype(int)\n",
    "train[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictors = ['Pclass', 'IsFemale', 'Age']\n",
    "X_train = train[predictors].values\n",
    "X_test = test[predictors].values\n",
    "y_train = train['Survived'].values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 3.,  0., 22.],\n",
       "       [ 1.,  1., 38.],\n",
       "       [ 3.,  1., 26.],\n",
       "       [ 1.,  1., 35.],\n",
       "       [ 3.,  0., 35.]])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 1, 1, 0], dtype=int64)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "model = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='auto', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 1, 0, 1, 0, 1, 0], dtype=int64)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_predict = model.predict(X_test)\n",
    "y_predict[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(y_true == y_predict).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "model_cv = LogisticRegressionCV(10)\n",
    "model_cv.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "model = LogisticRegression(C=10)\n",
    "scores = cross_val_score(model, X_train, y_train, cv=4)\n",
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Continuing Your Education"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.options.display.max_rows = PREVIOUS_MAX_ROWS"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}