project_XGBoost.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>vendor_id</th>\n",
       "      <th>pickup_datetime</th>\n",
       "      <th>dropoff_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>trip_duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id2875421</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-14 17:24:55</td>\n",
       "      <td>2016-03-14 17:32:30</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.982155</td>\n",
       "      <td>40.767937</td>\n",
       "      <td>-73.964630</td>\n",
       "      <td>40.765602</td>\n",
       "      <td>N</td>\n",
       "      <td>455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id2377394</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-12 00:43:35</td>\n",
       "      <td>2016-06-12 00:54:38</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.980415</td>\n",
       "      <td>40.738564</td>\n",
       "      <td>-73.999481</td>\n",
       "      <td>40.731152</td>\n",
       "      <td>N</td>\n",
       "      <td>663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id3858529</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-01-19 11:35:24</td>\n",
       "      <td>2016-01-19 12:10:48</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.979027</td>\n",
       "      <td>40.763939</td>\n",
       "      <td>-74.005333</td>\n",
       "      <td>40.710087</td>\n",
       "      <td>N</td>\n",
       "      <td>2124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id3504673</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-04-06 19:32:31</td>\n",
       "      <td>2016-04-06 19:39:40</td>\n",
       "      <td>1</td>\n",
       "      <td>-74.010040</td>\n",
       "      <td>40.719971</td>\n",
       "      <td>-74.012268</td>\n",
       "      <td>40.706718</td>\n",
       "      <td>N</td>\n",
       "      <td>429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id2181028</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-26 13:30:55</td>\n",
       "      <td>2016-03-26 13:38:10</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.973053</td>\n",
       "      <td>40.793209</td>\n",
       "      <td>-73.972923</td>\n",
       "      <td>40.782520</td>\n",
       "      <td>N</td>\n",
       "      <td>435</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  vendor_id      pickup_datetime     dropoff_datetime  \\\n",
       "0  id2875421          2  2016-03-14 17:24:55  2016-03-14 17:32:30   \n",
       "1  id2377394          1  2016-06-12 00:43:35  2016-06-12 00:54:38   \n",
       "2  id3858529          2  2016-01-19 11:35:24  2016-01-19 12:10:48   \n",
       "3  id3504673          2  2016-04-06 19:32:31  2016-04-06 19:39:40   \n",
       "4  id2181028          2  2016-03-26 13:30:55  2016-03-26 13:38:10   \n",
       "\n",
       "   passenger_count  pickup_longitude  pickup_latitude  dropoff_longitude  \\\n",
       "0                1        -73.982155        40.767937         -73.964630   \n",
       "1                1        -73.980415        40.738564         -73.999481   \n",
       "2                1        -73.979027        40.763939         -74.005333   \n",
       "3                1        -74.010040        40.719971         -74.012268   \n",
       "4                1        -73.973053        40.793209         -73.972923   \n",
       "\n",
       "   dropoff_latitude store_and_fwd_flag  trip_duration  \n",
       "0         40.765602                  N            455  \n",
       "1         40.731152                  N            663  \n",
       "2         40.710087                  N           2124  \n",
       "3         40.706718                  N            429  \n",
       "4         40.782520                  N            435  "
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(\"train.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_normal = train[(train['pickup_longitude'] > -74.023822) & (train['pickup_latitude'] < 40.886345) & (train['pickup_longitude'] < -73.901599) & (train['pickup_latitude'] > 40.698160)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sns.regplot(x = 'pickup_longitude', y = 'pickup_latitude', data = train_normal, scatter = True, scatter_kws={\"s\": 10})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    1033540\n",
       "2     210318\n",
       "5      78088\n",
       "3      59896\n",
       "6      48333\n",
       "4      28404\n",
       "0         60\n",
       "7          3\n",
       "9          1\n",
       "8          1\n",
       "Name: passenger_count, dtype: int64"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['passenger_count'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2    780302\n",
       "1    678342\n",
       "Name: vendor_id, dtype: int64"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['vendor_id'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "N    1450599\n",
       "Y       8045\n",
       "Name: store_and_fwd_flag, dtype: int64"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['store_and_fwd_flag'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.020304177490054781"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['vendor_id'].corr(train['trip_duration'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                    0\n",
       "vendor_id             0\n",
       "pickup_datetime       0\n",
       "dropoff_datetime      0\n",
       "passenger_count       0\n",
       "pickup_longitude      0\n",
       "pickup_latitude       0\n",
       "dropoff_longitude     0\n",
       "dropoff_latitude      0\n",
       "store_and_fwd_flag    0\n",
       "trip_duration         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                     object\n",
       "vendor_id               int64\n",
       "pickup_datetime        object\n",
       "dropoff_datetime       object\n",
       "passenger_count         int64\n",
       "pickup_longitude      float64\n",
       "pickup_latitude       float64\n",
       "dropoff_longitude     float64\n",
       "dropoff_latitude      float64\n",
       "store_and_fwd_flag     object\n",
       "trip_duration           int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                            object\n",
       "vendor_id                      int64\n",
       "pickup_datetime       datetime64[ns]\n",
       "dropoff_datetime      datetime64[ns]\n",
       "passenger_count                int64\n",
       "pickup_longitude             float64\n",
       "pickup_latitude              float64\n",
       "dropoff_longitude            float64\n",
       "dropoff_latitude             float64\n",
       "store_and_fwd_flag            object\n",
       "trip_duration                  int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['pickup_datetime'] = pd.to_datetime(train['pickup_datetime'])\n",
    "train['dropoff_datetime'] = pd.to_datetime(train['dropoff_datetime'])\n",
    "train.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Month and hour are the only features I care about"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>vendor_id</th>\n",
       "      <th>pickup_datetime</th>\n",
       "      <th>dropoff_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>trip_duration</th>\n",
       "      <th>pickup_hour</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id2875421</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-14 17:24:55</td>\n",
       "      <td>2016-03-14 17:32:30</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.982155</td>\n",
       "      <td>40.767937</td>\n",
       "      <td>-73.964630</td>\n",
       "      <td>40.765602</td>\n",
       "      <td>N</td>\n",
       "      <td>455</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id2377394</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-12 00:43:35</td>\n",
       "      <td>2016-06-12 00:54:38</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.980415</td>\n",
       "      <td>40.738564</td>\n",
       "      <td>-73.999481</td>\n",
       "      <td>40.731152</td>\n",
       "      <td>N</td>\n",
       "      <td>663</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id3858529</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-01-19 11:35:24</td>\n",
       "      <td>2016-01-19 12:10:48</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.979027</td>\n",
       "      <td>40.763939</td>\n",
       "      <td>-74.005333</td>\n",
       "      <td>40.710087</td>\n",
       "      <td>N</td>\n",
       "      <td>2124</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id3504673</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-04-06 19:32:31</td>\n",
       "      <td>2016-04-06 19:39:40</td>\n",
       "      <td>1</td>\n",
       "      <td>-74.010040</td>\n",
       "      <td>40.719971</td>\n",
       "      <td>-74.012268</td>\n",
       "      <td>40.706718</td>\n",
       "      <td>N</td>\n",
       "      <td>429</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id2181028</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-26 13:30:55</td>\n",
       "      <td>2016-03-26 13:38:10</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.973053</td>\n",
       "      <td>40.793209</td>\n",
       "      <td>-73.972923</td>\n",
       "      <td>40.782520</td>\n",
       "      <td>N</td>\n",
       "      <td>435</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  vendor_id     pickup_datetime    dropoff_datetime  \\\n",
       "0  id2875421          2 2016-03-14 17:24:55 2016-03-14 17:32:30   \n",
       "1  id2377394          1 2016-06-12 00:43:35 2016-06-12 00:54:38   \n",
       "2  id3858529          2 2016-01-19 11:35:24 2016-01-19 12:10:48   \n",
       "3  id3504673          2 2016-04-06 19:32:31 2016-04-06 19:39:40   \n",
       "4  id2181028          2 2016-03-26 13:30:55 2016-03-26 13:38:10   \n",
       "\n",
       "   passenger_count  pickup_longitude  pickup_latitude  dropoff_longitude  \\\n",
       "0                1        -73.982155        40.767937         -73.964630   \n",
       "1                1        -73.980415        40.738564         -73.999481   \n",
       "2                1        -73.979027        40.763939         -74.005333   \n",
       "3                1        -74.010040        40.719971         -74.012268   \n",
       "4                1        -73.973053        40.793209         -73.972923   \n",
       "\n",
       "   dropoff_latitude store_and_fwd_flag  trip_duration  pickup_hour  \n",
       "0         40.765602                  N            455           17  \n",
       "1         40.731152                  N            663            0  \n",
       "2         40.710087                  N           2124           11  \n",
       "3         40.706718                  N            429           19  \n",
       "4         40.782520                  N            435           13  "
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['pickup_hour'] = train['pickup_datetime'].dt.hour\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>vendor_id</th>\n",
       "      <th>pickup_datetime</th>\n",
       "      <th>dropoff_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>trip_duration</th>\n",
       "      <th>pickup_hour</th>\n",
       "      <th>pickup_month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id2875421</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-14 17:24:55</td>\n",
       "      <td>2016-03-14 17:32:30</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.982155</td>\n",
       "      <td>40.767937</td>\n",
       "      <td>-73.964630</td>\n",
       "      <td>40.765602</td>\n",
       "      <td>N</td>\n",
       "      <td>455</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id2377394</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-12 00:43:35</td>\n",
       "      <td>2016-06-12 00:54:38</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.980415</td>\n",
       "      <td>40.738564</td>\n",
       "      <td>-73.999481</td>\n",
       "      <td>40.731152</td>\n",
       "      <td>N</td>\n",
       "      <td>663</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id3858529</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-01-19 11:35:24</td>\n",
       "      <td>2016-01-19 12:10:48</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.979027</td>\n",
       "      <td>40.763939</td>\n",
       "      <td>-74.005333</td>\n",
       "      <td>40.710087</td>\n",
       "      <td>N</td>\n",
       "      <td>2124</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id3504673</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-04-06 19:32:31</td>\n",
       "      <td>2016-04-06 19:39:40</td>\n",
       "      <td>1</td>\n",
       "      <td>-74.010040</td>\n",
       "      <td>40.719971</td>\n",
       "      <td>-74.012268</td>\n",
       "      <td>40.706718</td>\n",
       "      <td>N</td>\n",
       "      <td>429</td>\n",
       "      <td>19</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id2181028</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-26 13:30:55</td>\n",
       "      <td>2016-03-26 13:38:10</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.973053</td>\n",
       "      <td>40.793209</td>\n",
       "      <td>-73.972923</td>\n",
       "      <td>40.782520</td>\n",
       "      <td>N</td>\n",
       "      <td>435</td>\n",
       "      <td>13</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  vendor_id     pickup_datetime    dropoff_datetime  \\\n",
       "0  id2875421          2 2016-03-14 17:24:55 2016-03-14 17:32:30   \n",
       "1  id2377394          1 2016-06-12 00:43:35 2016-06-12 00:54:38   \n",
       "2  id3858529          2 2016-01-19 11:35:24 2016-01-19 12:10:48   \n",
       "3  id3504673          2 2016-04-06 19:32:31 2016-04-06 19:39:40   \n",
       "4  id2181028          2 2016-03-26 13:30:55 2016-03-26 13:38:10   \n",
       "\n",
       "   passenger_count  pickup_longitude  pickup_latitude  dropoff_longitude  \\\n",
       "0                1        -73.982155        40.767937         -73.964630   \n",
       "1                1        -73.980415        40.738564         -73.999481   \n",
       "2                1        -73.979027        40.763939         -74.005333   \n",
       "3                1        -74.010040        40.719971         -74.012268   \n",
       "4                1        -73.973053        40.793209         -73.972923   \n",
       "\n",
       "   dropoff_latitude store_and_fwd_flag  trip_duration  pickup_hour  \\\n",
       "0         40.765602                  N            455           17   \n",
       "1         40.731152                  N            663            0   \n",
       "2         40.710087                  N           2124           11   \n",
       "3         40.706718                  N            429           19   \n",
       "4         40.782520                  N            435           13   \n",
       "\n",
       "   pickup_month  \n",
       "0             3  \n",
       "1             6  \n",
       "2             1  \n",
       "3             4  \n",
       "4             3  "
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['pickup_month'] = train['pickup_datetime'].dt.month\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['day_of_week'] = train['pickup_datetime'].dt.dayofweek"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## day_of_week, hour and month are categorical data, transform them to dummies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['day_of_week'] = train['day_of_week'].astype('category')\n",
    "day_of_week = pd.get_dummies(train['day_of_week'], prefix='day_of_week')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>day_of_week_0</th>\n",
       "      <th>day_of_week_1</th>\n",
       "      <th>day_of_week_2</th>\n",
       "      <th>day_of_week_3</th>\n",
       "      <th>day_of_week_4</th>\n",
       "      <th>day_of_week_5</th>\n",
       "      <th>day_of_week_6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   day_of_week_0  day_of_week_1  day_of_week_2  day_of_week_3  day_of_week_4  \\\n",
       "0              1              0              0              0              0   \n",
       "1              0              0              0              0              0   \n",
       "2              0              1              0              0              0   \n",
       "3              0              0              1              0              0   \n",
       "4              0              0              0              0              0   \n",
       "\n",
       "   day_of_week_5  day_of_week_6  \n",
       "0              0              0  \n",
       "1              0              1  \n",
       "2              0              0  \n",
       "3              0              0  \n",
       "4              1              0  "
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "day_of_week.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['pickup_hour'] = train['pickup_hour'].astype('category')\n",
    "pickup_hour = pd.get_dummies(train['pickup_hour'], prefix='hour')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['pickup_month'] = train['pickup_month'].astype('category')\n",
    "pickup_month = pd.get_dummies(train['pickup_month'], prefix='month')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dt = [train, pickup_hour, pickup_month, day_of_week]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.concat(dt, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>vendor_id</th>\n",
       "      <th>pickup_datetime</th>\n",
       "      <th>dropoff_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>...</th>\n",
       "      <th>month_4</th>\n",
       "      <th>month_5</th>\n",
       "      <th>month_6</th>\n",
       "      <th>day_of_week_0</th>\n",
       "      <th>day_of_week_1</th>\n",
       "      <th>day_of_week_2</th>\n",
       "      <th>day_of_week_3</th>\n",
       "      <th>day_of_week_4</th>\n",
       "      <th>day_of_week_5</th>\n",
       "      <th>day_of_week_6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id2875421</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-14 17:24:55</td>\n",
       "      <td>2016-03-14 17:32:30</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.982155</td>\n",
       "      <td>40.767937</td>\n",
       "      <td>-73.964630</td>\n",
       "      <td>40.765602</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id2377394</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-12 00:43:35</td>\n",
       "      <td>2016-06-12 00:54:38</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.980415</td>\n",
       "      <td>40.738564</td>\n",
       "      <td>-73.999481</td>\n",
       "      <td>40.731152</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id3858529</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-01-19 11:35:24</td>\n",
       "      <td>2016-01-19 12:10:48</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.979027</td>\n",
       "      <td>40.763939</td>\n",
       "      <td>-74.005333</td>\n",
       "      <td>40.710087</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id3504673</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-04-06 19:32:31</td>\n",
       "      <td>2016-04-06 19:39:40</td>\n",
       "      <td>1</td>\n",
       "      <td>-74.010040</td>\n",
       "      <td>40.719971</td>\n",
       "      <td>-74.012268</td>\n",
       "      <td>40.706718</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id2181028</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-26 13:30:55</td>\n",
       "      <td>2016-03-26 13:38:10</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.973053</td>\n",
       "      <td>40.793209</td>\n",
       "      <td>-73.972923</td>\n",
       "      <td>40.782520</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  vendor_id     pickup_datetime    dropoff_datetime  \\\n",
       "0  id2875421          2 2016-03-14 17:24:55 2016-03-14 17:32:30   \n",
       "1  id2377394          1 2016-06-12 00:43:35 2016-06-12 00:54:38   \n",
       "2  id3858529          2 2016-01-19 11:35:24 2016-01-19 12:10:48   \n",
       "3  id3504673          2 2016-04-06 19:32:31 2016-04-06 19:39:40   \n",
       "4  id2181028          2 2016-03-26 13:30:55 2016-03-26 13:38:10   \n",
       "\n",
       "   passenger_count  pickup_longitude  pickup_latitude  dropoff_longitude  \\\n",
       "0                1        -73.982155        40.767937         -73.964630   \n",
       "1                1        -73.980415        40.738564         -73.999481   \n",
       "2                1        -73.979027        40.763939         -74.005333   \n",
       "3                1        -74.010040        40.719971         -74.012268   \n",
       "4                1        -73.973053        40.793209         -73.972923   \n",
       "\n",
       "   dropoff_latitude store_and_fwd_flag      ...        month_4 month_5  \\\n",
       "0         40.765602                  N      ...              0       0   \n",
       "1         40.731152                  N      ...              0       0   \n",
       "2         40.710087                  N      ...              0       0   \n",
       "3         40.706718                  N      ...              1       0   \n",
       "4         40.782520                  N      ...              0       0   \n",
       "\n",
       "  month_6 day_of_week_0  day_of_week_1  day_of_week_2  day_of_week_3  \\\n",
       "0       0             1              0              0              0   \n",
       "1       1             0              0              0              0   \n",
       "2       0             0              1              0              0   \n",
       "3       0             0              0              1              0   \n",
       "4       0             0              0              0              0   \n",
       "\n",
       "   day_of_week_4  day_of_week_5  day_of_week_6  \n",
       "0              0              0              0  \n",
       "1              0              0              1  \n",
       "2              0              0              0  \n",
       "3              0              0              0  \n",
       "4              0              1              0  \n",
       "\n",
       "[5 rows x 51 columns]"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the difference between Log and Lat of Pickup and Dropoff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>vendor_id</th>\n",
       "      <th>pickup_datetime</th>\n",
       "      <th>dropoff_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>...</th>\n",
       "      <th>month_6</th>\n",
       "      <th>day_of_week_0</th>\n",
       "      <th>day_of_week_1</th>\n",
       "      <th>day_of_week_2</th>\n",
       "      <th>day_of_week_3</th>\n",
       "      <th>day_of_week_4</th>\n",
       "      <th>day_of_week_5</th>\n",
       "      <th>day_of_week_6</th>\n",
       "      <th>longitude_difference</th>\n",
       "      <th>latitude_difference</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id2875421</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-14 17:24:55</td>\n",
       "      <td>2016-03-14 17:32:30</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.982155</td>\n",
       "      <td>40.767937</td>\n",
       "      <td>-73.964630</td>\n",
       "      <td>40.765602</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.017525</td>\n",
       "      <td>-0.002335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id2377394</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-12 00:43:35</td>\n",
       "      <td>2016-06-12 00:54:38</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.980415</td>\n",
       "      <td>40.738564</td>\n",
       "      <td>-73.999481</td>\n",
       "      <td>40.731152</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.019066</td>\n",
       "      <td>-0.007412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id3858529</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-01-19 11:35:24</td>\n",
       "      <td>2016-01-19 12:10:48</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.979027</td>\n",
       "      <td>40.763939</td>\n",
       "      <td>-74.005333</td>\n",
       "      <td>40.710087</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.026306</td>\n",
       "      <td>-0.053852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id3504673</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-04-06 19:32:31</td>\n",
       "      <td>2016-04-06 19:39:40</td>\n",
       "      <td>1</td>\n",
       "      <td>-74.010040</td>\n",
       "      <td>40.719971</td>\n",
       "      <td>-74.012268</td>\n",
       "      <td>40.706718</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.002228</td>\n",
       "      <td>-0.013252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id2181028</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-26 13:30:55</td>\n",
       "      <td>2016-03-26 13:38:10</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.973053</td>\n",
       "      <td>40.793209</td>\n",
       "      <td>-73.972923</td>\n",
       "      <td>40.782520</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000130</td>\n",
       "      <td>-0.010689</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 53 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  vendor_id     pickup_datetime    dropoff_datetime  \\\n",
       "0  id2875421          2 2016-03-14 17:24:55 2016-03-14 17:32:30   \n",
       "1  id2377394          1 2016-06-12 00:43:35 2016-06-12 00:54:38   \n",
       "2  id3858529          2 2016-01-19 11:35:24 2016-01-19 12:10:48   \n",
       "3  id3504673          2 2016-04-06 19:32:31 2016-04-06 19:39:40   \n",
       "4  id2181028          2 2016-03-26 13:30:55 2016-03-26 13:38:10   \n",
       "\n",
       "   passenger_count  pickup_longitude  pickup_latitude  dropoff_longitude  \\\n",
       "0                1        -73.982155        40.767937         -73.964630   \n",
       "1                1        -73.980415        40.738564         -73.999481   \n",
       "2                1        -73.979027        40.763939         -74.005333   \n",
       "3                1        -74.010040        40.719971         -74.012268   \n",
       "4                1        -73.973053        40.793209         -73.972923   \n",
       "\n",
       "   dropoff_latitude store_and_fwd_flag         ...           month_6  \\\n",
       "0         40.765602                  N         ...                 0   \n",
       "1         40.731152                  N         ...                 1   \n",
       "2         40.710087                  N         ...                 0   \n",
       "3         40.706718                  N         ...                 0   \n",
       "4         40.782520                  N         ...                 0   \n",
       "\n",
       "  day_of_week_0 day_of_week_1 day_of_week_2  day_of_week_3  day_of_week_4  \\\n",
       "0             1             0             0              0              0   \n",
       "1             0             0             0              0              0   \n",
       "2             0             1             0              0              0   \n",
       "3             0             0             1              0              0   \n",
       "4             0             0             0              0              0   \n",
       "\n",
       "   day_of_week_5  day_of_week_6  longitude_difference  latitude_difference  \n",
       "0              0              0              0.017525            -0.002335  \n",
       "1              0              1             -0.019066            -0.007412  \n",
       "2              0              0             -0.026306            -0.053852  \n",
       "3              0              0             -0.002228            -0.013252  \n",
       "4              1              0              0.000130            -0.010689  \n",
       "\n",
       "[5 rows x 53 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['longitude_difference'] = train['dropoff_longitude'] - train['pickup_longitude']\n",
    "train['latitude_difference'] = train['dropoff_latitude'] - train['pickup_latitude']\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['id', 'vendor_id', 'pickup_datetime', 'dropoff_datetime',\n",
       "       'passenger_count', 'pickup_longitude', 'pickup_latitude',\n",
       "       'dropoff_longitude', 'dropoff_latitude', 'store_and_fwd_flag',\n",
       "       'trip_duration', 'pickup_hour', 'pickup_month', 'day_of_week', 'hour_0',\n",
       "       'hour_1', 'hour_2', 'hour_3', 'hour_4', 'hour_5', 'hour_6', 'hour_7',\n",
       "       'hour_8', 'hour_9', 'hour_10', 'hour_11', 'hour_12', 'hour_13',\n",
       "       'hour_14', 'hour_15', 'hour_16', 'hour_17', 'hour_18', 'hour_19',\n",
       "       'hour_20', 'hour_21', 'hour_22', 'hour_23', 'month_1', 'month_2',\n",
       "       'month_3', 'month_4', 'month_5', 'month_6', 'day_of_week_0',\n",
       "       'day_of_week_1', 'day_of_week_2', 'day_of_week_3', 'day_of_week_4',\n",
       "       'day_of_week_5', 'day_of_week_6', 'longitude_difference',\n",
       "       'latitude_difference'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                              object\n",
       "vendor_id                        int64\n",
       "pickup_datetime         datetime64[ns]\n",
       "dropoff_datetime        datetime64[ns]\n",
       "passenger_count                  int64\n",
       "pickup_longitude               float64\n",
       "pickup_latitude                float64\n",
       "dropoff_longitude              float64\n",
       "dropoff_latitude               float64\n",
       "store_and_fwd_flag              object\n",
       "trip_duration                    int64\n",
       "pickup_hour                   category\n",
       "pickup_month                  category\n",
       "day_of_week                   category\n",
       "hour_0                           uint8\n",
       "hour_1                           uint8\n",
       "hour_2                           uint8\n",
       "hour_3                           uint8\n",
       "hour_4                           uint8\n",
       "hour_5                           uint8\n",
       "hour_6                           uint8\n",
       "hour_7                           uint8\n",
       "hour_8                           uint8\n",
       "hour_9                           uint8\n",
       "hour_10                          uint8\n",
       "hour_11                          uint8\n",
       "hour_12                          uint8\n",
       "hour_13                          uint8\n",
       "hour_14                          uint8\n",
       "hour_15                          uint8\n",
       "hour_16                          uint8\n",
       "hour_17                          uint8\n",
       "hour_18                          uint8\n",
       "hour_19                          uint8\n",
       "hour_20                          uint8\n",
       "hour_21                          uint8\n",
       "hour_22                          uint8\n",
       "hour_23                          uint8\n",
       "month_1                          uint8\n",
       "month_2                          uint8\n",
       "month_3                          uint8\n",
       "month_4                          uint8\n",
       "month_5                          uint8\n",
       "month_6                          uint8\n",
       "day_of_week_0                    uint8\n",
       "day_of_week_1                    uint8\n",
       "day_of_week_2                    uint8\n",
       "day_of_week_3                    uint8\n",
       "day_of_week_4                    uint8\n",
       "day_of_week_5                    uint8\n",
       "day_of_week_6                    uint8\n",
       "longitude_difference           float64\n",
       "latitude_difference            float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features = ['passenger_count', 'pickup_longitude', 'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude', 'longitude_difference', 'latitude_difference', 'hour_0',\n",
    "       'hour_1', 'hour_2', 'hour_3', 'hour_4', 'hour_5', 'hour_6', 'hour_7',\n",
    "       'hour_8', 'hour_9', 'hour_10', 'hour_11', 'hour_12', 'hour_13',\n",
    "       'hour_14', 'hour_15', 'hour_16', 'hour_17', 'hour_18', 'hour_19',\n",
    "       'hour_20', 'hour_21', 'hour_22', 'hour_23', 'month_1', 'month_2',\n",
    "       'month_3', 'month_4', 'month_5', 'month_6', 'day_of_week_0',\n",
    "       'day_of_week_1', 'day_of_week_2', 'day_of_week_3', 'day_of_week_4',\n",
    "       'day_of_week_5', 'day_of_week_6']\n",
    "target = 'trip_duration'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1458644, 44)"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = train[features]\n",
    "y_train = train[target]\n",
    "X_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fit and Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>vendor_id</th>\n",
       "      <th>pickup_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id3004672</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-30 23:59:58</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.988129</td>\n",
       "      <td>40.732029</td>\n",
       "      <td>-73.990173</td>\n",
       "      <td>40.756680</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id3505355</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-30 23:59:53</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.964203</td>\n",
       "      <td>40.679993</td>\n",
       "      <td>-73.959808</td>\n",
       "      <td>40.655403</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id1217141</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-30 23:59:47</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.997437</td>\n",
       "      <td>40.737583</td>\n",
       "      <td>-73.986160</td>\n",
       "      <td>40.729523</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id2150126</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-06-30 23:59:41</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.956070</td>\n",
       "      <td>40.771900</td>\n",
       "      <td>-73.986427</td>\n",
       "      <td>40.730469</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id1598245</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-30 23:59:33</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.970215</td>\n",
       "      <td>40.761475</td>\n",
       "      <td>-73.961510</td>\n",
       "      <td>40.755890</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  vendor_id      pickup_datetime  passenger_count  \\\n",
       "0  id3004672          1  2016-06-30 23:59:58                1   \n",
       "1  id3505355          1  2016-06-30 23:59:53                1   \n",
       "2  id1217141          1  2016-06-30 23:59:47                1   \n",
       "3  id2150126          2  2016-06-30 23:59:41                1   \n",
       "4  id1598245          1  2016-06-30 23:59:33                1   \n",
       "\n",
       "   pickup_longitude  pickup_latitude  dropoff_longitude  dropoff_latitude  \\\n",
       "0        -73.988129        40.732029         -73.990173         40.756680   \n",
       "1        -73.964203        40.679993         -73.959808         40.655403   \n",
       "2        -73.997437        40.737583         -73.986160         40.729523   \n",
       "3        -73.956070        40.771900         -73.986427         40.730469   \n",
       "4        -73.970215        40.761475         -73.961510         40.755890   \n",
       "\n",
       "  store_and_fwd_flag  \n",
       "0                  N  \n",
       "1                  N  \n",
       "2                  N  \n",
       "3                  N  \n",
       "4                  N  "
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = pd.read_csv(\"test.csv\")\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test['pickup_month'] = test['pickup_datetime'].dt.month\n",
    "test['pickup_hour'] = test['pickup_datetime'].dt.hour\n",
    "test['day_of_week'] = test['pickup_datetime'].dt.dayofweek"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test['longitude_difference'] = test['dropoff_longitude'] - test['pickup_longitude']\n",
    "test['latitude_difference'] = test['dropoff_latitude'] - test['pickup_latitude']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>vendor_id</th>\n",
       "      <th>pickup_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>pickup_month</th>\n",
       "      <th>pickup_hour</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>longitude_difference</th>\n",
       "      <th>latitude_difference</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id3004672</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-30 23:59:58</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.988129</td>\n",
       "      <td>40.732029</td>\n",
       "      <td>-73.990173</td>\n",
       "      <td>40.756680</td>\n",
       "      <td>N</td>\n",
       "      <td>6</td>\n",
       "      <td>23</td>\n",
       "      <td>3</td>\n",
       "      <td>-0.002045</td>\n",
       "      <td>0.024651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id3505355</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-30 23:59:53</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.964203</td>\n",
       "      <td>40.679993</td>\n",
       "      <td>-73.959808</td>\n",
       "      <td>40.655403</td>\n",
       "      <td>N</td>\n",
       "      <td>6</td>\n",
       "      <td>23</td>\n",
       "      <td>3</td>\n",
       "      <td>0.004395</td>\n",
       "      <td>-0.024590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id1217141</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-30 23:59:47</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.997437</td>\n",
       "      <td>40.737583</td>\n",
       "      <td>-73.986160</td>\n",
       "      <td>40.729523</td>\n",
       "      <td>N</td>\n",
       "      <td>6</td>\n",
       "      <td>23</td>\n",
       "      <td>3</td>\n",
       "      <td>0.011276</td>\n",
       "      <td>-0.008060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id2150126</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-06-30 23:59:41</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.956070</td>\n",
       "      <td>40.771900</td>\n",
       "      <td>-73.986427</td>\n",
       "      <td>40.730469</td>\n",
       "      <td>N</td>\n",
       "      <td>6</td>\n",
       "      <td>23</td>\n",
       "      <td>3</td>\n",
       "      <td>-0.030357</td>\n",
       "      <td>-0.041431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id1598245</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-30 23:59:33</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.970215</td>\n",
       "      <td>40.761475</td>\n",
       "      <td>-73.961510</td>\n",
       "      <td>40.755890</td>\n",
       "      <td>N</td>\n",
       "      <td>6</td>\n",
       "      <td>23</td>\n",
       "      <td>3</td>\n",
       "      <td>0.008705</td>\n",
       "      <td>-0.005585</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  vendor_id     pickup_datetime  passenger_count  \\\n",
       "0  id3004672          1 2016-06-30 23:59:58                1   \n",
       "1  id3505355          1 2016-06-30 23:59:53                1   \n",
       "2  id1217141          1 2016-06-30 23:59:47                1   \n",
       "3  id2150126          2 2016-06-30 23:59:41                1   \n",
       "4  id1598245          1 2016-06-30 23:59:33                1   \n",
       "\n",
       "   pickup_longitude  pickup_latitude  dropoff_longitude  dropoff_latitude  \\\n",
       "0        -73.988129        40.732029         -73.990173         40.756680   \n",
       "1        -73.964203        40.679993         -73.959808         40.655403   \n",
       "2        -73.997437        40.737583         -73.986160         40.729523   \n",
       "3        -73.956070        40.771900         -73.986427         40.730469   \n",
       "4        -73.970215        40.761475         -73.961510         40.755890   \n",
       "\n",
       "  store_and_fwd_flag  pickup_month  pickup_hour  day_of_week  \\\n",
       "0                  N             6           23            3   \n",
       "1                  N             6           23            3   \n",
       "2                  N             6           23            3   \n",
       "3                  N             6           23            3   \n",
       "4                  N             6           23            3   \n",
       "\n",
       "   longitude_difference  latitude_difference  \n",
       "0             -0.002045             0.024651  \n",
       "1              0.004395            -0.024590  \n",
       "2              0.011276            -0.008060  \n",
       "3             -0.030357            -0.041431  \n",
       "4              0.008705            -0.005585  "
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test['day_of_week'] = test['day_of_week'].astype('category')\n",
    "day_of_week = pd.get_dummies(test['day_of_week'], prefix='day_of_week')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test['pickup_hour'] = test['pickup_hour'].astype('category')\n",
    "pickup_hour = pd.get_dummies(test['pickup_hour'], prefix='hour')\n",
    "test['pickup_month'] = test['pickup_month'].astype('category')\n",
    "pickup_month = pd.get_dummies(test['pickup_month'], prefix='month')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dt2 = [test, pickup_hour, pickup_month, day_of_week]\n",
    "test = pd.concat(dt2, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['id', 'vendor_id', 'pickup_datetime', 'passenger_count',\n",
       "       'pickup_longitude', 'pickup_latitude', 'dropoff_longitude',\n",
       "       'dropoff_latitude', 'store_and_fwd_flag', 'pickup_month', 'pickup_hour',\n",
       "       'day_of_week', 'longitude_difference', 'latitude_difference', 'hour_0',\n",
       "       'hour_1', 'hour_2', 'hour_3', 'hour_4', 'hour_5', 'hour_6', 'hour_7',\n",
       "       'hour_8', 'hour_9', 'hour_10', 'hour_11', 'hour_12', 'hour_13',\n",
       "       'hour_14', 'hour_15', 'hour_16', 'hour_17', 'hour_18', 'hour_19',\n",
       "       'hour_20', 'hour_21', 'hour_22', 'hour_23', 'month_1', 'month_2',\n",
       "       'month_3', 'month_4', 'month_5', 'month_6', 'day_of_week_0',\n",
       "       'day_of_week_1', 'day_of_week_2', 'day_of_week_3', 'day_of_week_4',\n",
       "       'day_of_week_5', 'day_of_week_6'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>longitude_difference</th>\n",
       "      <th>latitude_difference</th>\n",
       "      <th>hour_0</th>\n",
       "      <th>hour_1</th>\n",
       "      <th>hour_2</th>\n",
       "      <th>...</th>\n",
       "      <th>month_4</th>\n",
       "      <th>month_5</th>\n",
       "      <th>month_6</th>\n",
       "      <th>day_of_week_0</th>\n",
       "      <th>day_of_week_1</th>\n",
       "      <th>day_of_week_2</th>\n",
       "      <th>day_of_week_3</th>\n",
       "      <th>day_of_week_4</th>\n",
       "      <th>day_of_week_5</th>\n",
       "      <th>day_of_week_6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>-73.988129</td>\n",
       "      <td>40.732029</td>\n",
       "      <td>-73.990173</td>\n",
       "      <td>40.756680</td>\n",
       "      <td>-0.002045</td>\n",
       "      <td>0.024651</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>-73.964203</td>\n",
       "      <td>40.679993</td>\n",
       "      <td>-73.959808</td>\n",
       "      <td>40.655403</td>\n",
       "      <td>0.004395</td>\n",
       "      <td>-0.024590</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>-73.997437</td>\n",
       "      <td>40.737583</td>\n",
       "      <td>-73.986160</td>\n",
       "      <td>40.729523</td>\n",
       "      <td>0.011276</td>\n",
       "      <td>-0.008060</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>-73.956070</td>\n",
       "      <td>40.771900</td>\n",
       "      <td>-73.986427</td>\n",
       "      <td>40.730469</td>\n",
       "      <td>-0.030357</td>\n",
       "      <td>-0.041431</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>-73.970215</td>\n",
       "      <td>40.761475</td>\n",
       "      <td>-73.961510</td>\n",
       "      <td>40.755890</td>\n",
       "      <td>0.008705</td>\n",
       "      <td>-0.005585</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 44 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   passenger_count  pickup_longitude  pickup_latitude  dropoff_longitude  \\\n",
       "0                1        -73.988129        40.732029         -73.990173   \n",
       "1                1        -73.964203        40.679993         -73.959808   \n",
       "2                1        -73.997437        40.737583         -73.986160   \n",
       "3                1        -73.956070        40.771900         -73.986427   \n",
       "4                1        -73.970215        40.761475         -73.961510   \n",
       "\n",
       "   dropoff_latitude  longitude_difference  latitude_difference  hour_0  \\\n",
       "0         40.756680             -0.002045             0.024651       0   \n",
       "1         40.655403              0.004395            -0.024590       0   \n",
       "2         40.729523              0.011276            -0.008060       0   \n",
       "3         40.730469             -0.030357            -0.041431       0   \n",
       "4         40.755890              0.008705            -0.005585       0   \n",
       "\n",
       "   hour_1  hour_2      ...        month_4  month_5  month_6  day_of_week_0  \\\n",
       "0       0       0      ...              0        0        1              0   \n",
       "1       0       0      ...              0        0        1              0   \n",
       "2       0       0      ...              0        0        1              0   \n",
       "3       0       0      ...              0        0        1              0   \n",
       "4       0       0      ...              0        0        1              0   \n",
       "\n",
       "   day_of_week_1  day_of_week_2  day_of_week_3  day_of_week_4  day_of_week_5  \\\n",
       "0              0              0              1              0              0   \n",
       "1              0              0              1              0              0   \n",
       "2              0              0              1              0              0   \n",
       "3              0              0              1              0              0   \n",
       "4              0              0              1              0              0   \n",
       "\n",
       "   day_of_week_6  \n",
       "0              0  \n",
       "1              0  \n",
       "2              0  \n",
       "3              0  \n",
       "4              0  \n",
       "\n",
       "[5 rows x 44 columns]"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test = test[features]\n",
    "X_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBRegressor(base_score=0.5, colsample_bylevel=1, colsample_bytree=1, gamma=0,\n",
       "       learning_rate=0.1, max_delta_step=0, max_depth=3,\n",
       "       min_child_weight=1, missing=None, n_estimators=100, nthread=-1,\n",
       "       objective='reg:linear', reg_alpha=0, reg_lambda=1,\n",
       "       scale_pos_weight=1, seed=0, silent=True, subsample=1)"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from xgboost import XGBRegressor\n",
    "model = XGBRegressor()\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/lib/python3.6/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "X_test['trip_duration'] = np.absolute(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>longitude_difference</th>\n",
       "      <th>latitude_difference</th>\n",
       "      <th>hour_0</th>\n",
       "      <th>hour_1</th>\n",
       "      <th>hour_2</th>\n",
       "      <th>...</th>\n",
       "      <th>month_5</th>\n",
       "      <th>month_6</th>\n",
       "      <th>day_of_week_0</th>\n",
       "      <th>day_of_week_1</th>\n",
       "      <th>day_of_week_2</th>\n",
       "      <th>day_of_week_3</th>\n",
       "      <th>day_of_week_4</th>\n",
       "      <th>day_of_week_5</th>\n",
       "      <th>day_of_week_6</th>\n",
       "      <th>trip_duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>-73.988129</td>\n",
       "      <td>40.732029</td>\n",
       "      <td>-73.990173</td>\n",
       "      <td>40.756680</td>\n",
       "      <td>-0.002045</td>\n",
       "      <td>0.024651</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>958.982544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>-73.964203</td>\n",
       "      <td>40.679993</td>\n",
       "      <td>-73.959808</td>\n",
       "      <td>40.655403</td>\n",
       "      <td>0.004395</td>\n",
       "      <td>-0.024590</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>934.089661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>-73.997437</td>\n",
       "      <td>40.737583</td>\n",
       "      <td>-73.986160</td>\n",
       "      <td>40.729523</td>\n",
       "      <td>0.011276</td>\n",
       "      <td>-0.008060</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>580.370605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>-73.956070</td>\n",
       "      <td>40.771900</td>\n",
       "      <td>-73.986427</td>\n",
       "      <td>40.730469</td>\n",
       "      <td>-0.030357</td>\n",
       "      <td>-0.041431</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1392.856689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>-73.970215</td>\n",
       "      <td>40.761475</td>\n",
       "      <td>-73.961510</td>\n",
       "      <td>40.755890</td>\n",
       "      <td>0.008705</td>\n",
       "      <td>-0.005585</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>537.424438</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 45 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   passenger_count  pickup_longitude  pickup_latitude  dropoff_longitude  \\\n",
       "0                1        -73.988129        40.732029         -73.990173   \n",
       "1                1        -73.964203        40.679993         -73.959808   \n",
       "2                1        -73.997437        40.737583         -73.986160   \n",
       "3                1        -73.956070        40.771900         -73.986427   \n",
       "4                1        -73.970215        40.761475         -73.961510   \n",
       "\n",
       "   dropoff_latitude  longitude_difference  latitude_difference  hour_0  \\\n",
       "0         40.756680             -0.002045             0.024651       0   \n",
       "1         40.655403              0.004395            -0.024590       0   \n",
       "2         40.729523              0.011276            -0.008060       0   \n",
       "3         40.730469             -0.030357            -0.041431       0   \n",
       "4         40.755890              0.008705            -0.005585       0   \n",
       "\n",
       "   hour_1  hour_2      ...        month_5  month_6  day_of_week_0  \\\n",
       "0       0       0      ...              0        1              0   \n",
       "1       0       0      ...              0        1              0   \n",
       "2       0       0      ...              0        1              0   \n",
       "3       0       0      ...              0        1              0   \n",
       "4       0       0      ...              0        1              0   \n",
       "\n",
       "   day_of_week_1  day_of_week_2  day_of_week_3  day_of_week_4  day_of_week_5  \\\n",
       "0              0              0              1              0              0   \n",
       "1              0              0              1              0              0   \n",
       "2              0              0              1              0              0   \n",
       "3              0              0              1              0              0   \n",
       "4              0              0              1              0              0   \n",
       "\n",
       "   day_of_week_6  trip_duration  \n",
       "0              0     958.982544  \n",
       "1              0     934.089661  \n",
       "2              0     580.370605  \n",
       "3              0    1392.856689  \n",
       "4              0     537.424438  \n",
       "\n",
       "[5 rows x 45 columns]"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "result = pd.concat([test, X_test], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>vendor_id</th>\n",
       "      <th>pickup_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>pickup_month</th>\n",
       "      <th>...</th>\n",
       "      <th>pickup_hour</th>\n",
       "      <th>pickup_month</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>longitude_difference</th>\n",
       "      <th>latitude_difference</th>\n",
       "      <th>trip_duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id3004672</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-30 23:59:58</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.988129</td>\n",
       "      <td>40.732029</td>\n",
       "      <td>-73.990173</td>\n",
       "      <td>40.756680</td>\n",
       "      <td>N</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>23</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>-73.988129</td>\n",
       "      <td>40.732029</td>\n",
       "      <td>-73.990173</td>\n",
       "      <td>40.756680</td>\n",
       "      <td>-0.002045</td>\n",
       "      <td>0.024651</td>\n",
       "      <td>1047.276245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id3505355</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-30 23:59:53</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.964203</td>\n",
       "      <td>40.679993</td>\n",
       "      <td>-73.959808</td>\n",
       "      <td>40.655403</td>\n",
       "      <td>N</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>23</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>-73.964203</td>\n",
       "      <td>40.679993</td>\n",
       "      <td>-73.959808</td>\n",
       "      <td>40.655403</td>\n",
       "      <td>0.004395</td>\n",
       "      <td>-0.024590</td>\n",
       "      <td>891.129700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id1217141</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-30 23:59:47</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.997437</td>\n",
       "      <td>40.737583</td>\n",
       "      <td>-73.986160</td>\n",
       "      <td>40.729523</td>\n",
       "      <td>N</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>23</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>-73.997437</td>\n",
       "      <td>40.737583</td>\n",
       "      <td>-73.986160</td>\n",
       "      <td>40.729523</td>\n",
       "      <td>0.011276</td>\n",
       "      <td>-0.008060</td>\n",
       "      <td>565.815430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id2150126</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-06-30 23:59:41</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.956070</td>\n",
       "      <td>40.771900</td>\n",
       "      <td>-73.986427</td>\n",
       "      <td>40.730469</td>\n",
       "      <td>N</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>23</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>-73.956070</td>\n",
       "      <td>40.771900</td>\n",
       "      <td>-73.986427</td>\n",
       "      <td>40.730469</td>\n",
       "      <td>-0.030357</td>\n",
       "      <td>-0.041431</td>\n",
       "      <td>1396.858398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id1598245</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-30 23:59:33</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.970215</td>\n",
       "      <td>40.761475</td>\n",
       "      <td>-73.961510</td>\n",
       "      <td>40.755890</td>\n",
       "      <td>N</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>23</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>-73.970215</td>\n",
       "      <td>40.761475</td>\n",
       "      <td>-73.961510</td>\n",
       "      <td>40.755890</td>\n",
       "      <td>0.008705</td>\n",
       "      <td>-0.005585</td>\n",
       "      <td>497.615967</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  vendor_id     pickup_datetime  passenger_count  \\\n",
       "0  id3004672          1 2016-06-30 23:59:58                1   \n",
       "1  id3505355          1 2016-06-30 23:59:53                1   \n",
       "2  id1217141          1 2016-06-30 23:59:47                1   \n",
       "3  id2150126          2 2016-06-30 23:59:41                1   \n",
       "4  id1598245          1 2016-06-30 23:59:33                1   \n",
       "\n",
       "   pickup_longitude  pickup_latitude  dropoff_longitude  dropoff_latitude  \\\n",
       "0        -73.988129        40.732029         -73.990173         40.756680   \n",
       "1        -73.964203        40.679993         -73.959808         40.655403   \n",
       "2        -73.997437        40.737583         -73.986160         40.729523   \n",
       "3        -73.956070        40.771900         -73.986427         40.730469   \n",
       "4        -73.970215        40.761475         -73.961510         40.755890   \n",
       "\n",
       "  store_and_fwd_flag  pickup_month      ...        pickup_hour  pickup_month  \\\n",
       "0                  N             6      ...                 23             6   \n",
       "1                  N             6      ...                 23             6   \n",
       "2                  N             6      ...                 23             6   \n",
       "3                  N             6      ...                 23             6   \n",
       "4                  N             6      ...                 23             6   \n",
       "\n",
       "   day_of_week  pickup_longitude  pickup_latitude  dropoff_longitude  \\\n",
       "0            3        -73.988129        40.732029         -73.990173   \n",
       "1            3        -73.964203        40.679993         -73.959808   \n",
       "2            3        -73.997437        40.737583         -73.986160   \n",
       "3            3        -73.956070        40.771900         -73.986427   \n",
       "4            3        -73.970215        40.761475         -73.961510   \n",
       "\n",
       "   dropoff_latitude  longitude_difference  latitude_difference  trip_duration  \n",
       "0         40.756680             -0.002045             0.024651    1047.276245  \n",
       "1         40.655403              0.004395            -0.024590     891.129700  \n",
       "2         40.729523              0.011276            -0.008060     565.815430  \n",
       "3         40.730469             -0.030357            -0.041431    1396.858398  \n",
       "4         40.755890              0.008705            -0.005585     497.615967  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submit = result[['id', 'trip_duration']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>trip_duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id3004672</td>\n",
       "      <td>1093.755537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id3505355</td>\n",
       "      <td>1115.739490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id1217141</td>\n",
       "      <td>583.309216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id2150126</td>\n",
       "      <td>1421.304341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id1598245</td>\n",
       "      <td>583.309216</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  trip_duration\n",
       "0  id3004672    1093.755537\n",
       "1  id3505355    1115.739490\n",
       "2  id1217141     583.309216\n",
       "3  id2150126    1421.304341\n",
       "4  id1598245     583.309216"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submit.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submit.to_csv('submissions.csv', sep=',', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>trip_duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [id, trip_duration]\n",
       "Index: []"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submit[submit['trip_duration'] < 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(625134,)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions_id = test['id']\n",
    "predictions_id.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     id3004672\n",
       "1     id3505355\n",
       "2     id1217141\n",
       "3     id2150126\n",
       "4     id1598245\n",
       "5     id0668992\n",
       "6     id1765014\n",
       "7     id0898117\n",
       "8     id3905224\n",
       "9     id1543102\n",
       "10    id3024712\n",
       "11    id3665810\n",
       "12    id1836461\n",
       "13    id3457080\n",
       "14    id3376065\n",
       "15    id3008739\n",
       "16    id0902216\n",
       "17    id3564824\n",
       "18    id0820280\n",
       "19    id0775088\n",
       "Name: id, dtype: object"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions_id[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions_id = np.asarray(predictions_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions_id = predictions_id.reshape(len(predictions_id), 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(625134, 1)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions_id.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(625134, 1)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(625134, 1)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions_id.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['id3004672'],\n",
       "       ['id3505355'],\n",
       "       ['id1217141'],\n",
       "       ['id2150126'],\n",
       "       ['id1598245'],\n",
       "       ['id0668992'],\n",
       "       ['id1765014'],\n",
       "       ['id0898117'],\n",
       "       ['id3905224'],\n",
       "       ['id1543102']], dtype=object)"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions_id[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1030.46522693],\n",
       "       [  983.56059708],\n",
       "       [  981.45641874],\n",
       "       [ 1040.45682435],\n",
       "       [  988.25047959],\n",
       "       [ 1013.53964154],\n",
       "       [  940.47679908],\n",
       "       [ 1003.43816107],\n",
       "       [  912.8443156 ],\n",
       "       [  990.64978806]])"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def rmsle(predicted,real):\n",
    "    sum=0.0\n",
    "    for x in range(len(predicted)):\n",
    "        p = np.log(predicted[x]+1)\n",
    "        r = np.log(real[x]+1)\n",
    "        sum = sum + (p - r)**2\n",
    "    return (sum/len(predicted))**0.5"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}