final_project.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import pandas as pd\n",
    "import os\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "import statistics \n",
    "import statsmodels.formula.api as smf\n",
    "import math\n",
    "import statsmodels.api as sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load training data \n",
    "df = pd.read_csv(r'/Users/Anna/Dropbox (Penn)/Fall 2022/CIS 5200/Final Project/data/train.csv')\n",
    "# df = pd.read_csv(r'C:\\Users\\gaoran\\Dropbox (Penn)\\Fall 2022\\CIS 5200\\Final Project\\data\\train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         srch_id            date_time  site_id  visitor_location_country_id  \\\n",
      "0              1  2013-04-04 08:32:15       12                          187   \n",
      "1              1  2013-04-04 08:32:15       12                          187   \n",
      "2              1  2013-04-04 08:32:15       12                          187   \n",
      "3              1  2013-04-04 08:32:15       12                          187   \n",
      "4              1  2013-04-04 08:32:15       12                          187   \n",
      "...          ...                  ...      ...                          ...   \n",
      "9917525   665574  2013-05-21 11:06:37       24                          216   \n",
      "9917526   665574  2013-05-21 11:06:37       24                          216   \n",
      "9917527   665574  2013-05-21 11:06:37       24                          216   \n",
      "9917528   665574  2013-05-21 11:06:37       24                          216   \n",
      "9917529   665574  2013-05-21 11:06:37       24                          216   \n",
      "\n",
      "         visitor_hist_starrating  visitor_hist_adr_usd  prop_country_id  \\\n",
      "0                            NaN                   NaN              219   \n",
      "1                            NaN                   NaN              219   \n",
      "2                            NaN                   NaN              219   \n",
      "3                            NaN                   NaN              219   \n",
      "4                            NaN                   NaN              219   \n",
      "...                          ...                   ...              ...   \n",
      "9917525                      NaN                   NaN              117   \n",
      "9917526                      NaN                   NaN              117   \n",
      "9917527                      NaN                   NaN              117   \n",
      "9917528                      NaN                   NaN              117   \n",
      "9917529                      NaN                   NaN              117   \n",
      "\n",
      "         prop_id  prop_starrating  prop_review_score  ...  \\\n",
      "0            893                3                3.5  ...   \n",
      "1          10404                4                4.0  ...   \n",
      "2          21315                3                4.5  ...   \n",
      "3          27348                2                4.0  ...   \n",
      "4          29604                4                3.5  ...   \n",
      "...          ...              ...                ...  ...   \n",
      "9917525    32019                4                3.5  ...   \n",
      "9917526    33959                4                3.0  ...   \n",
      "9917527    35240                4                0.0  ...   \n",
      "9917528    94437                4                0.0  ...   \n",
      "9917529    99509                4                4.5  ...   \n",
      "\n",
      "         comp6_rate_percent_diff  comp7_rate  comp7_inv  \\\n",
      "0                            NaN         NaN        NaN   \n",
      "1                            NaN         NaN        NaN   \n",
      "2                            NaN         NaN        NaN   \n",
      "3                            NaN         NaN        NaN   \n",
      "4                            NaN         NaN        NaN   \n",
      "...                          ...         ...        ...   \n",
      "9917525                      NaN         NaN        NaN   \n",
      "9917526                      NaN         NaN        NaN   \n",
      "9917527                      NaN         NaN        NaN   \n",
      "9917528                      NaN         NaN        NaN   \n",
      "9917529                      NaN         NaN        NaN   \n",
      "\n",
      "         comp7_rate_percent_diff  comp8_rate  comp8_inv  \\\n",
      "0                            NaN         0.0        0.0   \n",
      "1                            NaN         0.0        0.0   \n",
      "2                            NaN         0.0        0.0   \n",
      "3                            NaN        -1.0        0.0   \n",
      "4                            NaN         0.0        0.0   \n",
      "...                          ...         ...        ...   \n",
      "9917525                      NaN         NaN        NaN   \n",
      "9917526                      NaN         NaN        NaN   \n",
      "9917527                      NaN         NaN        NaN   \n",
      "9917528                      NaN         NaN        NaN   \n",
      "9917529                      NaN         NaN        NaN   \n",
      "\n",
      "         comp8_rate_percent_diff  click_bool  gross_bookings_usd  booking_bool  \n",
      "0                            NaN           0                 NaN             0  \n",
      "1                            NaN           0                 NaN             0  \n",
      "2                            NaN           0                 NaN             0  \n",
      "3                            5.0           0                 NaN             0  \n",
      "4                            NaN           0                 NaN             0  \n",
      "...                          ...         ...                 ...           ...  \n",
      "9917525                      NaN           0                 NaN             0  \n",
      "9917526                      NaN           1              154.34             1  \n",
      "9917527                      NaN           0                 NaN             0  \n",
      "9917528                      NaN           0                 NaN             0  \n",
      "9917529                      NaN           0                 NaN             0  \n",
      "\n",
      "[9917530 rows x 54 columns]\n"
     ]
    }
   ],
   "source": [
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['srch_id', 'date_time', 'site_id', 'visitor_location_country_id',\n",
       "       'visitor_hist_starrating', 'visitor_hist_adr_usd', 'prop_country_id',\n",
       "       'prop_id', 'prop_starrating', 'prop_review_score', 'prop_brand_bool',\n",
       "       'prop_location_score1', 'prop_location_score2',\n",
       "       'prop_log_historical_price', 'position', 'price_usd', 'promotion_flag',\n",
       "       'srch_destination_id', 'srch_length_of_stay', 'srch_booking_window',\n",
       "       'srch_adults_count', 'srch_children_count', 'srch_room_count',\n",
       "       'srch_saturday_night_bool', 'srch_query_affinity_score',\n",
       "       'orig_destination_distance', 'random_bool', 'comp1_rate', 'comp1_inv',\n",
       "       'comp1_rate_percent_diff', 'comp2_rate', 'comp2_inv',\n",
       "       'comp2_rate_percent_diff', 'comp3_rate', 'comp3_inv',\n",
       "       'comp3_rate_percent_diff', 'comp4_rate', 'comp4_inv',\n",
       "       'comp4_rate_percent_diff', 'comp5_rate', 'comp5_inv',\n",
       "       'comp5_rate_percent_diff', 'comp6_rate', 'comp6_inv',\n",
       "       'comp6_rate_percent_diff', 'comp7_rate', 'comp7_inv',\n",
       "       'comp7_rate_percent_diff', 'comp8_rate', 'comp8_inv',\n",
       "       'comp8_rate_percent_diff', 'click_bool', 'gross_bookings_usd',\n",
       "       'booking_bool'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "218"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select relevant variables \n",
    "visitor_hist_starrating\n",
    "visitor_hist_adr_usd\n",
    "prop_starrating\n",
    "prop_review_score\n",
    "prop_brand_bool\n",
    "prop_location_score1\n",
    "prop_log_historical_price\n",
    "position\n",
    "price_usd\n",
    "promotion_flag\n",
    "srch_length_of_stay\n",
    "srch_booking_window\n",
    "srch_adults_count\n",
    "srch_children_count\n",
    "srch_room_count\n",
    "srch_saturday_night_bool\n",
    "orig_destination_distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unique country ID:  218\n",
      "unique ID of the country the hotel is located in:  172\n",
      "unique ID of the hotel:  136886\n",
      "unique star rating of the hotel:  6\n",
      "unique mean customer review score:  11\n"
     ]
    }
   ],
   "source": [
    "# Check what variables needed to change to dummies \n",
    "print(\"unique country ID: \", len(df['visitor_location_country_id'].unique()))\n",
    "print(\"unique ID of the country the hotel is located in: \", len(df['prop_country_id'].unique()))\n",
    "print(\"unique ID of the hotel: \", len(df['prop_id'].unique()))\n",
    "print(\"unique star rating of the hotel: \", len(df['prop_starrating'].unique()))\n",
    "print(\"unique mean customer review score: \", len(df['prop_review_score'].unique()))\n",
    "\n",
    "# Categorical variables have too many levels, better not to include many dummies?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total number of observations:  9917530\n",
      "historical star rating obs:  9412233\n",
      "historical spending obs:  9409918\n",
      "star rating of the hotel obs:  0\n",
      "mean customer review score obs:  14630\n",
      "major hotel chain dummy obs:  0\n",
      "desirability hotel location score 1:  0\n",
      "desirability hotel location score 2:  2178380\n",
      "log mean hotel price:  0\n",
      "position of hotel listing:  0\n",
      "displayed price of the hotel for the given search:  0\n",
      "disaplyed sale price promotion:  0\n",
      "Number of nights stay that was searched:  0\n",
      "Number of future stay days:  0\n",
      "Number of adult:  0\n",
      "Number of children:  0\n",
      "Number of rooms:  0\n",
      "Number of Saturday nights:  0\n",
      "log probability the hotel will be clicked:  9281966\n",
      "physical distance between the hotel and the customer:  3216461\n",
      "dummy if Expedia has a lower price than competitor 1 :  9681724\n",
      "dummy if competitor 1 does not have availability  :  9663097\n",
      "percentage difference between Expedia and competitor 1's price :  9732623\n"
     ]
    }
   ],
   "source": [
    "# NaN means there is no purchase history on this customer \n",
    "print(\"total number of observations: \", len(df))\n",
    "print(\"historical star rating obs: \", df['visitor_hist_starrating'].isna().sum())\n",
    "print(\"historical spending obs: \", df['visitor_hist_adr_usd'].isna().sum())\n",
    "print(\"star rating of the hotel obs: \", df['prop_starrating'].isna().sum())\n",
    "print(\"mean customer review score obs: \", df['prop_review_score'].isna().sum())\n",
    "print(\"major hotel chain dummy obs: \", df['prop_brand_bool'].isna().sum())\n",
    "print(\"desirability hotel location score 1: \", df['prop_location_score1'].isna().sum())\n",
    "print(\"desirability hotel location score 2: \", df['prop_location_score2'].isna().sum())\n",
    "print(\"log mean hotel price: \", df['prop_log_historical_price'].isna().sum())\n",
    "print(\"position of hotel listing: \", df['position'].isna().sum())\n",
    "print(\"displayed price of the hotel for the given search: \", df['price_usd'].isna().sum())\n",
    "print(\"disaplyed sale price promotion: \", df['promotion_flag'].isna().sum())\n",
    "print(\"Number of nights stay that was searched: \", df['srch_length_of_stay'].isna().sum())\n",
    "print(\"Number of future stay days: \", df['srch_booking_window'].isna().sum())\n",
    "print(\"Number of adult: \", df['srch_adults_count'].isna().sum())\n",
    "print(\"Number of children: \", df['srch_children_count'].isna().sum())\n",
    "print(\"Number of rooms: \", df['srch_room_count'].isna().sum())\n",
    "print(\"Number of Saturday nights: \", df['srch_saturday_night_bool'].isna().sum())\n",
    "print(\"log probability the hotel will be clicked: \", df['srch_query_affinity_score'].isna().sum())\n",
    "print(\"physical distance between the hotel and the customer: \", df['orig_destination_distance'].isna().sum())\n",
    "print(\"dummy if Expedia has a lower price than competitor 1 : \", df['comp1_rate'].isna().sum())\n",
    "print(\"dummy if competitor 1 does not have availability  : \", df['comp1_inv'].isna().sum())\n",
    "print(\"percentage difference between Expedia and competitor 1's price : \", df['comp1_rate_percent_diff'].isna().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>srch_id</th>\n",
       "      <th>date_time</th>\n",
       "      <th>site_id</th>\n",
       "      <th>visitor_location_country_id</th>\n",
       "      <th>visitor_hist_starrating</th>\n",
       "      <th>visitor_hist_adr_usd</th>\n",
       "      <th>prop_country_id</th>\n",
       "      <th>prop_id</th>\n",
       "      <th>prop_starrating</th>\n",
       "      <th>prop_review_score</th>\n",
       "      <th>...</th>\n",
       "      <th>comp6_rate_percent_diff</th>\n",
       "      <th>comp7_rate</th>\n",
       "      <th>comp7_inv</th>\n",
       "      <th>comp7_rate_percent_diff</th>\n",
       "      <th>comp8_rate</th>\n",
       "      <th>comp8_inv</th>\n",
       "      <th>comp8_rate_percent_diff</th>\n",
       "      <th>click_bool</th>\n",
       "      <th>gross_bookings_usd</th>\n",
       "      <th>booking_bool</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2013-04-04 08:32:15</td>\n",
       "      <td>12</td>\n",
       "      <td>187</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>219</td>\n",
       "      <td>893</td>\n",
       "      <td>3</td>\n",
       "      <td>3.5</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2013-04-04 08:32:15</td>\n",
       "      <td>12</td>\n",
       "      <td>187</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>219</td>\n",
       "      <td>10404</td>\n",
       "      <td>4</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>2013-04-04 08:32:15</td>\n",
       "      <td>12</td>\n",
       "      <td>187</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>219</td>\n",
       "      <td>21315</td>\n",
       "      <td>3</td>\n",
       "      <td>4.5</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>2013-04-04 08:32:15</td>\n",
       "      <td>12</td>\n",
       "      <td>187</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>219</td>\n",
       "      <td>27348</td>\n",
       "      <td>2</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>2013-04-04 08:32:15</td>\n",
       "      <td>12</td>\n",
       "      <td>187</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>219</td>\n",
       "      <td>29604</td>\n",
       "      <td>4</td>\n",
       "      <td>3.5</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9917525</th>\n",
       "      <td>665574</td>\n",
       "      <td>2013-05-21 11:06:37</td>\n",
       "      <td>24</td>\n",
       "      <td>216</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>117</td>\n",
       "      <td>32019</td>\n",
       "      <td>4</td>\n",
       "      <td>3.5</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9917526</th>\n",
       "      <td>665574</td>\n",
       "      <td>2013-05-21 11:06:37</td>\n",
       "      <td>24</td>\n",
       "      <td>216</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>117</td>\n",
       "      <td>33959</td>\n",
       "      <td>4</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>154.34</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9917527</th>\n",
       "      <td>665574</td>\n",
       "      <td>2013-05-21 11:06:37</td>\n",
       "      <td>24</td>\n",
       "      <td>216</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>117</td>\n",
       "      <td>35240</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9917528</th>\n",
       "      <td>665574</td>\n",
       "      <td>2013-05-21 11:06:37</td>\n",
       "      <td>24</td>\n",
       "      <td>216</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>117</td>\n",
       "      <td>94437</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9917529</th>\n",
       "      <td>665574</td>\n",
       "      <td>2013-05-21 11:06:37</td>\n",
       "      <td>24</td>\n",
       "      <td>216</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>117</td>\n",
       "      <td>99509</td>\n",
       "      <td>4</td>\n",
       "      <td>4.5</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9917530 rows × 54 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         srch_id            date_time  site_id  visitor_location_country_id  \\\n",
       "0              1  2013-04-04 08:32:15       12                          187   \n",
       "1              1  2013-04-04 08:32:15       12                          187   \n",
       "2              1  2013-04-04 08:32:15       12                          187   \n",
       "3              1  2013-04-04 08:32:15       12                          187   \n",
       "4              1  2013-04-04 08:32:15       12                          187   \n",
       "...          ...                  ...      ...                          ...   \n",
       "9917525   665574  2013-05-21 11:06:37       24                          216   \n",
       "9917526   665574  2013-05-21 11:06:37       24                          216   \n",
       "9917527   665574  2013-05-21 11:06:37       24                          216   \n",
       "9917528   665574  2013-05-21 11:06:37       24                          216   \n",
       "9917529   665574  2013-05-21 11:06:37       24                          216   \n",
       "\n",
       "         visitor_hist_starrating  visitor_hist_adr_usd  prop_country_id  \\\n",
       "0                       3.374634            176.588512              219   \n",
       "1                       3.374634            176.588512              219   \n",
       "2                       3.374634            176.588512              219   \n",
       "3                       3.374634            176.588512              219   \n",
       "4                       3.374634            176.588512              219   \n",
       "...                          ...                   ...              ...   \n",
       "9917525                 3.374634            176.588512              117   \n",
       "9917526                 3.374634            176.588512              117   \n",
       "9917527                 3.374634            176.588512              117   \n",
       "9917528                 3.374634            176.588512              117   \n",
       "9917529                 3.374634            176.588512              117   \n",
       "\n",
       "         prop_id  prop_starrating  prop_review_score  ...  \\\n",
       "0            893                3                3.5  ...   \n",
       "1          10404                4                4.0  ...   \n",
       "2          21315                3                4.5  ...   \n",
       "3          27348                2                4.0  ...   \n",
       "4          29604                4                3.5  ...   \n",
       "...          ...              ...                ...  ...   \n",
       "9917525    32019                4                3.5  ...   \n",
       "9917526    33959                4                3.0  ...   \n",
       "9917527    35240                4                0.0  ...   \n",
       "9917528    94437                4                0.0  ...   \n",
       "9917529    99509                4                4.5  ...   \n",
       "\n",
       "         comp6_rate_percent_diff  comp7_rate  comp7_inv  \\\n",
       "0                            NaN         NaN        NaN   \n",
       "1                            NaN         NaN        NaN   \n",
       "2                            NaN         NaN        NaN   \n",
       "3                            NaN         NaN        NaN   \n",
       "4                            NaN         NaN        NaN   \n",
       "...                          ...         ...        ...   \n",
       "9917525                      NaN         NaN        NaN   \n",
       "9917526                      NaN         NaN        NaN   \n",
       "9917527                      NaN         NaN        NaN   \n",
       "9917528                      NaN         NaN        NaN   \n",
       "9917529                      NaN         NaN        NaN   \n",
       "\n",
       "         comp7_rate_percent_diff  comp8_rate  comp8_inv  \\\n",
       "0                            NaN         0.0        0.0   \n",
       "1                            NaN         0.0        0.0   \n",
       "2                            NaN         0.0        0.0   \n",
       "3                            NaN        -1.0        0.0   \n",
       "4                            NaN         0.0        0.0   \n",
       "...                          ...         ...        ...   \n",
       "9917525                      NaN         NaN        NaN   \n",
       "9917526                      NaN         NaN        NaN   \n",
       "9917527                      NaN         NaN        NaN   \n",
       "9917528                      NaN         NaN        NaN   \n",
       "9917529                      NaN         NaN        NaN   \n",
       "\n",
       "         comp8_rate_percent_diff  click_bool  gross_bookings_usd  booking_bool  \n",
       "0                            NaN           0                 NaN             0  \n",
       "1                            NaN           0                 NaN             0  \n",
       "2                            NaN           0                 NaN             0  \n",
       "3                            5.0           0                 NaN             0  \n",
       "4                            NaN           0                 NaN             0  \n",
       "...                          ...         ...                 ...           ...  \n",
       "9917525                      NaN           0                 NaN             0  \n",
       "9917526                      NaN           1              154.34             1  \n",
       "9917527                      NaN           0                 NaN             0  \n",
       "9917528                      NaN           0                 NaN             0  \n",
       "9917529                      NaN           0                 NaN             0  \n",
       "\n",
       "[9917530 rows x 54 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# missing data imputation using the average value of the columns\n",
    "df[\"visitor_hist_starrating\"] = df[\"visitor_hist_starrating\"].fillna(df[\"visitor_hist_starrating\"].mean())\n",
    "df[\"visitor_hist_adr_usd\"] = df[\"visitor_hist_adr_usd\"].fillna(df[\"visitor_hist_adr_usd\"].mean())\n",
    "df[\"prop_review_score\"] = df[\"prop_review_score\"].fillna(df[\"prop_review_score\"].mean())\n",
    "df[\"orig_destination_distance\"] = df[\"orig_destination_distance\"].fillna(df[\"orig_destination_distance\"].mean())\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         click_bool  booking_bool  random_bool  visitor_hist_starrating  \\\n",
      "0                 0             0            1                 3.374634   \n",
      "1                 0             0            1                 3.374634   \n",
      "2                 0             0            1                 3.374634   \n",
      "3                 0             0            1                 3.374634   \n",
      "4                 0             0            1                 3.374634   \n",
      "...             ...           ...          ...                      ...   \n",
      "9917525           0             0            0                 3.374634   \n",
      "9917526           1             1            0                 3.374634   \n",
      "9917527           0             0            0                 3.374634   \n",
      "9917528           0             0            0                 3.374634   \n",
      "9917529           0             0            0                 3.374634   \n",
      "\n",
      "         visitor_hist_adr_usd  prop_starrating  prop_review_score  \\\n",
      "0                  176.588512                3                3.5   \n",
      "1                  176.588512                4                4.0   \n",
      "2                  176.588512                3                4.5   \n",
      "3                  176.588512                2                4.0   \n",
      "4                  176.588512                4                3.5   \n",
      "...                       ...              ...                ...   \n",
      "9917525            176.588512                4                3.5   \n",
      "9917526            176.588512                4                3.0   \n",
      "9917527            176.588512                4                0.0   \n",
      "9917528            176.588512                4                0.0   \n",
      "9917529            176.588512                4                4.5   \n",
      "\n",
      "         prop_brand_bool  prop_location_score1  prop_log_historical_price  \\\n",
      "0                      1                  2.83                       4.95   \n",
      "1                      1                  2.20                       5.03   \n",
      "2                      1                  2.20                       4.92   \n",
      "3                      1                  2.83                       4.39   \n",
      "4                      1                  2.64                       4.93   \n",
      "...                  ...                   ...                        ...   \n",
      "9917525                0                  2.48                       4.53   \n",
      "9917526                1                  2.20                       4.39   \n",
      "9917527                0                  1.79                       4.64   \n",
      "9917528                0                  2.94                       4.64   \n",
      "9917529                1                  2.08                       4.64   \n",
      "\n",
      "         position  price_usd  promotion_flag  srch_length_of_stay  \\\n",
      "0              27     104.77               0                    1   \n",
      "1              26     170.74               0                    1   \n",
      "2              21     179.80               0                    1   \n",
      "3              34     602.77               0                    1   \n",
      "4               4     143.58               0                    1   \n",
      "...           ...        ...             ...                  ...   \n",
      "9917525         2      66.07               0                    2   \n",
      "9917526         6      67.10               0                    2   \n",
      "9917527         8      73.91               0                    2   \n",
      "9917528         7      66.07               0                    2   \n",
      "9917529         4      82.06               0                    2   \n",
      "\n",
      "         srch_booking_window  srch_adults_count  srch_children_count  \\\n",
      "0                          0                  4                    0   \n",
      "1                          0                  4                    0   \n",
      "2                          0                  4                    0   \n",
      "3                          0                  4                    0   \n",
      "4                          0                  4                    0   \n",
      "...                      ...                ...                  ...   \n",
      "9917525                    7                  1                    0   \n",
      "9917526                    7                  1                    0   \n",
      "9917527                    7                  1                    0   \n",
      "9917528                    7                  1                    0   \n",
      "9917529                    7                  1                    0   \n",
      "\n",
      "         srch_room_count  srch_saturday_night_bool  orig_destination_distance  \n",
      "0                      1                         1                1307.017977  \n",
      "1                      1                         1                1307.017977  \n",
      "2                      1                         1                1307.017977  \n",
      "3                      1                         1                1307.017977  \n",
      "4                      1                         1                1307.017977  \n",
      "...                  ...                       ...                        ...  \n",
      "9917525                1                         0                1307.017977  \n",
      "9917526                1                         0                1307.017977  \n",
      "9917527                1                         0                1307.017977  \n",
      "9917528                1                         0                1307.017977  \n",
      "9917529                1                         0                1307.017977  \n",
      "\n",
      "[9917530 rows x 20 columns]\n"
     ]
    }
   ],
   "source": [
    "# create new dataframe based on selected features and data imputations\n",
    "df_new = df[['click_bool', 'booking_bool', 'random_bool', 'visitor_hist_starrating', 'visitor_hist_adr_usd', \n",
    "             'prop_starrating', 'prop_review_score', \n",
    "             'prop_brand_bool', 'prop_location_score1', 'prop_log_historical_price', 'position', 'price_usd', \n",
    "             'promotion_flag', 'srch_length_of_stay', 'srch_booking_window', 'srch_adults_count', \n",
    "             'srch_children_count', 'srch_room_count', 'srch_saturday_night_bool', 'orig_destination_distance']]\n",
    "print(df_new)\n",
    "df_new.to_csv('/Users/Anna/Dropbox (Penn)/Fall 2022/CIS 5200/Final Project/data/train_new.csv')  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    9.917530e+06\n",
       "mean     4.473614e-02\n",
       "std      2.067240e-01\n",
       "min      0.000000e+00\n",
       "25%      0.000000e+00\n",
       "50%      0.000000e+00\n",
       "75%      0.000000e+00\n",
       "max      1.000000e+00\n",
       "Name: click_bool, dtype: float64"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['click_bool'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    9.917530e+06\n",
       "mean     2.788930e-02\n",
       "std      1.646557e-01\n",
       "min      0.000000e+00\n",
       "25%      0.000000e+00\n",
       "50%      0.000000e+00\n",
       "75%      0.000000e+00\n",
       "max      1.000000e+00\n",
       "Name: booking_bool, dtype: float64"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['booking_bool'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    9.917530e+06\n",
       "mean     3.374634e+00\n",
       "std      1.562479e-01\n",
       "min      1.000000e+00\n",
       "25%      3.374634e+00\n",
       "50%      3.374634e+00\n",
       "75%      3.374634e+00\n",
       "max      5.000000e+00\n",
       "Name: visitor_hist_starrating, dtype: float64"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['visitor_hist_starrating'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "       <td></td>          <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th>   <td>    0.0440</td> <td> 7.83e-05</td> <td>  561.683</td> <td> 0.000</td> <td>    0.044</td> <td>    0.044</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>random_bool</th> <td>    0.0026</td> <td>    0.000</td> <td>   18.323</td> <td> 0.000</td> <td>    0.002</td> <td>    0.003</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.table.SimpleTable'>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# clicking ATE without controls \n",
    "# random_bool = 0 normal sorting, = 1 random sorting\n",
    "smf.ols('click_bool ~ random_bool', data=df_new).fit().summary().tables[1]\n",
    "\n",
    "# when the sorting is random, consumers are more likely to click because consumers have higher search cost "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "       <td></td>          <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th>   <td>    0.0374</td> <td> 6.21e-05</td> <td>  601.863</td> <td> 0.000</td> <td>    0.037</td> <td>    0.037</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>random_bool</th> <td>   -0.0320</td> <td>    0.000</td> <td> -280.410</td> <td> 0.000</td> <td>   -0.032</td> <td>   -0.032</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.table.SimpleTable'>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# booking ATE without controls \n",
    "smf.ols('booking_bool ~ random_bool', data=df_new).fit().summary().tables[1]\n",
    "\n",
    "# when the sorting is personalized, users are more likley to book the hotel "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "              <td></td>                 <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th>                 <td>    0.0842</td> <td>    0.001</td> <td>   57.293</td> <td> 0.000</td> <td>    0.081</td> <td>    0.087</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>random_bool</th>               <td>    0.0022</td> <td>    0.000</td> <td>   15.256</td> <td> 0.000</td> <td>    0.002</td> <td>    0.003</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>visitor_hist_starrating</th>   <td>   -0.0007</td> <td>    0.000</td> <td>   -1.497</td> <td> 0.134</td> <td>   -0.002</td> <td>    0.000</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>visitor_hist_adr_usd</th>      <td> 3.782e-07</td> <td> 3.03e-06</td> <td>    0.125</td> <td> 0.901</td> <td>-5.56e-06</td> <td> 6.31e-06</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>prop_starrating</th>           <td>    0.0014</td> <td> 6.92e-05</td> <td>   20.778</td> <td> 0.000</td> <td>    0.001</td> <td>    0.002</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>prop_review_score</th>         <td>    0.0021</td> <td> 6.68e-05</td> <td>   31.481</td> <td> 0.000</td> <td>    0.002</td> <td>    0.002</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>prop_brand_bool</th>           <td>    0.0020</td> <td>    0.000</td> <td>   14.142</td> <td> 0.000</td> <td>    0.002</td> <td>    0.002</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>prop_location_score1</th>      <td>   -0.0004</td> <td>  4.6e-05</td> <td>   -8.173</td> <td> 0.000</td> <td>   -0.000</td> <td>   -0.000</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>prop_log_historical_price</th> <td>   -0.0001</td> <td>  3.6e-05</td> <td>   -3.399</td> <td> 0.001</td> <td>   -0.000</td> <td>-5.19e-05</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>position</th>                  <td>   -0.0032</td> <td> 6.28e-06</td> <td> -510.074</td> <td> 0.000</td> <td>   -0.003</td> <td>   -0.003</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>price_usd</th>                 <td>-3.544e-09</td> <td> 4.51e-09</td> <td>   -0.785</td> <td> 0.432</td> <td>-1.24e-08</td> <td>  5.3e-09</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>promotion_flag</th>            <td>    0.0112</td> <td>    0.000</td> <td>   68.744</td> <td> 0.000</td> <td>    0.011</td> <td>    0.012</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>srch_length_of_stay</th>       <td>-9.067e-05</td> <td> 3.46e-05</td> <td>   -2.620</td> <td> 0.009</td> <td>   -0.000</td> <td>-2.28e-05</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>srch_booking_window</th>       <td>-3.024e-06</td> <td> 1.34e-06</td> <td>   -2.250</td> <td> 0.024</td> <td>-5.66e-06</td> <td> -3.9e-07</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>srch_adults_count</th>         <td>    0.0004</td> <td> 8.62e-05</td> <td>    4.293</td> <td> 0.000</td> <td>    0.000</td> <td>    0.001</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>srch_children_count</th>       <td>    0.0006</td> <td>    9e-05</td> <td>    6.141</td> <td> 0.000</td> <td>    0.000</td> <td>    0.001</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>srch_room_count</th>           <td>    0.0007</td> <td>    0.000</td> <td>    4.108</td> <td> 0.000</td> <td>    0.000</td> <td>    0.001</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>srch_saturday_night_bool</th>  <td>    0.0004</td> <td>    0.000</td> <td>    2.650</td> <td> 0.008</td> <td> 9.42e-05</td> <td>    0.001</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>orig_destination_distance</th> <td>  8.22e-08</td> <td> 4.16e-08</td> <td>    1.978</td> <td> 0.048</td> <td> 7.67e-10</td> <td> 1.64e-07</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.table.SimpleTable'>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# clicking ATE with control variables \n",
    "smf.ols('click_bool ~ random_bool + visitor_hist_starrating + visitor_hist_adr_usd + prop_starrating + \\\n",
    "prop_review_score + prop_brand_bool + prop_location_score1+ prop_log_historical_price+ position + price_usd + \\\n",
    "promotion_flag+srch_length_of_stay+srch_booking_window+srch_adults_count+srch_children_count + srch_room_count + \\\n",
    "srch_saturday_night_bool + orig_destination_distance', data=df_new).fit().summary().tables[1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "              <td></td>                 <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th>                 <td>    0.0714</td> <td>    0.001</td> <td>   61.066</td> <td> 0.000</td> <td>    0.069</td> <td>    0.074</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>random_bool</th>               <td>   -0.0313</td> <td>    0.000</td> <td> -270.649</td> <td> 0.000</td> <td>   -0.032</td> <td>   -0.031</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>visitor_hist_starrating</th>   <td>   -0.0006</td> <td>    0.000</td> <td>   -1.523</td> <td> 0.128</td> <td>   -0.001</td> <td>    0.000</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>visitor_hist_adr_usd</th>      <td> 2.909e-06</td> <td> 2.41e-06</td> <td>    1.208</td> <td> 0.227</td> <td>-1.81e-06</td> <td> 7.63e-06</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>prop_starrating</th>           <td> 6.998e-05</td> <td>  5.5e-05</td> <td>    1.271</td> <td> 0.204</td> <td>-3.79e-05</td> <td>    0.000</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>prop_review_score</th>         <td>    0.0017</td> <td> 5.31e-05</td> <td>   32.867</td> <td> 0.000</td> <td>    0.002</td> <td>    0.002</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>prop_brand_bool</th>           <td>    0.0026</td> <td>    0.000</td> <td>   23.103</td> <td> 0.000</td> <td>    0.002</td> <td>    0.003</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>prop_location_score1</th>      <td>   -0.0005</td> <td> 3.66e-05</td> <td>  -12.700</td> <td> 0.000</td> <td>   -0.001</td> <td>   -0.000</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>prop_log_historical_price</th> <td>   -0.0002</td> <td> 2.87e-05</td> <td>   -7.512</td> <td> 0.000</td> <td>   -0.000</td> <td>   -0.000</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>position</th>                  <td>   -0.0023</td> <td>    5e-06</td> <td> -464.755</td> <td> 0.000</td> <td>   -0.002</td> <td>   -0.002</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>price_usd</th>                 <td>-6.967e-09</td> <td> 3.59e-09</td> <td>   -1.941</td> <td> 0.052</td> <td> -1.4e-08</td> <td>  6.7e-11</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>promotion_flag</th>            <td>    0.0084</td> <td>    0.000</td> <td>   64.492</td> <td> 0.000</td> <td>    0.008</td> <td>    0.009</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>srch_length_of_stay</th>       <td>   -0.0008</td> <td> 2.75e-05</td> <td>  -28.599</td> <td> 0.000</td> <td>   -0.001</td> <td>   -0.001</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>srch_booking_window</th>       <td>-1.525e-05</td> <td> 1.07e-06</td> <td>  -14.266</td> <td> 0.000</td> <td>-1.73e-05</td> <td>-1.32e-05</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>srch_adults_count</th>         <td>   -0.0010</td> <td> 6.86e-05</td> <td>  -14.814</td> <td> 0.000</td> <td>   -0.001</td> <td>   -0.001</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>srch_children_count</th>       <td>    0.0004</td> <td> 7.16e-05</td> <td>    5.812</td> <td> 0.000</td> <td>    0.000</td> <td>    0.001</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>srch_room_count</th>           <td>    0.0023</td> <td>    0.000</td> <td>   16.424</td> <td> 0.000</td> <td>    0.002</td> <td>    0.003</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>srch_saturday_night_bool</th>  <td>  -6.9e-05</td> <td>    0.000</td> <td>   -0.636</td> <td> 0.525</td> <td>   -0.000</td> <td>    0.000</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>orig_destination_distance</th> <td> 1.588e-07</td> <td>  3.3e-08</td> <td>    4.805</td> <td> 0.000</td> <td>  9.4e-08</td> <td> 2.24e-07</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.table.SimpleTable'>"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# booking ATE with control variables \n",
    "smf.ols('booking_bool ~ random_bool + visitor_hist_starrating + visitor_hist_adr_usd + prop_starrating + \\\n",
    "prop_review_score + prop_brand_bool + prop_location_score1+ prop_log_historical_price+ position + price_usd + \\\n",
    "promotion_flag+srch_length_of_stay+srch_booking_window+srch_adults_count+srch_children_count + srch_room_count + \\\n",
    "srch_saturday_night_bool + orig_destination_distance', data=df_new).fit().summary().tables[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>click_bool</th>\n",
       "      <th>booking_bool</th>\n",
       "      <th>random_bool</th>\n",
       "      <th>visitor_hist_starrating</th>\n",
       "      <th>visitor_hist_adr_usd</th>\n",
       "      <th>prop_starrating</th>\n",
       "      <th>prop_review_score</th>\n",
       "      <th>prop_brand_bool</th>\n",
       "      <th>prop_location_score1</th>\n",
       "      <th>prop_log_historical_price</th>\n",
       "      <th>position</th>\n",
       "      <th>price_usd</th>\n",
       "      <th>promotion_flag</th>\n",
       "      <th>srch_length_of_stay</th>\n",
       "      <th>srch_booking_window</th>\n",
       "      <th>srch_adults_count</th>\n",
       "      <th>srch_children_count</th>\n",
       "      <th>srch_room_count</th>\n",
       "      <th>srch_saturday_night_bool</th>\n",
       "      <th>orig_destination_distance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>3</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1</td>\n",
       "      <td>2.83</td>\n",
       "      <td>4.95</td>\n",
       "      <td>27</td>\n",
       "      <td>104.77</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1307.017977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>4</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2.20</td>\n",
       "      <td>5.03</td>\n",
       "      <td>26</td>\n",
       "      <td>170.74</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1307.017977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>3</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1</td>\n",
       "      <td>2.20</td>\n",
       "      <td>4.92</td>\n",
       "      <td>21</td>\n",
       "      <td>179.80</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1307.017977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>2</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2.83</td>\n",
       "      <td>4.39</td>\n",
       "      <td>34</td>\n",
       "      <td>602.77</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1307.017977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>4</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1</td>\n",
       "      <td>2.64</td>\n",
       "      <td>4.93</td>\n",
       "      <td>4</td>\n",
       "      <td>143.58</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1307.017977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9917518</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>4</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1</td>\n",
       "      <td>2.94</td>\n",
       "      <td>5.91</td>\n",
       "      <td>15</td>\n",
       "      <td>259.00</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2563.840000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9917519</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>4</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1</td>\n",
       "      <td>3.09</td>\n",
       "      <td>6.21</td>\n",
       "      <td>8</td>\n",
       "      <td>349.00</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2558.310000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9917520</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1</td>\n",
       "      <td>2.56</td>\n",
       "      <td>6.21</td>\n",
       "      <td>16</td>\n",
       "      <td>429.00</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2560.300000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9917521</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3.33</td>\n",
       "      <td>5.63</td>\n",
       "      <td>6</td>\n",
       "      <td>175.00</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2557.770000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9917522</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3.374634</td>\n",
       "      <td>176.588512</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2.83</td>\n",
       "      <td>5.80</td>\n",
       "      <td>25</td>\n",
       "      <td>218.00</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2563.690000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2939652 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         click_bool  booking_bool  random_bool  visitor_hist_starrating  \\\n",
       "0                 0             0            1                 3.374634   \n",
       "1                 0             0            1                 3.374634   \n",
       "2                 0             0            1                 3.374634   \n",
       "3                 0             0            1                 3.374634   \n",
       "4                 0             0            1                 3.374634   \n",
       "...             ...           ...          ...                      ...   \n",
       "9917518           0             0            1                 3.374634   \n",
       "9917519           0             0            1                 3.374634   \n",
       "9917520           0             0            1                 3.374634   \n",
       "9917521           0             0            1                 3.374634   \n",
       "9917522           0             0            1                 3.374634   \n",
       "\n",
       "         visitor_hist_adr_usd  prop_starrating  prop_review_score  \\\n",
       "0                  176.588512                3                3.5   \n",
       "1                  176.588512                4                4.0   \n",
       "2                  176.588512                3                4.5   \n",
       "3                  176.588512                2                4.0   \n",
       "4                  176.588512                4                3.5   \n",
       "...                       ...              ...                ...   \n",
       "9917518            176.588512                4                4.5   \n",
       "9917519            176.588512                4                4.5   \n",
       "9917520            176.588512                5                4.5   \n",
       "9917521            176.588512                3                4.0   \n",
       "9917522            176.588512                3                4.0   \n",
       "\n",
       "         prop_brand_bool  prop_location_score1  prop_log_historical_price  \\\n",
       "0                      1                  2.83                       4.95   \n",
       "1                      1                  2.20                       5.03   \n",
       "2                      1                  2.20                       4.92   \n",
       "3                      1                  2.83                       4.39   \n",
       "4                      1                  2.64                       4.93   \n",
       "...                  ...                   ...                        ...   \n",
       "9917518                1                  2.94                       5.91   \n",
       "9917519                1                  3.09                       6.21   \n",
       "9917520                1                  2.56                       6.21   \n",
       "9917521                1                  3.33                       5.63   \n",
       "9917522                1                  2.83                       5.80   \n",
       "\n",
       "         position  price_usd  promotion_flag  srch_length_of_stay  \\\n",
       "0              27     104.77               0                    1   \n",
       "1              26     170.74               0                    1   \n",
       "2              21     179.80               0                    1   \n",
       "3              34     602.77               0                    1   \n",
       "4               4     143.58               0                    1   \n",
       "...           ...        ...             ...                  ...   \n",
       "9917518        15     259.00               1                    1   \n",
       "9917519         8     349.00               0                    1   \n",
       "9917520        16     429.00               0                    1   \n",
       "9917521         6     175.00               1                    1   \n",
       "9917522        25     218.00               1                    1   \n",
       "\n",
       "         srch_booking_window  srch_adults_count  srch_children_count  \\\n",
       "0                          0                  4                    0   \n",
       "1                          0                  4                    0   \n",
       "2                          0                  4                    0   \n",
       "3                          0                  4                    0   \n",
       "4                          0                  4                    0   \n",
       "...                      ...                ...                  ...   \n",
       "9917518                    3                  2                    2   \n",
       "9917519                    3                  2                    2   \n",
       "9917520                    3                  2                    2   \n",
       "9917521                    3                  2                    2   \n",
       "9917522                    3                  2                    2   \n",
       "\n",
       "         srch_room_count  srch_saturday_night_bool  orig_destination_distance  \n",
       "0                      1                         1                1307.017977  \n",
       "1                      1                         1                1307.017977  \n",
       "2                      1                         1                1307.017977  \n",
       "3                      1                         1                1307.017977  \n",
       "4                      1                         1                1307.017977  \n",
       "...                  ...                       ...                        ...  \n",
       "9917518                1                         0                2563.840000  \n",
       "9917519                1                         0                2558.310000  \n",
       "9917520                1                         0                2560.300000  \n",
       "9917521                1                         0                2557.770000  \n",
       "9917522                1                         0                2563.690000  \n",
       "\n",
       "[2939652 rows x 20 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create data when sorting is random \n",
    "df_random = df_new[df_new['random_bool'] == 1]\n",
    "df_random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "      <td></td>         <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th> <td>    0.0878</td> <td>    0.000</td> <td>  380.452</td> <td> 0.000</td> <td>    0.087</td> <td>    0.088</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>position</th>  <td>   -0.0025</td> <td> 1.18e-05</td> <td> -210.363</td> <td> 0.000</td> <td>   -0.003</td> <td>   -0.002</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.table.SimpleTable'>"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# clicking ATE without controls \n",
    "smf.ols('click_bool ~ position', data=df_random).fit().summary().tables[1]\n",
    "\n",
    "# higher the position, more likely to click "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "      <td></td>         <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th> <td>    0.0112</td> <td> 8.07e-05</td> <td>  139.171</td> <td> 0.000</td> <td>    0.011</td> <td>    0.011</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>position</th>  <td>   -0.0004</td> <td> 4.12e-06</td> <td>  -85.242</td> <td> 0.000</td> <td>   -0.000</td> <td>   -0.000</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.table.SimpleTable'>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# booking ATE without controls \n",
    "smf.ols('booking_bool ~ position', data=df_random).fit().summary().tables[1]\n",
    "\n",
    "# higher the position, more likely to book"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# treatment effect by user's historical ratings\n",
    "df_5 = df[df['visitor_hist_starrating'] == 5]\n",
    "df_4 = df[df['visitor_hist_starrating'] == 4]\n",
    "df_3 = df[df['visitor_hist_starrating'] == 3]\n",
    "df_2 = df[df['visitor_hist_starrating'] == 2]\n",
    "df_1 = df[df['visitor_hist_starrating'] == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9117\n",
      "47756\n",
      "42316\n",
      "18166\n",
      "119\n"
     ]
    }
   ],
   "source": [
    "print(len(df_5))\n",
    "print(len(df_4))\n",
    "print(len(df_3))\n",
    "print(len(df_2))\n",
    "print(len(df_1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, \"Vistor's Historical Ratings\")"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEKCAYAAADTgGjXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAW2ElEQVR4nO3de7BlZXnn8e/PbhFGRSC2FNKYJtqJopkotkhCkkGwoBFHmBmJGEfQwemKQSOa0rRJaph4mcIxpYaMl0EhXKIg3iIC2vZw0TLFpRu5ixY9iNIDI20aEaLRAZ/5Y72N2+b06cNL77PP8Xw/VbvOWs9619rP3n/0r9dlr5WqQpKkHo+ZdAOSpPnLEJEkdTNEJEndDBFJUjdDRJLUzRCRJHUba4gkuT3JjUmuS7K+1fZIsjbJre3v7q2eJKcm2ZDkhiT7j2zn+Db+1iTHj9Sf37a/oa2bcX4eSdIvmo09kRdV1XOrakWbXw1cUlXLgUvaPMARwPL2WgV8GIbQAU4GXggcAJy8JXjamFUj660c/8eRJG0xicNZRwFntemzgKNH6mfX4EpgtyR7AYcDa6tqc1XdA6wFVrZlu1bVFTX8YvLskW1JkmbB4jFvv4AvJyngf1bVacCeVXUXQFXdleQpbezewB0j625stenqG6eoT+vJT35yLVu2rO/TSNICdM0113y/qpZMtWzcIXJQVd3ZgmJtkm9OM3aq8xnVUX/4hpNVDIe9eNrTnsb69eun71qS9JAk39nWsrEezqqqO9vfu4HPMZzT+F47FEX7e3cbvhHYZ2T1pcCd26kvnaI+VR+nVdWKqlqxZMmUYSpJ6jC2EEny+CRP3DINHAbcBFwAbLnC6njg8236AuC4dpXWgcC97bDXGuCwJLu3E+qHAWvasvuSHNiuyjpuZFuSpFkwzsNZewKfa1fdLgY+UVVfSrIOOD/JCcB3gWPa+IuBlwAbgB8BrwWoqs1J3gmsa+PeUVWb2/TrgTOBXYAvtpckaZZkod0KfsWKFeU5EUmauSTXjPxM4xf4i3VJUjdDRJLUzRCRJHUzRCRJ3QwRSVK3cf9iXdIvuWWrL5p0C9O6/ZQjJ93CLzX3RCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd0MEUlSN0NEktTNEJEkdTNEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd0MEUlSN0NEktTNEJEkdTNEJEndDBFJUrfFk25Amq+Wrb5o0i1M6/ZTjpx0C1oA3BORJHUzRCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd3GHiJJFiW5NsmFbX7fJFcluTXJJ5Ps1OqPa/Mb2vJlI9t4e6t/K8nhI/WVrbYhyepxfxZJ0i+ajT2RNwG3jMy/B3h/VS0H7gFOaPUTgHuq6hnA+9s4kuwHHAs8G1gJfKgF0yLgg8ARwH7AK9tYSdIsGWuIJFkKHAl8rM0HOAT4dBtyFnB0mz6qzdOWH9rGHwWcV1U/qapvAxuAA9prQ1XdVlU/Bc5rYyVJs2TceyIfAN4G/KzN/wrwg6p6oM1vBPZu03sDdwC05fe28Q/Vt1pnW/WHSbIqyfok6zdt2vRoP5MkqRlbiCR5KXB3VV0zWp5iaG1n2SOtP7xYdVpVraiqFUuWLJmma0nSIzHOe2cdBLwsyUuAnYFdGfZMdkuyuO1tLAXubOM3AvsAG5MsBp4EbB6pbzG6zrbqkqRZMLY9kap6e1UtraplDCfGL62qVwGXAS9vw44HPt+mL2jztOWXVlW1+rHt6q19geXA1cA6YHm72mun9h4XjOvzSJIebhJ38f0z4Lwk7wKuBU5v9dOBc5JsYNgDORagqm5Ocj7wDeAB4MSqehAgyRuANcAi4IyqunlWP4kkLXCzEiJVdTlweZu+jeHKqq3H/AtwzDbWfzfw7inqFwMX78BWJUmPgL9YlyR1M0QkSd0MEUlSN0NEktTNEJEkdTNEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd0MEUlSN0NEktTNEJEkdTNEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd0MEUlSN0NEktTNEJEkdTNEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3cYWIkl2TnJ1kuuT3Jzkr1p93yRXJbk1ySeT7NTqj2vzG9ryZSPbenurfyvJ4SP1la22IcnqcX0WSdLUxrkn8hPgkKr6LeC5wMokBwLvAd5fVcuBe4AT2vgTgHuq6hnA+9s4kuwHHAs8G1gJfCjJoiSLgA8CRwD7Aa9sYyVJs2RsIVKD+9vsY9urgEOAT7f6WcDRbfqoNk9bfmiStPp5VfWTqvo2sAE4oL02VNVtVfVT4Lw2VpI0S8Z6TqTtMVwH3A2sBf438IOqeqAN2Qjs3ab3Bu4AaMvvBX5ltL7VOtuqS5JmyVhDpKoerKrnAksZ9hyeNdWw9jfbWPZI6w+TZFWS9UnWb9q0afuNS5JmZFauzqqqHwCXAwcCuyVZ3BYtBe5s0xuBfQDa8icBm0frW62zrfpU739aVa2oqhVLlizZER9JksR4r85akmS3Nr0L8GLgFuAy4OVt2PHA59v0BW2etvzSqqpWP7ZdvbUvsBy4GlgHLG9Xe+3EcPL9gnF9HknSwy3e/pBuewFntauoHgOcX1UXJvkGcF6SdwHXAqe38acD5yTZwLAHcixAVd2c5HzgG8ADwIlV9SBAkjcAa4BFwBlVdfMYP48kaStjC5GqugF43hT12xjOj2xd/xfgmG1s693Au6eoXwxc/KiblSR18RfrkqRuhogkqZshIknqNqMQSXLQTGqSpIVlpnsifzvDmiRpAZn26qwkvw38DrAkyVtGFu3KcFmtJGkB294lvjsBT2jjnjhS/yE//8GgJGmBmjZEquorwFeSnFlV35mlniRJ88RMf2z4uCSnActG16mqQ8bRlCRpfphpiHwK+AjwMeDB8bUjSZpPZhoiD1TVh8faiSRp3pnpJb5fSPLHSfZKsseW11g7kyTNeTPdE9lyi/a3jtQK+LUd244kaT6ZUYhU1b7jbkSSNP/MKESSHDdVvarO3rHtSJLmk5keznrByPTOwKHA1wFDRJIWsJkeznrj6HySJwHnjKUjSdK80Xsr+B8xPOtckrSAzfScyBcYrsaC4caLzwLOH1dTkqT5YabnRP56ZPoB4DtVtXEM/UiS5pEZHc5qN2L8JsOdfHcHfjrOpiRJ88NMn2z4B8DVwDHAHwBXJfFW8JK0wM30cNZfAC+oqrsBkiwB/hfw6XE1Jkma+2Z6ddZjtgRI80+PYF1J0i+pme6JfCnJGuDcNv8K4OLxtCRJmi+294z1ZwB7VtVbk/x74HeBAFcAH5+F/iRJc9j2Dkl9ALgPoKo+W1Vvqao3M+yFfGDczUmS5rbthciyqrph62JVrWd4VK4kaQHbXojsPM2yXXZkI5Kk+Wd7IbIuyX/eupjkBOCa8bQkSZovtnd11knA55K8ip+HxgpgJ+DfjbMxSdLcN22IVNX3gN9J8iLgOa18UVVdOvbOJElz3kyfJ3IZcNmYe5EkzTP+6lyS1G1sIZJknySXJbklyc1J3tTqeyRZm+TW9nf3Vk+SU5NsSHJDkv1HtnV8G39rkuNH6s9PcmNb59QkGdfnkSQ93Dj3RB4A/rSqngUcCJyYZD9gNXBJVS0HLmnzAEcwPC1xObAK+DAMoQOcDLwQOAA4eUvwtDGrRtZbOcbPI0naythCpKruqqqvt+n7gFuAvYGjgLPasLOAo9v0UcDZNbgS2C3JXsDhwNqq2lxV9wBrgZVt2a5VdUVVFXD2yLYkSbNgVs6JJFkGPA+4iuFeXHfBEDTAU9qwvYE7Rlbb2GrT1TdOUZckzZKxh0iSJwCfAU6qqh9ON3SKWnXUp+phVZL1SdZv2rRpey1LkmZorCGS5LEMAfLxqvpsK3+vHYqi/d3ynJKNwD4jqy8F7txOfekU9YepqtOqakVVrViyZMmj+1CSpIeM8+qsAKcDt1TV+0YWXQBsucLqeODzI/Xj2lVaBwL3tsNda4DDkuzeTqgfBqxpy+5LcmB7r+NGtiVJmgUzfShVj4OAVwM3Jrmu1f4cOAU4v91/67sMz22H4fbyLwE2AD8CXgtQVZuTvBNY18a9o6o2t+nXA2cy3Azyi+0lSZolYwuRqvoaU5+3ADh0ivEFnLiNbZ0BnDFFfT0/vx2LJGmW+Yt1SVI3Q0SS1M0QkSR1G+eJdf2SWrb6okm3MK3bTzly0i1IC4Z7IpKkboaIJKmbISJJ6maISJK6GSKSpG6GiCSpmyEiSepmiEiSuhkikqRuhogkqZshIknqZohIkroZIpKkboaIJKmbISJJ6maISJK6GSKSpG6GiCSpmyEiSepmiEiSuhkikqRuhogkqZshIknqZohIkroZIpKkboaIJKmbISJJ6maISJK6GSKSpG6GiCSpmyEiSeo2thBJckaSu5PcNFLbI8naJLe2v7u3epKcmmRDkhuS7D+yzvFt/K1Jjh+pPz/JjW2dU5NkXJ9FkjS1ce6JnAms3Kq2GrikqpYDl7R5gCOA5e21CvgwDKEDnAy8EDgAOHlL8LQxq0bW2/q9JEljNrYQqaqvApu3Kh8FnNWmzwKOHqmfXYMrgd2S7AUcDqytqs1VdQ+wFljZlu1aVVdUVQFnj2xLkjRLZvucyJ5VdRdA+/uUVt8buGNk3MZWm66+cYr6lJKsSrI+yfpNmzY96g8hSRrMlRPrU53PqI76lKrqtKpaUVUrlixZ0tmiJGlrsx0i32uHomh/7271jcA+I+OWAndup750irokaRbNdohcAGy5wup44PMj9ePaVVoHAve2w11rgMOS7N5OqB8GrGnL7ktyYLsq67iRbUmSZsnicW04ybnAwcCTk2xkuMrqFOD8JCcA3wWOacMvBl4CbAB+BLwWoKo2J3knsK6Ne0dVbTlZ/3qGK8B2Ab7YXpI0EctWXzTpFqZ1+ylHjmW7YwuRqnrlNhYdOsXYAk7cxnbOAM6Yor4eeM6j6VGS9OjMlRPrkqR5yBCRJHUzRCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd0MEUlSN0NEktTNEJEkdTNEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd0MEUlSN0NEktTNEJEkdTNEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd0MEUlSN0NEktTNEJEkdZv3IZJkZZJvJdmQZPWk+5GkhWReh0iSRcAHgSOA/YBXJtlvsl1J0sIxr0MEOADYUFW3VdVPgfOAoybckyQtGIsn3cCjtDdwx8j8RuCF43qzZasvGtemH7XbTzly0i1IWoBSVZPuoVuSY4DDq+p1bf7VwAFV9catxq0CVrXZ3wC+NauNbtuTge9Puok5wO9h4Pcw8HsYzKXv4VeraslUC+b7nshGYJ+R+aXAnVsPqqrTgNNmq6mZSrK+qlZMuo9J83sY+D0M/B4G8+V7mO/nRNYBy5Psm2Qn4Fjgggn3JEkLxrzeE6mqB5K8AVgDLALOqKqbJ9yWJC0Y8zpEAKrqYuDiSffRac4dYpsQv4eB38PA72EwL76HeX1iXZI0WfP9nIgkaYIMkQlIckaSu5PcNOleJiXJPkkuS3JLkpuTvGnSPU1Kkp2TXJ3k+vZd/NWke5qUJIuSXJvkwkn3MklJbk9yY5LrkqyfdD/T8XDWBCT5feB+4Oyqes6k+5mEJHsBe1XV15M8EbgGOLqqvjHh1mZdkgCPr6r7kzwW+Brwpqq6csKtzbokbwFWALtW1Usn3c+kJLkdWFFVc+V3ItvknsgEVNVXgc2T7mOSququqvp6m74PuIXhDgQLTg3ub7OPba8F97+7JEuBI4GPTboXzZwhoolLsgx4HnDVZDuZnHYY5zrgbmBtVS3E7+IDwNuAn026kTmggC8nuabdcWPOMkQ0UUmeAHwGOKmqfjjpfialqh6squcy3HXhgCQL6jBnkpcCd1fVNZPuZY44qKr2Z7hD+YntEPicZIhoYtrx/88AH6+qz066n7mgqn4AXA6snHArs+0g4GXtXMB5wCFJ/n6yLU1OVd3Z/t4NfI7hjuVzkiGiiWgnk08Hbqmq9026n0lKsiTJbm16F+DFwDcn29Xsqqq3V9XSqlrGcPuiS6vqP064rYlI8vh2sQlJHg8cBszZKzkNkQlIci5wBfAbSTYmOWHSPU3AQcCrGf7HeV17vWTSTU3IXsBlSW5guB/c2qpa0Je4LnB7Al9Lcj1wNXBRVX1pwj1tk5f4SpK6uSciSepmiEiSuhkikqRuhogkqZshIknqZohozkhyeZLDt6qdlORDSZ6a5NPTrLtbkj/ufN/bH2GPK0bml225G3OSFUlOnWbdZUn+sKO/aT/7dtY9M8nLt1H/dru0+vokh85gW69J8tSR+Y8l2a+nL/3yMEQ0l5zL8EOzUccC51bVnVX1sH8MR+wGPKIQSbLoEfY3rapaX1V/Ms2QZcAjCpEki2fw2Xu9td1q5STgIzMY/xrgoRCpqtctxLsu6xcZIppLPg28NMnj4KEbMz6V4YdXo//jf3Z7/sZ1SW5Ishw4BXh6q703g/cmuak9l+EVbd2D23NMPgHc2N53U1u2V5Kvtm3clOT3HknzbdsXtul/M/IjymvbL5BPAX6v1d7cniPyd62/a5O8qK37miSfSvIFhpvwjX72RUn+uq1zQ5I3tvp/SbKu9X1auyPATF3ByB2Up9pW25tZAXy89b/L6F5ZkvuTvLvt1VyZZM9Wf3qbX5fkHUnu3xHfteaQqvLla868gIuAo9r0auC9bXoZcFOb/lvgVW16J2CX0eWt/h+AtcAihl8Af5fhl+EHA/8M7DvFe/8p8BdtehHwxCnGXA58C7iuvb4x0tfBwIVt+gsMN9EDeAKweHT5yPv9XZt+ZutxZ4b/8W8E9pjis7+e4X5ji9v8HqN/2/Q5wL9t02cCL5/iczxUB44GPjGybFvbupzhGRdsPc9w19kt4/478Jdt+kLglW36j4D7Z/pd+5ofL/dENNeMHtI6ts1v7Qrgz5P8GfCrVfXjKcb8LsNhsAer6nvAV4AXtGVXV9W3p1hnHfDaJP8V+M0annMylVdV1XNrOBS0rVu1/CPwviR/AuxWVQ9so8dzAKrqm8B3gF9vy9ZW1VTPnHkx8JEt2xsZ86IkVyW5ETgEePY2+hr13iS3AX8P/LeRes+2fsoQGDA8YGxZm/5t4FNt+hMj42f6XWuOM0Q01/wDcGiS/YFdqj24alRVfQJ4GfBjYE2SQ6bYznSHc/55qmINDwv7feD/AOckOe6RNj+yrVOA1zHsJV2Z5Jk7ose2zi/cqyjJzsCHGPYsfhP4KMMezfa8FXgG8JfAWY9yW/+vqrb09SDDntc27cjvWpNliGhOqeEJf5cDZzD1XghJfg24rapOBS4A/jVwH/DEkWFfBV7RziEsYfgH6+rp3jvJrzI80+KjDHcY3r/3cyR5elXdWFXvAdYzHK6aqsdXtfG/DjyN4VDZdL4M/FGSxW29Pfj5P/Lfz/B8lhmfhK+qnwF/Azwmw5Vx021r6/5n4kqGQ4swctHEjvyuNVmGiOaic4HfYniuxFReAdyU4UmAz2R4Vv0/Af/YTtK+l+EZDDcA1wOXAm+rqv+7nfc9GLguybUM//D9zaP4DCe1Xq5n2GP6YuvngXby+c0M/+Nf1A4bfRJ4TVX9ZDvb/RjDuZMb2rb/sIZnkHyU4UKBf2A4VDRjbQ/iXQzf0XTbOhP4yJYT6zPc/EnAW5JczXBO6t5WP5gd911rgryLr6SxSfKvgB9XVSU5luEk+1GT7ks7zrTHLSXpUXo+8D/aJcc/AP7ThPvRDuaeiCSpm+dEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3QwRSVK3/w9Fmj+/IxvCAQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot histograph for historical ratings \n",
    "y = [119, 18166, 42316, 47756, 9117]\n",
    "x = [\"1\", \"2\", \"3\", \"4\", \"5\"]\n",
    "plt.bar(x, y, width = 0.6)\n",
    "plt.ylabel(\"Count\")\n",
    "plt.xlabel(\"Vistor's Historical Ratings\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average treatment effect for 5 starting rating users:\n",
      "===============================================================================\n",
      "                  coef    std err          t      P>|t|      [0.025      0.975]\n",
      "-------------------------------------------------------------------------------\n",
      "Intercept       0.0452      0.002     19.022      0.000       0.040       0.050\n",
      "random_bool     0.0072      0.006      1.144      0.253      -0.005       0.020\n",
      "===============================================================================\n",
      "average treatment effect for 4 starting rating users:\n",
      "===============================================================================\n",
      "                  coef    std err          t      P>|t|      [0.025      0.975]\n",
      "-------------------------------------------------------------------------------\n",
      "Intercept       0.0439      0.001     42.678      0.000       0.042       0.046\n",
      "random_bool     0.0009      0.003      0.345      0.730      -0.004       0.006\n",
      "===============================================================================\n",
      "average treatment effect for 3 starting rating users:\n",
      "===============================================================================\n",
      "                  coef    std err          t      P>|t|      [0.025      0.975]\n",
      "-------------------------------------------------------------------------------\n",
      "Intercept       0.0449      0.001     41.302      0.000       0.043       0.047\n",
      "random_bool    -0.0013      0.003     -0.444      0.657      -0.007       0.004\n",
      "===============================================================================\n",
      "average treatment effect for 2 starting rating users:\n",
      "===============================================================================\n",
      "                  coef    std err          t      P>|t|      [0.025      0.975]\n",
      "-------------------------------------------------------------------------------\n",
      "Intercept       0.0443      0.002     26.404      0.000       0.041       0.048\n",
      "random_bool    -0.0025      0.004     -0.626      0.531      -0.010       0.005\n",
      "===============================================================================\n",
      "average treatment effect for 1 starting rating users:\n",
      "===============================================================================\n",
      "                  coef    std err          t      P>|t|      [0.025      0.975]\n",
      "-------------------------------------------------------------------------------\n",
      "Intercept       0.0549      0.023      2.377      0.019       0.009       0.101\n",
      "random_bool    -0.0192      0.048     -0.404      0.687      -0.114       0.075\n",
      "===============================================================================\n"
     ]
    }
   ],
   "source": [
    "print(\"average treatment effect for 5 starting rating users:\")\n",
    "print(smf.ols('click_bool ~ random_bool', data=df_5).fit().summary().tables[1])\n",
    "print(\"average treatment effect for 4 starting rating users:\")\n",
    "print(smf.ols('click_bool ~ random_bool', data=df_4).fit().summary().tables[1])\n",
    "print(\"average treatment effect for 3 starting rating users:\")\n",
    "print(smf.ols('click_bool ~ random_bool', data=df_3).fit().summary().tables[1])\n",
    "print(\"average treatment effect for 2 starting rating users:\")\n",
    "print(smf.ols('click_bool ~ random_bool', data=df_2).fit().summary().tables[1])\n",
    "print(\"average treatment effect for 1 starting rating users:\")\n",
    "print(smf.ols('click_bool ~ random_bool', data=df_1).fit().summary().tables[1])\n",
    "\n",
    "# 2 observations:\n",
    "# first, when calcuate the ATE by groups, results become no longer statistically significant\n",
    "# second, the treatment has the largest effect for high star rating users; for low star rating users, the treatment effect is even negative"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAD4CAYAAAAZ1BptAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAXdklEQVR4nO3df6zd9X3f8edrdk2TtMQmmIzZZnaa224O6hbiJt6yRVlYjCFRzKQwGXXDSy1Zy6BLp1WNWSZRJUGCristEkFyYw8TRTiIpsNazFyL0EWTwg8TEsBQ4hvD4AaKndqhbFGgTt7743xucnJ9vvf6nmuu7zXPh3R0vt/39/P9ns9H5+i+7vfHOd9UFZIkDfK3TncHJElzlyEhSepkSEiSOhkSkqROhoQkqdPC092BU+3cc8+tlStXnu5uSNK88vDDD3+vqpZOrJ9xIbFy5Ur2799/urshSfNKkv8zqO7hJklSJ0NCktTJkJAkdTIkJEmdpgyJJDuSHE7y+IT6byZ5KsmBJL/XV782yWhbdklffX2rjSbZ2ldfleSBJAeTfCnJolY/q82PtuUrT8WAJUkn72T2JG4D1vcXkvwzYAPwq1X1DuD3W301sBF4R1vnc0kWJFkA3AJcCqwGrmxtAW4EbqqqEeAYsLnVNwPHqurtwE2tnSRpFk0ZElX1NeDohPLHgRuq6pXW5nCrbwB2VdUrVfU0MAq8uz1Gq+pQVb0K7AI2JAnwAeCutv5O4PK+be1s03cBF7f2kqRZMuw5iV8G/mk7DPS/kvxaqy8DnutrN9ZqXfW3AN+vquMT6j+zrbb8pdZekjRLhv0y3UJgCbAW+DXgziRvAwb9p18MDqOapD1TLPsZSbYAWwAuuOCCSTsuSTp5w4bEGPDl6t2x6MEkPwbObfUVfe2WA8+36UH17wGLkyxsewv97ce3NZZkIfBmTjzsBUBVbQO2AaxZs2bW7qK0cutXBtafueFDs9UFSXpNDXu46b/TO5dAkl8GFtH7g78b2NiuTFoFjAAPAg8BI+1KpkX0Tm7vbiFzH/DRtt1NwN1tenebpy3/ankbPUmaVVPuSSS5A3g/cG6SMeA6YAewo10W+yqwqf0BP5DkTuAJ4DhwdVX9qG3nGmAvsADYUVUH2kt8EtiV5LPAI8D2Vt8OfCHJKL09iI2nYLySpGmYMiSq6sqORf+qo/31wPUD6nuAPQPqh+hd/TSx/kPgiqn6J0l67fiNa0lSJ0NCktTJkJAkdTIkJEmdDAlJUidDQpLUyZCQJHUyJCRJnQwJSVInQ0KS1MmQkCR1MiQkSZ0MCUlSJ0NCktTJkJAkdTIkJEmdDAlJUqcpQyLJjiSH261KJy777SSV5Nw2nyQ3JxlN8miSi/rabkpysD029dXfleSxts7NSdLq5yTZ19rvS7Lk1AxZknSyTmZP4jZg/cRikhXAB4Fn+8qXAiPtsQW4tbU9h969sd9D71al1/X90b+1tR1fb/y1tgL3VtUIcG+blyTNoilDoqq+BhwdsOgm4HeA6qttAG6vnvuBxUnOBy4B9lXV0ao6BuwD1rdlZ1fV16uqgNuBy/u2tbNN7+yrS5JmyVDnJJJ8BPhuVX1rwqJlwHN982OtNll9bEAd4K1V9QJAez5vmL5Kkoa3cLorJHkj8Clg3aDFA2o1RH26fdpC75AVF1xwwXRXlyR1GGZP4peAVcC3kjwDLAe+keRv09sTWNHXdjnw/BT15QPqAC+2w1G058NdHaqqbVW1pqrWLF26dIghSZIGmXZIVNVjVXVeVa2sqpX0/tBfVFV/CewGrmpXOa0FXmqHivYC65IsaSes1wF727KXk6xtVzVdBdzdXmo3MH4V1Ka+uiRplpzMJbB3AF8HfiXJWJLNkzTfAxwCRoE/Bv4dQFUdBT4DPNQen241gI8Dn2/rfAe4p9VvAD6Y5CC9q6humN7QJEkzNeU5iaq6corlK/umC7i6o90OYMeA+n7gwgH1vwIunqp/kqTXjt+4liR1MiQkSZ0MCUlSJ0NCktTJkJAkdTIkJEmdDAlJUidDQpLUyZCQJHUyJCRJnQwJSVInQ0KS1MmQkCR1MiQkSZ0MCUlSJ0NCktTJkJAkdTqZ25fuSHI4yeN9tf+S5C+SPJrkT5Ms7lt2bZLRJE8luaSvvr7VRpNs7auvSvJAkoNJvpRkUauf1eZH2/KVp2rQkqSTczJ7ErcB6yfU9gEXVtWvAt8GrgVIshrYCLyjrfO5JAuSLABuAS4FVgNXtrYANwI3VdUIcAwYv4f2ZuBYVb0duKm1kyTNoilDoqq+BhydUPuzqjreZu8HlrfpDcCuqnqlqp4GRoF3t8doVR2qqleBXcCGJAE+ANzV1t8JXN63rZ1t+i7g4tZekjRLTsU5id8A7mnTy4Dn+paNtVpX/S3A9/sCZ7z+M9tqy19q7SVJs2RGIZHkU8Bx4IvjpQHNaoj6ZNsa1I8tSfYn2X/kyJHJOy1JOmlDh0SSTcCHgV+vqvE/3mPAir5my4HnJ6l/D1icZOGE+s9sqy1/MxMOe42rqm1Vtaaq1ixdunTYIUmSJhgqJJKsBz4JfKSqftC3aDewsV2ZtAoYAR4EHgJG2pVMi+id3N7dwuU+4KNt/U3A3X3b2tSmPwp8tS+MJEmzYOFUDZLcAbwfODfJGHAdvauZzgL2tXPJ91fVv62qA0nuBJ6gdxjq6qr6UdvONcBeYAGwo6oOtJf4JLAryWeBR4Dtrb4d+EKSUXp7EBtPwXglSdMwZUhU1ZUDytsH1MbbXw9cP6C+B9gzoH6I3tVPE+s/BK6Yqn+SpNfOlCEhWLn1K6e7C5J0WvizHJKkToaEJKmTISFJ6mRISJI6GRKSpE6GhCSpkyEhSepkSEiSOhkSkqROhoQkqZMhIUnqZEhIkjoZEpKkToaEJKmTISFJ6mRISJI6ncztS3cAHwYOV9WFrXYO8CVgJfAM8C+r6lh69zL9I+Ay4AfAv6mqb7R1NgH/uW32s1W1s9XfBdwGvIHenes+UVXV9RozHvEsmOwmRc/c8KFZ7IkkzczJ7EncBqyfUNsK3FtVI8C9bR7gUmCkPbYAt8JPQuU64D30blV6XZIlbZ1bW9vx9dZP8RqSpFkyZUhU1deAoxPKG4CdbXoncHlf/fbquR9YnOR84BJgX1UdbXsD+4D1bdnZVfX1qirg9gnbGvQakqRZMuw5ibdW1QsA7fm8Vl8GPNfXbqzVJquPDahP9hqSpFlyqk9cZ0CthqhP70WTLUn2J9l/5MiR6a4uSeowbEi82A4V0Z4Pt/oYsKKv3XLg+SnqywfUJ3uNE1TVtqpaU1Vrli5dOuSQJEkTDRsSu4FNbXoTcHdf/ar0rAVeaoeK9gLrkixpJ6zXAXvbspeTrG1XRl01YVuDXkOSNEtO5hLYO4D3A+cmGaN3ldINwJ1JNgPPAle05nvoXf46Su8S2I8BVNXRJJ8BHmrtPl1V4yfDP85PL4G9pz2Y5DUkSbNkypCoqis7Fl08oG0BV3dsZwewY0B9P3DhgPpfDXoNSdLs8RvXkqROhoQkqZMhIUnqZEhIkjoZEpKkToaEJKmTISFJ6mRISJI6GRKSpE6GhCSpkyEhSepkSEiSOhkSkqROhoQkqZMhIUnqZEhIkjoZEpKkToaEJKnTjEIiyX9IciDJ40nuSPLzSVYleSDJwSRfSrKotT2rzY+25Sv7tnNtqz+V5JK++vpWG02ydSZ9lSRN39AhkWQZ8O+BNVV1IbAA2AjcCNxUVSPAMWBzW2UzcKyq3g7c1NqRZHVb7x3AeuBzSRYkWQDcAlwKrAaubG0lSbNkpoebFgJvSLIQeCPwAvAB4K62fCdweZve0OZpyy9OklbfVVWvVNXTwCjw7vYYrapDVfUqsKu1lSTNkqFDoqq+C/w+8Cy9cHgJeBj4flUdb83GgGVtehnwXFv3eGv/lv76hHW66idIsiXJ/iT7jxw5MuyQJEkTzORw0xJ6/9mvAv4O8CZ6h4YmqvFVOpZNt35isWpbVa2pqjVLly6dquuSpJM0k8NN/xx4uqqOVNXfAF8G/jGwuB1+AlgOPN+mx4AVAG35m4Gj/fUJ63TVJUmzZCYh8SywNskb27mFi4EngPuAj7Y2m4C72/TuNk9b/tWqqlbf2K5+WgWMAA8CDwEj7WqpRfRObu+eQX8lSdO0cOomg1XVA0nuAr4BHAceAbYBXwF2Jflsq21vq2wHvpBklN4exMa2nQNJ7qQXMMeBq6vqRwBJrgH20rtyakdVHRi2v5Kk6Rs6JACq6jrgugnlQ/SuTJrY9ofAFR3buR64fkB9D7BnJn2UJA3Pb1xLkjoZEpKkToaEJKmTISFJ6mRISJI6GRKSpE6GhCSpkyEhSepkSEiSOhkSkqROhoQkqZMhIUnqZEhIkjoZEpKkToaEJKmTISFJ6mRISJI6zSgkkixOcleSv0jyZJJ/lOScJPuSHGzPS1rbJLk5yWiSR5Nc1LedTa39wSSb+urvSvJYW+fmdi9tSdIsmemexB8B/7Oq/h7wD4Anga3AvVU1Atzb5gEuBUbaYwtwK0CSc+jdAvU99G57et14sLQ2W/rWWz/D/kqSpmHokEhyNvA+YDtAVb1aVd8HNgA7W7OdwOVtegNwe/XcDyxOcj5wCbCvqo5W1TFgH7C+LTu7qr5eVQXc3rctSdIsmMmexNuAI8B/S/JIks8neRPw1qp6AaA9n9faLwOe61t/rNUmq48NqJ8gyZYk+5PsP3LkyAyGJEnqN5OQWAhcBNxaVe8E/h8/PbQ0yKDzCTVE/cRi1baqWlNVa5YuXTp5ryVJJ20mITEGjFXVA23+Lnqh8WI7VER7PtzXfkXf+suB56eoLx9QlyTNkqFDoqr+Enguya+00sXAE8BuYPwKpU3A3W16N3BVu8ppLfBSOxy1F1iXZEk7Yb0O2NuWvZxkbbuq6aq+bUmSZsHCGa7/m8AXkywCDgEfoxc8dybZDDwLXNHa7gEuA0aBH7S2VNXRJJ8BHmrtPl1VR9v0x4HbgDcA97THvLZy61cG1p+54UOz3BNJmtqMQqKqvgmsGbDo4gFtC7i6Yzs7gB0D6vuBC2fSR0nS8PzGtSSpkyEhSepkSEiSOhkSkqROhoQkqZMhIUnqZEhIkjoZEpKkToaEJKmTISFJ6mRISJI6GRKSpE6GhCSpkyEhSepkSEiSOhkSkqROhoQkqdOMQyLJgiSPJPkfbX5VkgeSHEzypXZrU5Kc1eZH2/KVfdu4ttWfSnJJX319q40m2TrTvkqSpudU7El8Aniyb/5G4KaqGgGOAZtbfTNwrKreDtzU2pFkNbAReAewHvhcC54FwC3ApcBq4MrWVpI0S2YUEkmWAx8CPt/mA3wAuKs12Qlc3qY3tHna8otb+w3Arqp6paqeBkaBd7fHaFUdqqpXgV2trSRplsx0T+IPgd8Bftzm3wJ8v6qOt/kxYFmbXgY8B9CWv9Ta/6Q+YZ2u+gmSbEmyP8n+I0eOzHBIkqRxQ4dEkg8Dh6vq4f7ygKY1xbLp1k8sVm2rqjVVtWbp0qWT9FqSNB0LZ7Due4GPJLkM+HngbHp7FouTLGx7C8uB51v7MWAFMJZkIfBm4GhffVz/Ol11SdIsGHpPoqqurarlVbWS3onnr1bVrwP3AR9tzTYBd7fp3W2etvyrVVWtvrFd/bQKGAEeBB4CRtrVUovaa+wetr+SpOmbyZ5El08Cu5J8FngE2N7q24EvJBmltwexEaCqDiS5E3gCOA5cXVU/AkhyDbAXWADsqKoDr0F/JUkdTklIVNWfA3/epg/RuzJpYpsfAld0rH89cP2A+h5gz6nooyRp+vzGtSSpkyEhSepkSEiSOhkSkqROhoQkqZMhIUnqZEhIkjoZEpKkToaEJKmTISFJ6mRISJI6vRY/8KchrNz6lYH1Z2740Cz3RJJ+yj0JSVInQ0KS1MmQkCR1MiQkSZ0MCUlSp6FDIsmKJPcleTLJgSSfaPVzkuxLcrA9L2n1JLk5yWiSR5Nc1LetTa39wSSb+urvSvJYW+fmJJnJYCVJ0zOTPYnjwH+sqr8PrAWuTrIa2ArcW1UjwL1tHuBSYKQ9tgC3Qi9UgOuA99C77el148HS2mzpW2/9DPorSZqmoUOiql6oqm+06ZeBJ4FlwAZgZ2u2E7i8TW8Abq+e+4HFSc4HLgH2VdXRqjoG7APWt2VnV9XXq6qA2/u2JUmaBafknESSlcA7gQeAt1bVC9ALEuC81mwZ8FzfamOtNll9bEB90OtvSbI/yf4jR47MdDiSpGbGIZHkF4A/AX6rqv56sqYDajVE/cRi1baqWlNVa5YuXTpVlyVJJ2lGIZHk5+gFxBer6sut/GI7VER7PtzqY8CKvtWXA89PUV8+oC5JmiUzubopwHbgyar6g75Fu4HxK5Q2AXf31a9qVzmtBV5qh6P2AuuSLGknrNcBe9uyl5Osba91Vd+2JEmzYCY/8Pde4F8DjyX5Zqv9J+AG4M4km4FngSvasj3AZcAo8APgYwBVdTTJZ4CHWrtPV9XRNv1x4DbgDcA97SFJmiVDh0RV/W8GnzcAuHhA+wKu7tjWDmDHgPp+4MJh+yhJmhm/cS1J6mRISJI6edOhOc6bEUk6nQyJPl1/kCXp9crDTZKkToaEJKmTISFJ6mRISJI6GRKSpE6GhCSpk5fAzlN+f0LSbHBPQpLUyZCQJHUyJCRJnTwncYbxXIWkU8k9CUlSJ/ckXifcw5A0jDm/J5FkfZKnkowm2Xq6+yNJrydzek8iyQLgFuCDwBjwUJLdVfXE6e3ZmWO6P4/unof0+jKnQwJ4NzBaVYcAkuwCNgCGxGlyqu65MVnYeGhMmjvmekgsA57rmx8D3jOxUZItwJY2+3+TPDXk650LfG/Idee6OTW23HhK15lTYzvFHNv8NB/H9ncHFed6SGRArU4oVG0Dts34xZL9VbVmptuZixzb/OTY5qczaWxz/cT1GLCib3458Pxp6oskve7M9ZB4CBhJsirJImAjsPs090mSXjfm9OGmqjqe5BpgL7AA2FFVB17Dl5zxIas5zLHNT45tfjpjxpaqEw7xS5IEzP3DTZKk08iQkCR1MiSaM+HnP5I8k+SxJN9Msr/VzkmyL8nB9ryk1ZPk5jbeR5NcdHp7/7OS7EhyOMnjfbVpjyXJptb+YJJNp2MsE3WM7XeTfLe9d99Mclnfsmvb2J5KcklffU59ZpOsSHJfkieTHEjyiVaf9+/bJGOb9+/blKrqdf+gd1L8O8DbgEXAt4DVp7tfQ4zjGeDcCbXfA7a26a3AjW36MuAeet9FWQs8cLr7P6Hf7wMuAh4fdizAOcCh9rykTS+Zo2P7XeC3B7Rd3T6PZwGr2ud0wVz8zALnAxe16V8Evt36P+/ft0nGNu/ft6ke7kn0/OTnP6rqVWD85z/OBBuAnW16J3B5X/326rkfWJzk/NPRwUGq6mvA0Qnl6Y7lEmBfVR2tqmPAPmD9a9/7yXWMrcsGYFdVvVJVTwOj9D6vc+4zW1UvVNU32vTLwJP0fjVh3r9vk4yty7x536ZiSPQM+vmPyT4Ac1UBf5bk4fZTJQBvraoXoPdBB85r9fk45umOZb6N8Zp22GXH+CEZ5unYkqwE3gk8wBn2vk0YG5xB79sghkTPSf38xzzw3qq6CLgUuDrJ+yZpe6aMGbrHMp/GeCvwS8A/BF4A/murz7uxJfkF4E+A36qqv56s6YDafBvbGfO+dTEkes6In/+oqufb82HgT+nt2r44fhipPR9uzefjmKc7lnkzxqp6sap+VFU/Bv6Y3nsH82xsSX6O3h/RL1bVl1v5jHjfBo3tTHnfJmNI9Mz7n/9I8qYkvzg+DawDHqc3jvGrQzYBd7fp3cBV7QqTtcBL44cE5rDpjmUvsC7JknYYYF2rzTkTzgf9C3rvHfTGtjHJWUlWASPAg8zBz2ySANuBJ6vqD/oWzfv3rWtsZ8L7NqXTfeZ8rjzoXWnxbXpXHnzqdPdniP6/jd6VEt8CDoyPAXgLcC9wsD2f0+qhd0On7wCPAWtO9xgmjOcOervvf0Pvv6/Nw4wF+A16Jw1HgY+d7nFNMrYvtL4/Su+Pxvl97T/VxvYUcOlc/cwC/4TeoZNHgW+2x2Vnwvs2ydjm/fs21cOf5ZAkdfJwkySpkyEhSepkSEiSOhkSkqROhoQkqZMhIUnqZEhIkjr9f2oOQ1HtenYxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(df['visitor_hist_adr_usd'], bins=50)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    507612.000000\n",
       "mean        176.588512\n",
       "std         108.434842\n",
       "min           0.000000\n",
       "25%         110.330000\n",
       "50%         151.830000\n",
       "75%         214.350000\n",
       "max        2768.930000\n",
       "Name: visitor_hist_adr_usd, dtype: float64"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['visitor_hist_adr_usd'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# treatment effect by user's historical spendings\n",
    "df_low = df[df['visitor_hist_adr_usd'] <= 100]\n",
    "df_lowm = df[(df['visitor_hist_adr_usd'] > 100) & (df['visitor_hist_adr_usd'] <= 200)]\n",
    "df_medium = df[(df['visitor_hist_adr_usd'] > 200) & (df['visitor_hist_adr_usd'] <= 300)]\n",
    "df_mhigh = df[(df['visitor_hist_adr_usd'] > 300) & (df['visitor_hist_adr_usd'] <= 400)]\n",
    "df_high = df[df['visitor_hist_adr_usd'] >= 400]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average treatment effect for high spending users:\n",
      "===============================================================================\n",
      "                  coef    std err          t      P>|t|      [0.025      0.975]\n",
      "-------------------------------------------------------------------------------\n",
      "Intercept       0.0455      0.002     26.624      0.000       0.042       0.049\n",
      "random_bool     0.0129      0.005      2.776      0.006       0.004       0.022\n",
      "===============================================================================\n",
      "average treatment effect for medium-high spending users:\n",
      "===============================================================================\n",
      "                  coef    std err          t      P>|t|      [0.025      0.975]\n",
      "-------------------------------------------------------------------------------\n",
      "Intercept       0.0442      0.001     34.201      0.000       0.042       0.047\n",
      "random_bool     0.0063      0.003      1.856      0.063      -0.000       0.013\n",
      "===============================================================================\n",
      "average treatment effect for medium spending users:\n",
      "===============================================================================\n",
      "                  coef    std err          t      P>|t|      [0.025      0.975]\n",
      "-------------------------------------------------------------------------------\n",
      "Intercept       0.0435      0.001     61.266      0.000       0.042       0.045\n",
      "random_bool     0.0043      0.002      2.670      0.008       0.001       0.007\n",
      "===============================================================================\n",
      "average treatment effect for medium-low spending users:\n",
      "===============================================================================\n",
      "                  coef    std err          t      P>|t|      [0.025      0.975]\n",
      "-------------------------------------------------------------------------------\n",
      "Intercept       0.0438      0.000    100.283      0.000       0.043       0.045\n",
      "random_bool     0.0033      0.001      2.903      0.004       0.001       0.006\n",
      "===============================================================================\n",
      "average treatment effect for low spending users:\n",
      "===============================================================================\n",
      "                  coef    std err          t      P>|t|      [0.025      0.975]\n",
      "-------------------------------------------------------------------------------\n",
      "Intercept       0.0435      0.001     60.621      0.000       0.042       0.045\n",
      "random_bool    -0.0007      0.002     -0.364      0.716      -0.004       0.003\n",
      "===============================================================================\n"
     ]
    }
   ],
   "source": [
    "print(\"average treatment effect for high spending users:\")\n",
    "print(smf.ols('click_bool ~ random_bool', data=df_high).fit().summary().tables[1])\n",
    "print(\"average treatment effect for medium-high spending users:\")\n",
    "print(smf.ols('click_bool ~ random_bool', data=df_mhigh).fit().summary().tables[1])\n",
    "print(\"average treatment effect for medium spending users:\")\n",
    "print(smf.ols('click_bool ~ random_bool', data=df_medium).fit().summary().tables[1])\n",
    "print(\"average treatment effect for medium-low spending users:\")\n",
    "print(smf.ols('click_bool ~ random_bool', data=df_lowm).fit().summary().tables[1])\n",
    "print(\"average treatment effect for low spending users:\")\n",
    "print(smf.ols('click_bool ~ random_bool', data=df_low).fit().summary().tables[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([219, 100, 158,  31, 216,  15,  98,  55,  30, 137, 215, 117,  56,\n",
       "       129,  99,  53, 181, 221,  35,  59, 202, 132, 127, 109, 195, 131,\n",
       "        70, 220,  13,  92,  91, 113, 205,  14,  41, 154, 164,   9, 138,\n",
       "       194, 103,  81,  50,   4, 102, 106,  60,  73,  26,  16, 225,  39,\n",
       "        23, 230, 178,  32, 212, 125, 157, 145, 203, 185,  88, 151,  71,\n",
       "        10,  80,  68,  33,  77, 134, 152,  61, 153, 159,  18, 182,  63,\n",
       "        48, 119, 206, 156, 173, 223,  97, 123,   2, 211, 184,  86, 213,\n",
       "        40,  83, 187,  37, 189, 197,  21,  20,  17,  45,  42,  74, 229,\n",
       "        47, 224, 112, 200, 149,  27, 122, 162,  93,  69,  34, 190,  96,\n",
       "       161,  11,  76,  90, 186,  38, 155, 141, 196, 163, 226, 124, 172,\n",
       "        51, 214, 177, 201,  22, 167, 169, 171,  65, 179, 143, 146,  72,\n",
       "       136, 128,  82, 120, 111,  19, 160,  25, 217, 192,  57, 208, 222,\n",
       "        44, 165, 180,  89, 204, 140,   1,  87,  62, 207,  46,   7,  79,\n",
       "       108, 135, 126])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['prop_country_id'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}