research/web_routineness/release/05_Predictability_and_Browsing_Behavior.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy.stats as st"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div class=\"alert alert-warning\">\n",
    "<b>Warning:</b> Update the data directory path before running.\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_directory = \"../../../../../../data/web_routineness_release_clean/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_users = pd.read_csv(data_directory + '/pre_processed/selected_users.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed = pd.read_csv(data_directory + \"/pre_processed/browsing_with_gap.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "USERS = pd.read_csv(data_directory + '/raw/users.csv').rename(columns={'panelist_id': 'user'})\n",
    "USERS = selected_users.merge(USERS, on='user')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We focus on stationary predictability:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "STAT_RES = pd.read_csv(data_directory + '/computation_output/stationary_trajectory/predictability.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Web browsing behavior"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### User activity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_activity = processed.groupby('panelist_id', as_index=False).agg({'active_seconds': 'sum', 'domain': 'count'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_activity = user_activity.merge(\n",
    "    STAT_RES[['user', 'domain_pi_max']], left_on='panelist_id', right_on='user')\n",
    "user_activity = selected_users.merge(user_activity.drop(columns=['panelist_id']), on='user')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "active_seconds    0.400080\n",
       "domain            0.007387\n",
       "Name: domain_pi_max, dtype: float64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_activity.corr(\"pearson\")[\"domain_pi_max\"][[\"active_seconds\", \"domain\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### User interests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed['category'] = processed.category_names.apply(lambda x: x.split(\",\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_interest = processed.groupby('panelist_id', as_index=False) \\\n",
    "                        .agg({'domain': list, 'category': list}) \\\n",
    "                        .assign(\n",
    "                            category=lambda df: df['category'].apply(np.concatenate),\n",
    "                            domain_count=lambda df: df['domain'].apply(len),\n",
    "                            un_domain=lambda df: df['domain'].apply(set),\n",
    "                            un_category=lambda df: df['category'].apply(set),\n",
    "                            un_domain_len=lambda df: df['un_domain'].apply(len),\n",
    "                            un_category_len=lambda df: df['un_category'].apply(len)\n",
    "                        ) \\\n",
    "                        .merge(STAT_RES[['user', 'domain_pi_max']], left_on='panelist_id', right_on='user')\n",
    "user_interest = selected_users.merge(user_interest.drop(columns=['panelist_id']), on='user')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "un_domain_len     -0.228612\n",
       "un_category_len   -0.161655\n",
       "Name: domain_pi_max, dtype: float64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_interest.corr(\"pearson\")[\"domain_pi_max\"][['un_domain_len', 'un_category_len']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### User stationarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_stationarity = processed.groupby('panelist_id', as_index=False) \\\n",
    "                            .agg({'domain': 'count', 'active_seconds': ['mean', 'median']})\n",
    "user_stationarity.columns = [ '_'.join(c) if c[1] != '' else c[0] for c in user_stationarity.columns]\n",
    "user_stationarity = user_stationarity.merge(STAT_RES[['user', 'domain_pi_max']], left_on='panelist_id', right_on='user')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_stationarity = selected_users.merge(user_stationarity.drop(columns=['panelist_id']), on='user')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "active_seconds_mean      0.650178\n",
       "active_seconds_median    0.332005\n",
       "Name: domain_pi_max, dtype: float64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_stationarity.corr(\"pearson\")[\"domain_pi_max\"][['active_seconds_mean', 'active_seconds_median']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gender- and age-based differences in browsing behavior"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_conf_interval(arr, percentage=.9):\n",
    "    return st.t.interval(percentage, len(arr)-1, loc=np.mean(arr), scale=st.sem(arr))\n",
    "\n",
    "\n",
    "def get_mean_with_intervals(df, var, intervals):\n",
    "    return (\n",
    "        np.mean(df[var]),\n",
    "        *[ get_conf_interval(df[var], i) for i in intervals ]\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gender"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of distinct domains visited"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "female \t140.22, 95% CI [132.16, 148.29]\n",
      "male \t161.47, 95% CI [151.81, 171.13]\n"
     ]
    }
   ],
   "source": [
    "for k, g in user_interest.merge(USERS, on='user').groupby('gender', as_index=False): \n",
    "    percentage = .95\n",
    "    m, (lower, upper) = get_mean_with_intervals(g, 'un_domain_len', [percentage])\n",
    "    print (k, \"\\t%1.2f, %d%% CI [%1.2f, %1.2f]\" % (m, percentage*100, lower, upper))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Mean seconds spent per domain visit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "female \t37.339, 95% CI [35.54, 39.14]\n",
      "male \t33.147, 95% CI [31.47, 34.83]\n"
     ]
    }
   ],
   "source": [
    "for k, g in user_stationarity.merge(USERS, on='user').groupby('gender', as_index=False): \n",
    "    percentage = .95\n",
    "    m, (lower, upper) = get_mean_with_intervals(g, 'active_seconds_mean', [percentage])\n",
    "    print (k, \"\\t%1.3f, %d%% CI [%1.2f, %1.2f]\" % (m, percentage*100, lower, upper))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Age"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of distinct domains visited"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(24,34] 143.15, 95% CI [129.55, 156.75]\n",
      "(34,44] 155.58, 95% CI [138.69, 172.47]\n",
      "(44,54] 157.41, 95% CI [144.95, 169.88]\n",
      "(54,64] 164.48, 95% CI [149.50, 179.46]\n",
      "(64,80] 142.94, 95% CI [113.99, 171.90]\n",
      "[18,24] 123.77, 95% CI [109.97, 137.58]\n"
     ]
    }
   ],
   "source": [
    "for k, g in user_interest.merge(USERS, on='user').groupby('age_recode', as_index=False): \n",
    "    percentage = .95\n",
    "    m, (lower, upper) = get_mean_with_intervals(g, 'un_domain_len', [percentage])\n",
    "    print (k, \"%1.2f, %d%% CI [%1.2f, %1.2f]\" % (m, percentage*100, lower, upper))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Mean seconds spent per domain visit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(24,34] 29.357, 95% CI [27.74, 30.97]\n",
      "(34,44] 29.316, 95% CI [27.21, 31.42]\n",
      "(44,54] 36.594, 95% CI [34.13, 39.06]\n",
      "(54,64] 41.608, 95% CI [38.09, 45.13]\n",
      "(64,80] 40.167, 95% CI [35.05, 45.29]\n",
      "[18,24] 34.441, 95% CI [31.17, 37.72]\n"
     ]
    }
   ],
   "source": [
    "for k, g in user_stationarity.merge(USERS, on='user').groupby('age_recode', as_index=False): \n",
    "    percentage = .95\n",
    "    m, (lower, upper) = get_mean_with_intervals(g, 'active_seconds_mean', [percentage])\n",
    "    print (k, \"%1.3f, %d%% CI [%1.2f, %1.2f]\" % (m, percentage*100, lower, upper))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}