research/web_routineness/release/00_2_Filtering_Users.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from web_tracking.pre_processing.raw_data_processor import RawDataProcessor\n",
    "from web_tracking.pre_processing.trajectories.binned_non_stationary_trajectory import BinnedNonStationaryTrajectory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load raw data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div class=\"alert alert-warning\">\n",
    "<b>Warning:</b> Update the data directory path before running. \n",
    "</div>\n",
    "\n",
    "We tested this code using the data available at https://zenodo.org/record/4383164"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_directory = \"../../../../../../data/web_routineness_release_clean/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = pd.read_csv(\n",
    "    data_directory + \"/pre_processed/browsing_with_gap.csv\", \n",
    "    dtype={'id': int, 'panelist_id': int, 'prev_id': int}, parse_dates=['used_at', 'left_at'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>prev_id</th>\n",
       "      <th>panelist_id</th>\n",
       "      <th>used_at</th>\n",
       "      <th>left_at</th>\n",
       "      <th>active_seconds</th>\n",
       "      <th>gap_seconds</th>\n",
       "      <th>top_level_domain</th>\n",
       "      <th>category_names_top</th>\n",
       "      <th>sub_level_domain</th>\n",
       "      <th>subdomain</th>\n",
       "      <th>category_names_sub</th>\n",
       "      <th>domain</th>\n",
       "      <th>category_names</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1009076504</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2018-10-01 00:03:29</td>\n",
       "      <td>2018-10-01 00:03:31</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>youtube.com</td>\n",
       "      <td>entertainment,streaming-media</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>youtube.com</td>\n",
       "      <td>entertainment,streaming-media</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1009076508</td>\n",
       "      <td>1009076504</td>\n",
       "      <td>0</td>\n",
       "      <td>2018-10-01 00:03:31</td>\n",
       "      <td>2018-10-01 00:03:37</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>youtube.com</td>\n",
       "      <td>entertainment,streaming-media</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>youtube.com</td>\n",
       "      <td>entertainment,streaming-media</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1009076512</td>\n",
       "      <td>1009076508</td>\n",
       "      <td>0</td>\n",
       "      <td>2018-10-01 00:03:37</td>\n",
       "      <td>2018-10-01 00:03:43</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>youtube.com</td>\n",
       "      <td>entertainment,streaming-media</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>youtube.com</td>\n",
       "      <td>entertainment,streaming-media</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1009076516</td>\n",
       "      <td>1009076512</td>\n",
       "      <td>0</td>\n",
       "      <td>2018-10-01 00:03:43</td>\n",
       "      <td>2018-10-01 00:03:49</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>youtube.com</td>\n",
       "      <td>entertainment,streaming-media</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>youtube.com</td>\n",
       "      <td>entertainment,streaming-media</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1009076520</td>\n",
       "      <td>1009076516</td>\n",
       "      <td>0</td>\n",
       "      <td>2018-10-01 00:03:49</td>\n",
       "      <td>2018-10-01 00:03:53</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>youtube.com</td>\n",
       "      <td>entertainment,streaming-media</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>youtube.com</td>\n",
       "      <td>entertainment,streaming-media</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id     prev_id  panelist_id             used_at  \\\n",
       "0  1009076504           0            0 2018-10-01 00:03:29   \n",
       "1  1009076508  1009076504            0 2018-10-01 00:03:31   \n",
       "2  1009076512  1009076508            0 2018-10-01 00:03:37   \n",
       "3  1009076516  1009076512            0 2018-10-01 00:03:43   \n",
       "4  1009076520  1009076516            0 2018-10-01 00:03:49   \n",
       "\n",
       "              left_at  active_seconds  gap_seconds top_level_domain  \\\n",
       "0 2018-10-01 00:03:31               2            0      youtube.com   \n",
       "1 2018-10-01 00:03:37               6            0      youtube.com   \n",
       "2 2018-10-01 00:03:43               6            0      youtube.com   \n",
       "3 2018-10-01 00:03:49               6            0      youtube.com   \n",
       "4 2018-10-01 00:03:53               4            0      youtube.com   \n",
       "\n",
       "              category_names_top sub_level_domain subdomain  \\\n",
       "0  entertainment,streaming-media              NaN       NaN   \n",
       "1  entertainment,streaming-media              NaN       NaN   \n",
       "2  entertainment,streaming-media              NaN       NaN   \n",
       "3  entertainment,streaming-media              NaN       NaN   \n",
       "4  entertainment,streaming-media              NaN       NaN   \n",
       "\n",
       "  category_names_sub       domain                 category_names  \n",
       "0                NaN  youtube.com  entertainment,streaming-media  \n",
       "1                NaN  youtube.com  entertainment,streaming-media  \n",
       "2                NaN  youtube.com  entertainment,streaming-media  \n",
       "3                NaN  youtube.com  entertainment,streaming-media  \n",
       "4                NaN  youtube.com  entertainment,streaming-media  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "urls.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Filtering users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "RDP = RawDataProcessor(urls)\n",
    "RDP.create_time_series(uid='id',\n",
    "                       user='panelist_id',\n",
    "                       url='url',\n",
    "                       domain='domain',\n",
    "                       starts='used_at',\n",
    "                       active_seconds='active_seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "BNST = BinnedNonStationaryTrajectory(RDP.df)\n",
    "BNST_domain, = BNST.create(features=['domain'], bin_size=60).values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_by_user = BNST_domain.groupby('user', as_index=False).agg({'domain': 'count'})\n",
    "selected_users = group_by_user.query('domain >= 100')[['user']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1455"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(selected_users)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_users.to_csv(data_directory + '/pre_processed/selected_users.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}