01 - intervention_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "#from deep_translator import GoogleTranslator\n",
    "import csv\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"historic_data.csv\", delimiter=\",\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Unique Users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_users_unique = df.drop_duplicates(subset=['participantUuid'])\n",
    "print(\"Number of unique users: \", len(df_users_unique))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_users_unique.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Purpose"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_purpose = df[df['purpose'].notnull()]\n",
    "df_purpose.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_purpose_unique = df_purpose.drop_duplicates(subset=['purpose']).reset_index(drop=True)\n",
    "df_purpose_unique['purpose']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df_purpose)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pattern = re.compile(r'Procra*')\n",
    "len(df_purpose[df_purpose['purpose'].str.contains(pattern, regex=True) ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## This loops through the unique Purpose entries and translates them all to english\n",
    "# note that a bunch are already in english, but some are in german\n",
    "# file_name = 'purpose_translations.csv'\n",
    "\n",
    "# # temp = pd.DataFrame(0, index=np.arange(len(df_purpose_unique['purpose'])), columns=range(1))\n",
    "\n",
    "# with open (file_name, 'w', newline='', encoding='utf-8-sig') as csv_file:\n",
    "#     csv_writer = csv.writer(csv_file)\n",
    "#     for index, row in df_purpose_unique.iloc[1086:-1].iterrows():\n",
    "#         try:\n",
    "#             temp[0].iloc[index] = GoogleTranslator(source='auto', target='en').translate(row['purpose'])\n",
    "#             new_row = [row['purpose'], temp[0].iloc[index]]\n",
    "#         except:\n",
    "#             new_row = [row['purpose'], row['purpose']]\n",
    "        \n",
    "#         csv_writer.writerow(new_row)\n",
    "\n",
    "# temp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Apps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.groupby('app')['app'].count().sort_values(ascending=False).iloc[-25:-1]\n",
    "# # df = df.sort_values(['participantUuid', 'timestamp'], ascending=[True, True])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a summary of the 'app' column\n",
    "summary_apps = df['app'].value_counts().reset_index()\n",
    "summary_apps.columns = ['app', 'count']\n",
    "\n",
    "# Calculate percentage and add a new column\n",
    "summary_apps['percentage'] = (summary_apps['count'] / summary_apps['count'].sum()) * 100\n",
    "summary_apps['Cumulative'] = summary_apps['percentage'].cumsum()\n",
    "# summary_apps.tail(20)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Generate the pie chart\n",
    "# plt.figure(figsize=(8, 6))\n",
    "# plt.pie(summary_apps['count'], labels=summary_apps['app'], autopct='%1.1f%%', startangle=140)\n",
    "# plt.title('App Distribution')\n",
    "# plt.axis('equal')  # Equal aspect ratio ensures that the pie chart is circular.\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "summary_apps.to_csv('app_usage.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a dictionary for mapping\n",
    "mapping_dict = {\n",
    "    'instagram': 'Social Media',\n",
    "    'twitter': 'Social Media',\n",
    "    'youtube': 'Social Media',\n",
    "    'tikTok': 'Social Media',\n",
    "    'facebook': 'Social Media',\n",
    "    'reddit': 'Social Media',\n",
    "    'snapchat': 'Social Media',\n",
    "    'apollo': 'Social Media',\n",
    "    'whastapp': 'Communication',\n",
    "    'whastapp': 'Communication',\n",
    "    'linkedin': 'Social Media',\n",
    "    'telegram': 'Communication',\n",
    "    'safari': 'Browser',\n",
    "    'mail': 'Communication',\n",
    "    'discord': 'Communication',\n",
    "    'imessage': 'Communication',\n",
    "    'gmail': 'Communication',\n",
    "    'grinder': 'Dating',\n",
    "    'chrome': 'Browser',\n",
    "    'wechat': 'Communication',\n",
    "    'facebookMessenger': 'Communication',\n",
    "    'tinder': 'Dating',\n",
    "    'amazon': 'Shopping',\n",
    "    'bumble': 'Dating',\n",
    "    'hinge': 'Dating',\n",
    "    'jodel': 'Social Media',\n",
    "    'appStore': 'Utility',\n",
    "    'sparkMail': 'Communication',\n",
    "    'newYorkTimes': 'News & Entertainment',\n",
    "    'hackerNews': 'News & Entertainment',\n",
    "    'netflix': 'News & Entertainment',\n",
    "    'heyEmail': 'Communication',\n",
    "    'outlook': 'Communication',\n",
    "    'pornhub': 'News & Entertainment',\n",
    "    'weibo': 'Social Media',\n",
    "    'twitch': 'Social Media',\n",
    "    'duckDuckGoBrowser': 'Dating',\n",
    "    'whatsAppBusiness': 'Dating',\n",
    "    'douban': 'Social Media',\n",
    "    'spiegel': 'News & Entertainment',\n",
    "    'tagesschau': 'News & Entertainment',\n",
    "    'appleNews': 'News & Entertainment',\n",
    "    'tweetBot': 'Social Media',\n",
    "    'pinterest': 'Social Media',\n",
    "    'clashRoyale': 'Gaming',\n",
    "    'brave': 'Browser',\n",
    "    'stocks': 'Finance',\n",
    "    'reeder': 'News & Entertainment',\n",
    "    'signal': 'Communication',\n",
    "    'vinted': 'Shopping',\n",
    "    'twitterrific': 'Social Media',\n",
    "    'slack': 'Communication',\n",
    "    'zeitOnline': 'News & Entertainment',\n",
    "    'narwhal': 'Social Media',\n",
    "    'bild': 'News & Entertainment',\n",
    "    'beReal': 'Social Media',\n",
    "    'tumblr': 'Social Media',\n",
    "    'chessCom': 'Gaming',\n",
    "    'binance': 'Finance',\n",
    "    'firefox': 'Browser',\n",
    "    'kuCoin': 'Finance',\n",
    "    'google': 'Browser',\n",
    "    'toot': 'Social Media',\n",
    "    'webtoon': 'News & Entertainment',\n",
    "    'mastodon': 'Social Media',\n",
    "    'scruff': 'Dating',\n",
    "    'tradeRepublic': 'Finance'\n",
    "    \n",
    "}\n",
    "\n",
    "\n",
    "# Map the values in the 'platform' column using the dictionary\n",
    "df['app_mapped'] = df['app'].map(mapping_dict).fillna('Other')\n",
    "\n",
    "# Create a summary of the 'app' column\n",
    "summary_app_categories = df['app_mapped'].value_counts().reset_index()\n",
    "summary_app_categories.columns = ['app_mapped', 'count']\n",
    "\n",
    "# Calculate percentage and add a new column\n",
    "summary_app_categories['percentage'] = (summary_app_categories['count'] / summary_app_categories['count'].sum()) * 100\n",
    "summary_app_categories['Cumulative'] = summary_app_categories['percentage'].cumsum()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "summary_app_categories.to_csv('app_categories.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_list = df_users_unique['userIndex']\n",
    "df_users_unique['has_social_media'] = False\n",
    "\n",
    "for user in user_list:\n",
    "    df_users_unique['has_social_media'].loc[df_users_unique['userIndex'] == user] = (df[df['userIndex'] == 1]['app_mapped'] == 'Social Media').any()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "100* (len(df_users_unique[df_users_unique['has_social_media'] == True]) / len(df_users_unique['userIndex']))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Time Difference Conversions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.sort_values(['timestamp'], ascending=[True])\n",
    "df['realSecondsDifference'] = (pd.to_datetime(df['timestamp']) - pd.to_datetime(df['timestamp'].iloc[0])).dt.total_seconds()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_users_adjust = df.sort_values(['participantUuid', 'timestamp'], ascending=[True, True]).drop_duplicates(['participantUuid']).reset_index(drop=True)\n",
    "df_users_adjust['toSubtractDate'] = df_users_adjust['timestamp']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.merge(df_users_adjust[['participantUuid', 'toSubtractDate']], on='participantUuid')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate the time differences as accurately as possible for each unit (seconds, minutes, hours, days, weeks). Convert to int so that everything is floored (e.g. everything from week 0.0 to 0.9 is week 0, etc.)\n",
    "\n",
    "df['realMinutesDifference'] = (df['realSecondsDifference'] / 60).astype(int)\n",
    "df['realHoursDifference'] = (df['realSecondsDifference'] / 3600).astype(int)\n",
    "df['realDaysDifference'] = (df['realSecondsDifference'] / 86400).astype(int)\n",
    "df['realWeeksDifference'] = (df['realSecondsDifference'] / 608400).astype(int)\n",
    "\n",
    "df['dateSinceStart'] = pd.to_datetime(df['timestamp']) - pd.to_datetime(df['toSubtractDate'])\n",
    "df['secondsSinceStart'] = ((pd.to_datetime(df['timestamp']) - pd.to_datetime(df['toSubtractDate'])).dt.total_seconds())\n",
    "df['minutesSinceStart'] = (df['secondsSinceStart'] / 60).astype(int)\n",
    "df['hoursSinceStart'] = (df['secondsSinceStart'] / 3600).astype(int)\n",
    "df['daysSinceStart2'] = (df['secondsSinceStart'] / 86400).astype(int)\n",
    "df['daysSinceStart'] = ((pd.to_datetime(df['timestamp']) - pd.to_datetime(df['toSubtractDate'])).dt.days).astype(int)\n",
    "df['weeksSinceStart'] = (df['daysSinceStart'] / 7).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['day_of_week'] = pd.to_datetime(df['timestamp']).dt.strftime('%A')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Resolution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['resolution'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['continuedToApp'] = np.where(df['resolution'] == 'openedApp', 'openedApp', np.where(df['resolution'] == 'closedApp', 'notAnIntervention','didNotOpenApp'))\n",
    "df.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Distribution of time differences - calculate what is a break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.sort_values(['participantUuid', 'timestamp'], ascending=[True, True])\n",
    "df['minuteDiffReverse'] = df['minutesSinceStart'].diff(periods=1).fillna(0) # this makes the first one NA, subtracts each row from the row before\n",
    "df['minuteDiffForward'] = df['minutesSinceStart'].diff(periods=-1).fillna(0) # this makes the last one NA, subtracts the next row from each row\n",
    "df['minuteDiffForward'] = abs(df['minuteDiffForward'])\n",
    "df['hourDiffReverse'] = (df['minuteDiffReverse']/60).astype(int)\n",
    "df['hourDiffForward'] = (df['minuteDiffForward']/60).astype(int)\n",
    "df['dayDiffReverse'] = (df['hourDiffReverse']/24).astype(int)\n",
    "df['dayDiffForward'] = (df['hourDiffForward']/24).astype(int)\n",
    "df['weekDiffReverse'] = (df['dayDiffReverse']/7).astype(int)\n",
    "df['weekDiffForward'] = (df['dayDiffForward']/7).astype(int)\n",
    "df['userDiffNext'] = df['participantUuid'] != df['participantUuid'].shift(-1)\n",
    "df['userDiffPrev'] = df['participantUuid'] != df['participantUuid'].shift(1)\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_diff_distribution = df[df['userDiffPrev'] != True]\n",
    "df_diff_distribution = df_diff_distribution[df_diff_distribution['resolution'] != 'closedApp']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Min: \", df_diff_distribution['hourDiffReverse'].min())\n",
    "print(\"Max: \", df_diff_distribution['hourDiffReverse'].max())\n",
    "print(\"Mean: \", df_diff_distribution['hourDiffReverse'].mean())\n",
    "print(\"Median: \", df_diff_distribution['hourDiffReverse'].median())\n",
    "print(\"Std Dev: \", df_diff_distribution['hourDiffReverse'].std())\n",
    "print(\"Two Std Dev above the mean (hours): \", df_diff_distribution['hourDiffReverse'].mean() + 2*df_diff_distribution['hourDiffReverse'].std())\n",
    "print(\"Two Std Dev above the mean (days): \", (df_diff_distribution['hourDiffReverse'].mean() + 2*df_diff_distribution['hourDiffReverse'].std())/24)\n",
    "print(\"Two Std Dev above the mean (weeks): \", (df_diff_distribution['hourDiffReverse'].mean() + 2*df_diff_distribution['hourDiffReverse'].std())/(7*24))\n",
    "\n",
    "mean_timeDiff_hours = df_diff_distribution['hourDiffReverse'].mean()\n",
    "std_timeDiff_hours = df_diff_distribution['hourDiffReverse'].std()\n",
    "break_cutoff_timeDiff_hours = mean_timeDiff_hours + 2*std_timeDiff_hours\n",
    "\n",
    "break_cutoff_timeDiff_hours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Min: \", df_diff_distribution['minuteDiffReverse'].min())\n",
    "print(\"Max: \", df_diff_distribution['minuteDiffReverse'].max())\n",
    "print(\"Mean: \", df_diff_distribution['minuteDiffReverse'].mean())\n",
    "print(\"Median: \", df_diff_distribution['minuteDiffReverse'].median())\n",
    "print(\"Std Dev: \", df_diff_distribution['minuteDiffReverse'].std())\n",
    "print(\"Two Std Dev above the mean (minutes): \", df_diff_distribution['minuteDiffReverse'].mean() + 2*df_diff_distribution['minuteDiffReverse'].std())\n",
    "print(\"Two Std Dev above the mean (hours): \", (df_diff_distribution['minuteDiffReverse'].mean() + 2*df_diff_distribution['minuteDiffReverse'].std())/60)\n",
    "print(\"Two Std Dev above the mean (days): \", (df_diff_distribution['minuteDiffReverse'].mean() + 2*df_diff_distribution['minuteDiffReverse'].std())/(60*24))\n",
    "\n",
    "mean_timeDiff_minutes = df_diff_distribution['minuteDiffReverse'].mean()\n",
    "std_timeDiff_minutes = df_diff_distribution['minuteDiffReverse'].std()\n",
    "break_cutoff_timeDiff_minutes = mean_timeDiff_minutes + 2*std_timeDiff_minutes\n",
    "\n",
    "break_cutoff_timeDiff_minutes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Labeling Breaks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['lifeStatus'] = np.where(((~df['userDiffNext']) & (~df['userDiffPrev']) & (abs(df['minuteDiffForward']) > break_cutoff_timeDiff_minutes) & (abs(df['minuteDiffReverse']) > break_cutoff_timeDiff_minutes)), 'Return/Break',\n",
    "                                np.where(((~df['userDiffNext']) & (~df['userDiffPrev']) & (abs(df['minuteDiffForward']) > break_cutoff_timeDiff_minutes)), 'Break','Alive'))\n",
    "\n",
    "df.loc[df['lifeStatus'].shift() == 'Break', 'lifeStatus'] = 'Return'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_noclose = df[df['resolution'] != 'closedApp'].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_noclose['lifeStatus'] = np.where(((~df_noclose['userDiffNext']) & (~df_noclose['userDiffPrev']) & (abs(df_noclose['minuteDiffForward']) > break_cutoff_timeDiff_minutes) & (abs(df_noclose['minuteDiffReverse']) > break_cutoff_timeDiff_minutes)), 'Return/Break',\n",
    "                                        np.where(((~df_noclose['userDiffNext']) & (~df_noclose['userDiffPrev']) & (abs(df_noclose['minuteDiffForward']) > break_cutoff_timeDiff_minutes)), 'Break', 'Alive'))\n",
    "\n",
    "df_noclose.loc[df_noclose['lifeStatus'].shift() == 'Break', 'lifeStatus'] = 'Return'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Alive entries: \", len(df_noclose[df_noclose['lifeStatus'] == 'Alive']))\n",
    "print(\"Break entries: \", len(df_noclose[df_noclose['lifeStatus'] == 'Break']))\n",
    "print(\"Return/Break entries: \", len(df_noclose[df_noclose['lifeStatus'] == 'Return/Break']))\n",
    "print(\"Return entries: \", len(df_noclose[df_noclose['lifeStatus'] == 'Return']))\n",
    "print(\"Dropout entries: \", len(df_noclose[df_noclose['lifeStatus'] == 'Dropout']))\n",
    "print(\"Participants with at least one break: \", len(df_noclose[df_noclose['lifeStatus'] == 'Break']['participantUuid'].unique()))\n",
    "print(\"Unique participants: \", len(df_noclose['participantUuid'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(df[df['lifeStatus'] == 'Alive']))\n",
    "print(len(df[df['lifeStatus'] == 'Break']))\n",
    "print(len(df[df['lifeStatus'] == 'Return/Break']))\n",
    "print(len(df[df['lifeStatus'] == 'Return']))\n",
    "print(len(df[df['lifeStatus'] == 'Dropout']))\n",
    "print(\"Unique participants: \", len(df['participantUuid'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('data_processed.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_noclose.to_csv('data_processed_noclose.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_hasclose = df[df['resolution'] == 'closedApp'].copy()\n",
    "users_without_close_enabled = df['participantUuid'].unique().size - df_hasclose['participantUuid'].unique().size\n",
    "users_without_close_enabled"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Time in study, users in study at x time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## First, without considering the result of the action\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_max_hours = df_noclose.groupby('userIndex')['hoursSinceStart'].max().reset_index(name='max_hours')\n",
    "\n",
    "# Create a list of hours from 0 to the maximum of max_hours\n",
    "max_hour = user_max_hours['max_hours'].max()\n",
    "hours = list(range(max_hour + 1))\n",
    "\n",
    "# Initialize a list to store the count of users for each hour\n",
    "hourly_counts = []\n",
    "\n",
    "# Iterate through the hours and count the number of users for each hour\n",
    "for hour in hours:\n",
    "    count = (user_max_hours['max_hours'] >= hour).sum()\n",
    "    hourly_counts.append(count)\n",
    "\n",
    "# Create the final dataframe\n",
    "df_users_remaining_hours = pd.DataFrame({'hoursSinceStart': hours, 'user_count': hourly_counts})\n",
    "\n",
    "# Print the result dataframe\n",
    "print(df_users_remaining_hours)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_max_days = df_noclose.groupby('userIndex')['daysSinceStart'].max().reset_index(name='max_days')\n",
    "\n",
    "# Create a list of hours from 0 to the maximum of max_hours\n",
    "max_day = user_max_days['max_days'].max()\n",
    "days = list(range(max_day + 1))\n",
    "\n",
    "# Initialize a list to store the count of users for each hour\n",
    "daily_counts = []\n",
    "\n",
    "# Iterate through the hours and count the number of users for each hour\n",
    "for day in days:\n",
    "    count = (user_max_days['max_days'] >= day).sum()\n",
    "    daily_counts.append(count)\n",
    "\n",
    "# Create the final dataframe\n",
    "df_users_remaining_days = pd.DataFrame({'daysSinceStart': days, 'user_count': daily_counts})\n",
    "\n",
    "# Print the result dataframe\n",
    "print(df_users_remaining_days)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_max_weeks = df_noclose.groupby('userIndex')['weeksSinceStart'].max().reset_index(name='max_weeks')\n",
    "\n",
    "# Create a list of hours from 0 to the maximum of max_hours\n",
    "max_week = user_max_weeks['max_weeks'].max()\n",
    "weeks = list(range(max_week + 1))\n",
    "\n",
    "# Initialize a list to store the count of users for each hour\n",
    "weekly_counts = []\n",
    "\n",
    "# Iterate through the hours and count the number of users for each hour\n",
    "for week in weeks:\n",
    "    count = (user_max_weeks['max_weeks'] >= week).sum()\n",
    "    weekly_counts.append(count)\n",
    "\n",
    "# Create the final dataframe\n",
    "df_users_remaining_weeks = pd.DataFrame({'weeksSinceStart': weeks, 'user_count': weekly_counts})\n",
    "\n",
    "# Print the result dataframe\n",
    "print(df_users_remaining_weeks)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_entries_per_hour = df_noclose.groupby(['hoursSinceStart'])['userIndex'].count().reset_index(name='total_interactions')\n",
    "df_entries_per_hour['std_interactions'] = df_noclose.groupby(['hoursSinceStart','userIndex'])['userIndex'].count().reset_index(name='total_interactions').groupby('hoursSinceStart')['total_interactions'].agg(['std']).reset_index()['std']\n",
    "df_entries_per_user_hours = df_users_remaining_hours.merge(df_entries_per_hour, on='hoursSinceStart', how='left')\n",
    "df_entries_per_user_hours['total_interactions'].fillna(method='ffill', inplace=True)\n",
    "df_entries_per_user_hours['std_interactions'].fillna(0, inplace=True)\n",
    "df_entries_per_user_hours['interactions_per_user'] = df_entries_per_user_hours['total_interactions'] / df_entries_per_user_hours['user_count']\n",
    "df_entries_per_user_hours['std_error_interactions'] = df_entries_per_user_hours['std_interactions'] / np.sqrt(df_entries_per_user_hours['user_count'])\n",
    "\n",
    "\n",
    "df_entries_per_day = df_noclose.groupby(['daysSinceStart'])['userIndex'].count().reset_index(name='total_interactions')\n",
    "df_entries_per_day['std_interactions'] = df_noclose.groupby(['daysSinceStart','userIndex'])['userIndex'].count().reset_index(name='total_interactions').groupby('daysSinceStart')['total_interactions'].agg(['std']).reset_index()['std']\n",
    "df_entries_per_user_days = df_users_remaining_days.merge(df_entries_per_day, on='daysSinceStart', how='left')\n",
    "df_entries_per_user_days['total_interactions'].fillna(method='ffill', inplace=True)\n",
    "df_entries_per_user_days['std_interactions'].fillna(0, inplace=True)\n",
    "df_entries_per_user_days['interactions_per_user'] = df_entries_per_user_days['total_interactions'] / df_entries_per_user_days['user_count']\n",
    "df_entries_per_user_days['std_error_interactions'] = df_entries_per_user_days['std_interactions'] / np.sqrt(df_entries_per_user_days['user_count'])\n",
    "\n",
    "\n",
    "df_entries_per_week = df_noclose.groupby(['weeksSinceStart'])['userIndex'].count().reset_index(name='total_interactions')\n",
    "df_entries_per_week['std_interactions'] = df_noclose.groupby(['weeksSinceStart','userIndex'])['userIndex'].count().reset_index(name='total_interactions').groupby('weeksSinceStart')['total_interactions'].agg(['std']).reset_index()['std']\n",
    "df_entries_per_user_weeks = df_users_remaining_weeks.merge(df_entries_per_week, on='weeksSinceStart', how='left')\n",
    "df_entries_per_user_weeks['total_interactions'].fillna(method='ffill', inplace=True)\n",
    "df_entries_per_user_weeks['std_interactions'].fillna(0, inplace=True)\n",
    "df_entries_per_user_weeks['interactions_per_user'] = df_entries_per_user_weeks['total_interactions'] / df_entries_per_user_weeks['user_count']\n",
    "df_entries_per_user_weeks['std_error_interactions'] = df_entries_per_user_weeks['std_interactions'] / np.sqrt(df_entries_per_user_weeks['user_count'])\n",
    "\n",
    "\n",
    "print(df_entries_per_user_hours.head())\n",
    "print(df_entries_per_user_days.head())\n",
    "print(df_entries_per_user_weeks.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_entries_per_user_hours.to_csv('data_users_remaining_hours.csv', index=False)\n",
    "df_entries_per_user_days.to_csv('data_users_remaining_days.csv', index=False)\n",
    "df_entries_per_user_weeks.to_csv('data_users_remaining_weeks.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Second, considering the result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_max_hours_result = df_noclose.groupby(['userIndex', 'continuedToApp'])['hoursSinceStart'].max().reset_index(name='max_hours')\n",
    "\n",
    "# Create a list of hours from 0 to the maximum of max_hours\n",
    "max_hour = user_max_hours_result['max_hours'].max()\n",
    "hours = [i for i in range(max_hour) for _ in range(2)]\n",
    "\n",
    "# Initialize a list to store the count of users for each hour\n",
    "hourly_counts = []\n",
    "continued_status = []\n",
    "flag = True\n",
    "\n",
    "\n",
    "# Iterate through the hours and count the number of users for each hour\n",
    "for idx, hour in enumerate(hours):\n",
    "    count = (user_max_hours['max_hours'] >= hour).sum()\n",
    "    hourly_counts.append(count)\n",
    "    if flag:\n",
    "        continued_status.append('didNotOpenApp')\n",
    "    else:\n",
    "        continued_status.append('openedApp')\n",
    "    flag = not flag\n",
    "\n",
    "# Create the final dataframe\n",
    "df_users_remaining_hours_result = pd.DataFrame({'hoursSinceStart': hours, 'user_count': hourly_counts, 'continuedToApp': continued_status})\n",
    "\n",
    "# Print the result dataframe\n",
    "print(df_users_remaining_hours_result)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_max_days_result = df_noclose.groupby(['userIndex', 'continuedToApp'])['daysSinceStart'].max().reset_index(name='max_days')\n",
    "\n",
    "# Create a list of hours from 0 to the maximum of max_hours\n",
    "max_day = user_max_days_result['max_days'].max()\n",
    "days = [i for i in range(max_day) for _ in range(2)]\n",
    "\n",
    "# Initialize a list to store the count of users for each hour\n",
    "daily_counts = []\n",
    "continued_status = []\n",
    "flag = True\n",
    "\n",
    "# Iterate through the hours and count the number of users for each hour\n",
    "for idx, day in enumerate(days):\n",
    "    count = (user_max_days['max_days'] >= day).sum()\n",
    "    daily_counts.append(count)\n",
    "    # continued_status.append(user_max_days_result['continuedToApp'].iloc[idx])\n",
    "    if flag:\n",
    "        continued_status.append('didNotOpenApp')\n",
    "    else:\n",
    "        continued_status.append('openedApp')\n",
    "    flag = not flag\n",
    "\n",
    "# Create the final dataframe\n",
    "df_users_remaining_days_result = pd.DataFrame({'daysSinceStart': days, 'user_count': daily_counts, 'continuedToApp': continued_status})\n",
    "\n",
    "# Print the result dataframe\n",
    "print(df_users_remaining_days_result)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_max_weeks_result = df_noclose.groupby(['userIndex', 'continuedToApp'])['weeksSinceStart'].max().reset_index(name='max_weeks')\n",
    "\n",
    "# Create a list of hours from 0 to the maximum of max_hours\n",
    "max_week = user_max_weeks_result['max_weeks'].max()\n",
    "weeks = [i for i in range(max_week) for _ in range(2)]\n",
    "\n",
    "# Initialize a list to store the count of users for each hour\n",
    "weekly_counts = []\n",
    "continued_status = []\n",
    "flag = True\n",
    "\n",
    "# Iterate through the hours and count the number of users for each hour\n",
    "for idx, week in enumerate(weeks):\n",
    "    count = (user_max_weeks['max_weeks'] >= week).sum()\n",
    "    weekly_counts.append(count)\n",
    "    # continued_status.append(user_max_weeks_result['continuedToApp'].iloc[idx])\n",
    "    if flag:\n",
    "        continued_status.append('didNotOpenApp')\n",
    "    else:\n",
    "        continued_status.append('openedApp')\n",
    "    flag = not flag\n",
    "\n",
    "# Create the final dataframe\n",
    "df_users_remaining_weeks_result = pd.DataFrame({'weeksSinceStart': weeks, 'user_count': weekly_counts, 'continuedToApp': continued_status})\n",
    "\n",
    "# Print the result dataframe\n",
    "print(df_users_remaining_weeks_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# first, calculate the total number of interactions for each user at each timepoint for each value of continued to app\n",
    "df_entries_per_hour_result = df_noclose.groupby(['hoursSinceStart', 'continuedToApp'])['userIndex'].count().reset_index(name='total_interactions')\n",
    "# next, calculate the standard deviation of all intaractions at each timepoint and continued to app value\n",
    "df_entries_per_hour_result['std_interactions'] = df_noclose.groupby(['hoursSinceStart','userIndex', 'continuedToApp'])['userIndex'].count().reset_index(name='total_interactions').groupby(['hoursSinceStart', 'continuedToApp'])['total_interactions'].agg(['std']).reset_index()['std']\n",
    "# join the number of users remaining at each timepoint to the dataframe\n",
    "df_entries_per_user_hours_result = df_users_remaining_hours_result.merge(df_entries_per_hour_result, on=['hoursSinceStart', 'continuedToApp'], how='left')\n",
    "# remove duplicated columns (might not be necessary if you just join on all the common columns)\n",
    "df_entries_per_user_hours_result = df_entries_per_user_hours_result.loc[:, ~df_entries_per_user_hours_result.columns.duplicated()]\n",
    "df_entries_per_hour_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# first, calculate the total number of interactions for each user at each timepoint for each value of continued to app\n",
    "df_entries_per_hour_result = df_noclose.groupby(['hoursSinceStart', 'continuedToApp'])['userIndex'].count().reset_index(name='total_interactions')\n",
    "# next, calculate the standard deviation of all intaractions at each timepoint and continued to app value\n",
    "df_entries_per_hour_result['std_interactions'] = df_noclose.groupby(['hoursSinceStart','userIndex', 'continuedToApp'])['userIndex'].count().reset_index(name='total_interactions').groupby(['hoursSinceStart', 'continuedToApp'])['total_interactions'].agg(['std']).reset_index()['std']\n",
    "# join the number of users remaining at each timepoint to the dataframe\n",
    "df_entries_per_user_hours_result = df_users_remaining_hours_result.merge(df_entries_per_hour_result, on=['hoursSinceStart', 'continuedToApp'], how='left')\n",
    "# remove duplicated columns (might not be necessary if you just join on all the common columns)\n",
    "df_entries_per_user_hours_result = df_entries_per_user_hours_result.loc[:, ~df_entries_per_user_hours_result.columns.duplicated()]\n",
    "# fill 0s for NAs, it means there are hours with no interactions\n",
    "df_entries_per_user_hours_result['total_interactions'].fillna(0, inplace=True)\n",
    "# fill 0s for NAs, it means there are hours with no interactions or with only one user\n",
    "df_entries_per_user_hours_result['std_interactions'].fillna(0, inplace=True)\n",
    "# interactions per user at each time point = total interactions at that time point divided by number of users still in the study at that time point\n",
    "df_entries_per_user_hours_result['interactions_per_user'] = df_entries_per_user_hours_result['total_interactions'] / df_entries_per_user_hours_result['user_count']\n",
    "# standard error is the standard deviation divided by the square root of the N\n",
    "df_entries_per_user_hours_result['std_error_interactions'] = df_entries_per_user_hours_result['std_interactions'] / np.sqrt(df_entries_per_user_hours_result['user_count'])\n",
    "# calculate the squared error ratio for use later when propagating the errors to the percentage\n",
    "df_entries_per_user_hours_result['squared_error_ratio'] = np.power(df_entries_per_user_hours_result['std_error_interactions'] / df_entries_per_user_hours_result['interactions_per_user'], 2)\n",
    "# find the sum  total of interactions at each hour, independent of continued to app value\n",
    "grouped_hour = df_entries_per_user_hours_result.groupby('hoursSinceStart')['interactions_per_user'].transform(sum)\n",
    "# find the sum  total of interaction errors at each hour, independent of continued to app value\n",
    "grouped_error_hour = df_entries_per_user_hours_result.groupby('hoursSinceStart')['std_error_interactions'].transform(sum)\n",
    "#  percentage for each continued to app value is the number of interactions for that value divided by the total for that time point\n",
    "df_entries_per_user_hours_result['percentage'] = 100 * (df_entries_per_user_hours_result['interactions_per_user'] / grouped_hour)\n",
    "#  the percentage error is the percentage value divided by the square root of the sum of the squares of that particular error value ratio plus the sum of the error value ratios\n",
    "df_entries_per_user_hours_result['std_error_percentage'] = df_entries_per_user_hours_result['percentage'] * np.sqrt(df_entries_per_user_hours_result['squared_error_ratio'] + np.power(grouped_error_hour,2) / np.power(grouped_hour,2) )\n",
    "\n",
    "df_entries_per_day_result = df_noclose.groupby(['daysSinceStart', 'continuedToApp'])['userIndex'].count().reset_index(name='total_interactions')\n",
    "df_entries_per_day_result['std_interactions'] = df_noclose.groupby(['daysSinceStart','userIndex', 'continuedToApp'])['userIndex'].count().reset_index(name='total_interactions').groupby(['daysSinceStart', 'continuedToApp'])['total_interactions'].agg(['std']).reset_index()['std']\n",
    "df_entries_per_user_days_result = df_users_remaining_days_result.merge(df_entries_per_day_result, on=['daysSinceStart', 'continuedToApp'], how='left')\n",
    "df_entries_per_user_days_result = df_entries_per_user_days_result.loc[:, ~df_entries_per_user_days_result.columns.duplicated()]\n",
    "df_entries_per_user_days_result['total_interactions'].fillna(0, inplace=True)\n",
    "df_entries_per_user_days_result['std_interactions'].fillna(0, inplace=True)\n",
    "df_entries_per_user_days_result['interactions_per_user'] = df_entries_per_user_days_result['total_interactions'] / df_entries_per_user_days_result['user_count']\n",
    "df_entries_per_user_days_result['std_error_interactions'] = df_entries_per_user_days_result['std_interactions'] / np.sqrt(df_entries_per_user_days_result['user_count'])\n",
    "df_entries_per_user_days_result['squared_error_ratio'] = np.power(df_entries_per_user_days_result['std_error_interactions'] / df_entries_per_user_days_result['interactions_per_user'], 2)\n",
    "grouped_day = df_entries_per_user_days_result.groupby('daysSinceStart')['interactions_per_user'].transform(sum)\n",
    "grouped_error_day = df_entries_per_user_days_result.groupby('daysSinceStart')['std_error_interactions'].transform(sum)\n",
    "df_entries_per_user_days_result['percentage'] = 100 * (df_entries_per_user_days_result['interactions_per_user'] / grouped_day)\n",
    "df_entries_per_user_days_result['std_error_percentage'] = df_entries_per_user_days_result['percentage'] * np.sqrt(df_entries_per_user_days_result['squared_error_ratio'] + np.power(grouped_error_day,2) / np.power(grouped_day,2) )\n",
    "\n",
    "df_entries_per_week_result = df_noclose.groupby(['weeksSinceStart', 'continuedToApp'])['userIndex'].count().reset_index(name='total_interactions')\n",
    "df_entries_per_week_result['std_interactions'] = df_noclose.groupby(['weeksSinceStart','userIndex', 'continuedToApp'])['userIndex'].count().reset_index(name='total_interactions').groupby(['weeksSinceStart', 'continuedToApp'])['total_interactions'].agg(['std']).reset_index()['std']\n",
    "df_entries_per_user_weeks_result = df_users_remaining_weeks_result.merge(df_entries_per_week_result, on=['weeksSinceStart','continuedToApp'], how='left')\n",
    "df_entries_per_user_weeks_result = df_entries_per_user_weeks_result.loc[:, ~df_entries_per_user_weeks_result.columns.duplicated()]\n",
    "df_entries_per_user_weeks_result['total_interactions'].fillna(0, inplace=True)\n",
    "df_entries_per_user_weeks_result['std_interactions'].fillna(0, inplace=True)\n",
    "df_entries_per_user_weeks_result['interactions_per_user'] = df_entries_per_user_weeks_result['total_interactions'] / df_entries_per_user_weeks_result['user_count']\n",
    "df_entries_per_user_weeks_result['std_error_interactions'] = df_entries_per_user_weeks_result['std_interactions'] / np.sqrt(df_entries_per_user_weeks_result['user_count'])\n",
    "df_entries_per_user_weeks_result['squared_error_ratio'] = np.power(df_entries_per_user_weeks_result['std_error_interactions'] / df_entries_per_user_weeks_result['interactions_per_user'], 2)\n",
    "grouped_week = df_entries_per_user_weeks_result.groupby('weeksSinceStart')['interactions_per_user'].transform(sum)\n",
    "grouped_error_week = df_entries_per_user_weeks_result.groupby('weeksSinceStart')['std_error_interactions'].transform(sum)\n",
    "df_entries_per_user_weeks_result['percentage'] = 100 * (df_entries_per_user_weeks_result['interactions_per_user'] / grouped_week)\n",
    "df_entries_per_user_weeks_result['std_error_percentage'] = df_entries_per_user_weeks_result['percentage'] * np.sqrt(df_entries_per_user_weeks_result['squared_error_ratio'] + np.power(grouped_error_week,2) / np.power(grouped_week,2) )\n",
    "\n",
    "print(df_entries_per_user_hours_result.head())\n",
    "print(df_entries_per_user_days_result.head())\n",
    "print(df_entries_per_user_weeks_result.head())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_entries_per_user_hours_result.to_csv('data_users_remaining_hours_result.csv', index=False)\n",
    "df_entries_per_user_days_result.to_csv('data_users_remaining_days_result.csv', index=False)\n",
    "df_entries_per_user_weeks_result.to_csv('data_users_remaining_weeks_result.csv', index=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Just before Breaks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assuming your original DataFrame is named 'df'\n",
    "# Create a list of participantUuid who have at least one \"Break\"\n",
    "participants_with_breaks = df_noclose[df_noclose['lifeStatus'] == 'Break']['participantUuid'].unique()\n",
    "\n",
    "# Create a new DataFrame containing all entries for participants with breaks\n",
    "filtered_df = df_noclose[df_noclose['participantUuid'].isin(participants_with_breaks)].copy()\n",
    "\n",
    "# Sort the filtered DataFrame by 'participantUuid' and 'daysSinceStart'\n",
    "filtered_df = filtered_df.sort_values(by=['participantUuid', 'hoursSinceStart'])\n",
    "\n",
    "# Initialize a new column 'within_65_hours_before_break' with default value False\n",
    "# filtered_df['within_65_hours_before_break'] = False\n",
    "\n",
    "# Initialize a new column 'time_to_break' with default value NaN\n",
    "filtered_df['time_to_break'] = float('nan')\n",
    "\n",
    "# Iterate through each participant's data\n",
    "for participant in participants_with_breaks:\n",
    "    participant_data = filtered_df[filtered_df['participantUuid'] == participant]\n",
    "    break_indices = participant_data[participant_data['lifeStatus'] == 'Break'].index\n",
    "    prev_break_index = participant_data.index[0]\n",
    "\n",
    "    for index in break_indices:\n",
    "        prev_break_time = participant_data.loc[prev_break_index, 'hoursSinceStart']\n",
    "        current_break_time = participant_data.loc[index, 'hoursSinceStart']\n",
    "\n",
    "        within_range = (participant_data['hoursSinceStart'] >= max((current_break_time - break_cutoff_timeDiff_hours), prev_break_time)) & (participant_data['hoursSinceStart'] <= current_break_time)\n",
    "        # filtered_df.loc[(filtered_df['participantUuid'] == participant) & within_range,'within_65_hours_before_break'] = True\n",
    "\n",
    "        # within_range = (participant_data['hoursSinceStart'] >= prev_break_time) & (participant_data['hoursSinceStart'] <= current_break_time)\n",
    "        \n",
    "        filtered_df.loc[(filtered_df['participantUuid'] == participant) & within_range,'time_to_break'] = filtered_df.loc[(filtered_df['participantUuid'] == participant) & within_range,'hoursSinceStart'] - current_break_time\n",
    "\n",
    "        prev_break_index = index\n",
    "\n",
    "# Now, filtered_df contains the labeled entries within 65 hours before each \"Break\" row\n",
    "# and the 'time_to_break' column indicating time to the next break\n",
    "filtered_df = filtered_df[filtered_df['time_to_break'].notnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_df.to_csv('data_breaks.csv', index=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Just after Returns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Assuming your original DataFrame is named 'df'\n",
    "# Create a list of participantUuid who have at least one \"Return\"\n",
    "participants_with_returns = df_noclose[df_noclose['lifeStatus'] == 'Return']['participantUuid'].unique()\n",
    "\n",
    "# Create a new DataFrame containing all entries for participants with returns\n",
    "filtered_df = df_noclose[df_noclose['participantUuid'].isin(participants_with_returns)].copy()\n",
    "\n",
    "# Sort the filtered DataFrame by 'participantUuid' and 'daysSinceStart'\n",
    "filtered_df = filtered_df.sort_values(by=['participantUuid', 'hoursSinceStart'])\n",
    "\n",
    "# Initialize a new column 'within_7_days_before_return' with default value False\n",
    "# filtered_df['within_65_hours_after_return'] = False\n",
    "\n",
    "# Initialize a new column 'time_to_return' with default value NaN\n",
    "filtered_df['time_after_return'] = float('nan')\n",
    "\n",
    "# Iterate through each participant's data\n",
    "for participant in participants_with_returns:\n",
    "    participant_data = filtered_df[filtered_df['participantUuid'] == participant]\n",
    "    return_indices = participant_data[participant_data['lifeStatus'] == 'Return'].index\n",
    "    prev_return_index = participant_data.index[0]\n",
    "\n",
    "    for index in return_indices:\n",
    "        current_return_time = participant_data.loc[prev_return_index, 'hoursSinceStart']\n",
    "        next_return_time = participant_data.loc[index, 'hoursSinceStart']\n",
    "    \n",
    "        within_range = (participant_data['hoursSinceStart'] <= (current_return_time + break_cutoff_timeDiff_hours)) & (participant_data['hoursSinceStart'] >= current_return_time) & (participant_data['hoursSinceStart'] <= next_return_time)\n",
    "        # filtered_df.loc[(filtered_df['participantUuid'] == participant) & within_range,'within_65_hours_after_return'] = True\n",
    "        \n",
    "        # within_range = (participant_data['hoursSinceStart'] >= current_return_time) & (participant_data['hoursSinceStart'] <= next_return_time)\n",
    "        \n",
    "        filtered_df.loc[(filtered_df['participantUuid'] == participant) & within_range,'time_after_return'] = filtered_df.loc[(filtered_df['participantUuid'] == participant) & within_range,'hoursSinceStart'] - current_return_time\n",
    "        \n",
    "\n",
    "        prev_return_index = index\n",
    "\n",
    "# Now, filtered_df contains the labeled entries within 7 days before each \"Return\" row\n",
    "# and the 'time_to_return' column indicating time to the next return\n",
    "filtered_df = filtered_df[filtered_df['time_after_return'].notnull()]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_df.to_csv('data_returns.csv', index=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Intervention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_intervention_unique = df.drop_duplicates(subset=['interventionType'])\n",
    "df_intervention_unique = df_intervention_unique[df_intervention_unique['interventionType'].notnull()]\n",
    "df_intervention_unique['interventionType']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(df_noclose['userIndex'].unique()))\n",
    "\n",
    "df_interventions = df_noclose.groupby(['userIndex', 'interventionType'])['userIndex'].count()\n",
    "print(df_interventions)\n",
    "\n",
    "\n",
    "df_interventions_count = df_interventions.groupby('userIndex').count()\n",
    "print(df_interventions_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "userIndexes_in_noclose = set(df_noclose['userIndex'].unique())\n",
    "\n",
    "# Get unique userIndex values in df_interventions\n",
    "userIndexes_in_interventions = set(df_interventions.index.get_level_values('userIndex').unique())\n",
    "\n",
    "# Find userIndexes that are in df_noclose but not in df_interventions\n",
    "userIndexes_not_in_interventions = userIndexes_in_noclose - userIndexes_in_interventions\n",
    "\n",
    "# Convert the result to a list\n",
    "userIndexes_not_in_interventions_list = list(userIndexes_not_in_interventions)\n",
    "\n",
    "print(len(df_noclose['interventionType'].loc[df_noclose['userIndex'].isin([769, 636, 302])]))\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Resolution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['resolution'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Opened the app', len(df[df['resolution'] == 'openedApp']))\n",
    "print('Did not continue to the app', len(df[df['resolution'] == 'dismissedAppOpening']))\n",
    "print('Structured intervention', len(df[df['resolution'] == 'dismissedByStructuredIntervention']))\n",
    "print('Clicked on a healthy alternative', len(df[df['resolution'] == 'dismissedWithHealthyAlternative']))\n",
    "print('Left the app (at the end)', len(df[df['resolution'] == 'closedApp']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df[df['resolution'] == 'dismissedByStructuredIntervention']['userIndex'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = plt.hist(df['resolution'], bins=5)  # arguments are passed to np.histogram\n",
    "plt.title(\"Resolution\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Trend Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pymannkendall as mk"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overall Use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trend = df_entries_per_user_days.loc[df_entries_per_user_days['daysSinceStart'] <= 307]\n",
    "df_trend_result = df_entries_per_user_days_result.loc[df_entries_per_user_days_result['daysSinceStart'] <= 307]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = mk.original_test(df_trend['interactions_per_user'])\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = mk.original_test(df_trend_result[df_trend_result['continuedToApp'] == 'didNotOpenApp']['percentage'])\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = mk.sens_slope(df_trend['interactions_per_user'])\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = mk.sens_slope(df_trend_result[df_trend_result['continuedToApp'] == 'didNotOpenApp']['percentage'])\n",
    "result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## First 48 hours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trend_hours = df_entries_per_user_hours.loc[df_entries_per_user_hours['hoursSinceStart'] <= 48]\n",
    "df_trend_hours_result = df_entries_per_user_hours_result.loc[df_entries_per_user_hours_result['hoursSinceStart'] <= 48]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = mk.original_test(df_trend_hours['interactions_per_user'])\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = mk.original_test(df_trend_hours_result[df_trend_hours_result['continuedToApp'] == 'didNotOpenApp']['percentage'])\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = mk.sens_slope(df_trend_hours['interactions_per_user'])\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = mk.sens_slope(df_trend_hours_result[df_trend_hours_result['continuedToApp'] == 'didNotOpenApp']['percentage'])\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = mk.hamed_rao_modification_test(df_trend_hours['interactions_per_user'])\n",
    "result"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}