Fitbit_Data_Collection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import fitbit as fitbit\n",
    "import gather_keys_oauth2 as Oauth2\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "import datetime\n",
    "from datetime import timedelta  \n",
    "\n",
    "\n",
    "from src import secret as secret"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Client Authentication"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create API clients as follows:\n",
    "* auth2_client - for V1 of API\n",
    "* [auth2_client_new](https://github.com/orcasgit/python-fitbit/issues/128) - for V1.2 of API (required for Sleep levels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[17/Oct/2018:07:23:52] ENGINE Listening for SIGHUP.\n",
      "[17/Oct/2018:07:23:52] ENGINE Listening for SIGTERM.\n",
      "[17/Oct/2018:07:23:52] ENGINE Listening for SIGUSR1.\n",
      "[17/Oct/2018:07:23:52] ENGINE Bus STARTING\n",
      "CherryPy Checker:\n",
      "The Application mounted at '' has an empty config.\n",
      "\n",
      "[17/Oct/2018:07:23:52] ENGINE Started monitor thread 'Autoreloader'.\n",
      "[17/Oct/2018:07:23:52] ENGINE Serving on http://127.0.0.1:8080\n",
      "[17/Oct/2018:07:23:52] ENGINE Bus STARTED\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "127.0.0.1 - - [17/Oct/2018:07:23:55] \"GET /?code=408e642051019d11c7f05b23bd44773d11f6083d&state=3pwrFnnLRv3SJvQOHVpTVTvj6frrsL HTTP/1.1\" 200 122 \"\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[17/Oct/2018:07:23:56] ENGINE Bus STOPPING\n",
      "[17/Oct/2018:07:24:01] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('127.0.0.1', 8080)) shut down\n",
      "[17/Oct/2018:07:24:01] ENGINE Stopped thread 'Autoreloader'.\n",
      "[17/Oct/2018:07:24:01] ENGINE Bus STOPPED\n",
      "[17/Oct/2018:07:24:01] ENGINE Bus EXITING\n",
      "[17/Oct/2018:07:24:01] ENGINE Bus EXITED\n",
      "[17/Oct/2018:07:24:01] ENGINE Waiting for child threads to terminate...\n"
     ]
    }
   ],
   "source": [
    "#https://dev.fitbit.com/apps/details/22CZJ9\n",
    "CLIENT_ID = secret.CLIENT_ID\n",
    "CLIENT_SECRET = secret.CLIENT_SECRET\n",
    "\n",
    "server = Oauth2.OAuth2Server(CLIENT_ID, CLIENT_SECRET)\n",
    "server.browser_authorize()\n",
    "\n",
    "ACCESS_TOKEN = str(server.fitbit.client.session.token['access_token'])\n",
    "REFRESH_TOKEN = str(server.fitbit.client.session.token['refresh_token'])\n",
    "\n",
    "##Build Auth Clients\n",
    "auth2_client = fitbit.Fitbit(CLIENT_ID, CLIENT_SECRET, oauth2=True, access_token=ACCESS_TOKEN, refresh_token=REFRESH_TOKEN, system='en_GB')\n",
    "\n",
    "#NEed this seeparately because version 1 doesn't give sleep breakdown\n",
    "auth2_client_new = fitbit.Fitbit(CLIENT_ID, CLIENT_SECRET, oauth2=True, access_token=ACCESS_TOKEN, refresh_token=REFRESH_TOKEN, system='en_GB')\n",
    "auth2_client_new.API_VERSION = 1.2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Database layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATE_FORMAT = '%Y-%m-%d'\n",
    "databaseFilename = 'data/database_main.xls'\n",
    "\n",
    "def connectAndLoadDb():\n",
    "    print(\"Connecting database...\")\n",
    "    database = pd.read_excel(databaseFilename)\n",
    "    print(\"Database connected!\")\n",
    "    return database;\n",
    "\n",
    "def getLastEntryDate(database):\n",
    "    lastDateEntry = database.iloc[-1]['Date']\n",
    "    lastDateEntry = datetime.datetime.strptime(lastDateEntry, DATE_FORMAT)    \n",
    "    return lastDateEntry\n",
    "\n",
    "def addEntriesInDB(dictionary, database):\n",
    "    #print(dictionary)\n",
    "    database = database.append(dictionary, ignore_index=True)\n",
    "    return addEntriesInDB;\n",
    "\n",
    "def writeDbToExcelFile(database):\n",
    "    print('Writing database to filename: '+ databaseFilename)\n",
    "    writer = pd.ExcelWriter(databaseFilename)\n",
    "    database.to_excel(writer, 'main')\n",
    "    writer.save()\n",
    "    print('Database updated with new entries!!')\n",
    "    \n",
    "def prettyPrintDate(date):\n",
    "    return date.strftime(DATE_FORMAT);    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Activity and Sleep Parsers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#x[4] if len(x) == 4 else 'No'\n",
    "def safeGet(obj, key, defaultVal = np.nan):\n",
    "    return obj.get(key, defaultVal)  \n",
    "\n",
    "def percent(val):\n",
    "    return np.ceil(val*100)\n",
    "\n",
    "def filterArrObj(arrList, keyName, keyValue):\n",
    "    for arrItem in arrList:\n",
    "        if arrItem.get(keyName) == keyValue:\n",
    "            return arrItem\n",
    "    return {} \n",
    "\n",
    "def mergeDicts(dicts):\n",
    "    super_dict = {}\n",
    "    for singleDict in dicts:\n",
    "        for k, v in singleDict.items(): \n",
    "            super_dict[k] = v\n",
    "    return super_dict\n",
    "\n",
    "    \n",
    "def getActivities(date):\n",
    "    activitiyResponse = auth2_client.activities(date=date)\n",
    "    \n",
    "    activitySummary = activitiyResponse['summary'];\n",
    "    activityData = {\n",
    "        'Calories Burned':safeGet(activitySummary,'caloriesOut'),\n",
    "        'Calories BMR': safeGet(activitySummary,'caloriesBMR'),\n",
    "        'Resting Heart Rate':safeGet(activitySummary,'restingHeartRate'),\n",
    "        'Steps':safeGet(activitySummary,'steps'),\n",
    "        'Distance (Km)':filterArrObj(activitySummary.get('distances', []), 'activity', 'total').get('distance', np.nan),\n",
    "        'Elevation (Ft)':activitySummary['elevation'],\n",
    "        'Floors':activitySummary['floors'],\n",
    "        'Minutes Sedentary':activitySummary['sedentaryMinutes'],\n",
    "        'Minutes Lightly Active':activitySummary['lightlyActiveMinutes'],\n",
    "        'Minutes Fairly Active':activitySummary['fairlyActiveMinutes'],\n",
    "        'Minutes Very Active':activitySummary['veryActiveMinutes'],\n",
    "        'Activity Calories': activitySummary['activityCalories'],\n",
    "        'Active Score': activitySummary['activeScore'],\n",
    "        'Cardio minutes': filterArrObj(activitySummary.get('heartRateZones', []), 'name', 'Cardio').get('minutes', np.nan),\n",
    "        'Cardio calories': filterArrObj(activitySummary.get('heartRateZones', []), 'name', 'Cardio').get('caloriesOut', np.nan),\n",
    "        'Fat Burn minutes': filterArrObj(activitySummary.get('heartRateZones', []), 'name', 'Fat Burn').get('minutes', np.nan),\n",
    "        'Fat Burn calories': filterArrObj(activitySummary.get('heartRateZones', []), 'name', 'Fat Burn').get('caloriesOut', np.nan),\n",
    "        'Peak minutes': filterArrObj(activitySummary.get('heartRateZones', []), 'name', 'Peak').get('minutes', np.nan),\n",
    "        'Peak calories': filterArrObj(activitySummary.get('heartRateZones', []), 'name', 'Peak').get('caloriesOut', np.nan),\n",
    "        'Normal Cardio minutes': filterArrObj(activitySummary.get('heartRateZones', []), 'name', 'Out of Range').get('minutes', np.nan),\n",
    "        'Normal Cardio calories': filterArrObj(activitySummary.get('heartRateZones', []), 'name', 'Out of Range').get('caloriesOut', np.nan),\n",
    "    }\n",
    "    \n",
    "    return activityData\n",
    "\n",
    "def getSleep(date):\n",
    "    sleepResponse = auth2_client_new.sleep(date=date)\n",
    "    \n",
    "    sleepData = {}\n",
    "    for sleepLog in sleepResponse.get('sleep', []):\n",
    "        if sleepLog.get('isMainSleep'):\n",
    "            sleepLevelsSummary = sleepLog.get('levels', {}).get('summary', {})\n",
    "            \n",
    "            sleepData['Sleep Efficiency'] = safeGet(sleepLog, 'efficiency')\n",
    "            sleepData['Minutes Asleep'] = safeGet(sleepLog, 'minutesAsleep')\n",
    "            sleepData['Minutes to fall asleep'] = safeGet(sleepLog, 'minutesToFallAsleep')\n",
    "            sleepData['Sleep Start time'] = safeGet(sleepLog, 'startTime')\n",
    "            sleepData['Sleep End time'] = safeGet(sleepLog, 'endTime')\n",
    "            \n",
    "            sleepData['Time in bed'] = safeGet(sleepLog, 'timeInBed')\n",
    "            \n",
    "            sleepData['Minutes Deep sleep'] = safeGet(sleepLevelsSummary.get('deep', {}), 'minutes')\n",
    "            sleepData['Deep sleep count'] = safeGet(sleepLevelsSummary.get('deep', {}), 'count')\n",
    "            sleepData['% Deep sleep'] = percent(safeGet(sleepData, 'Minutes Deep sleep', 0)/safeGet(sleepData, 'Time in bed', 0))\n",
    "            \n",
    "            sleepData['Minutes Light sleep'] = safeGet(sleepLevelsSummary.get('light', {}), 'minutes')\n",
    "            sleepData['Light sleep count'] = safeGet(sleepLevelsSummary.get('light', {}), 'count')\n",
    "            sleepData['% Light sleep'] = percent(safeGet(sleepData, 'Minutes Light sleep', 0)/safeGet(sleepData, 'Time in bed', 0))\n",
    "            \n",
    "            sleepData['Minutes REM sleep'] = safeGet(sleepLevelsSummary.get('rem', {}), 'minutes')\n",
    "            sleepData['REM sleep count'] = safeGet(sleepLevelsSummary.get('rem', {}), 'count')\n",
    "            sleepData['% REM sleep'] = percent(safeGet(sleepData, 'Minutes REM sleep', 0)/safeGet(sleepData, 'Time in bed', 0))\n",
    "            \n",
    "            sleepData['Minutes Asleep'] = sleepData['Minutes Deep sleep'] + sleepData['Minutes Light sleep'] + sleepData['Minutes REM sleep']\n",
    "            sleepData['Minutes Awake'] = safeGet(sleepLevelsSummary.get('wake', {}), 'minutes')\n",
    "            sleepData['Minutes Awake count'] = safeGet(sleepLevelsSummary.get('wake', {}), 'count')            \n",
    "    return sleepData\n",
    "\n",
    "def getDateData(date):\n",
    "    weekDayNum = date.isoweekday()\n",
    "    return {\n",
    "        'Day of Week': weekDayNum,\n",
    "        'Is Weekday': weekDayNum<6,\n",
    "        'Is Weekend': weekDayNum>5,\n",
    "        'Date': prettyPrintDate(date)\n",
    "    };\n",
    "\n",
    "def fetchAllData(date):\n",
    "    dateStr = prettyPrintDate(date)\n",
    "    \n",
    "    print(\"Fetching fitbit data for: \" + dateStr)\n",
    "    \n",
    "    nextDate = date + timedelta(days=1)\n",
    "    sleepData = getSleep(prettyPrintDate(nextDate))\n",
    "    \n",
    "    activitiesData = getActivities(dateStr)\n",
    "    dateData = getDateData(date)\n",
    "    \n",
    "    mergedData = mergeDicts([sleepData, activitiesData, dateData])\n",
    "    return mergedData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#getSleep('2018-08-21') #multiple entries\n",
    "#getSleep('2018-07-22') #https://www.fitbit.com/sleep/2018-07-22/18934545047\n",
    "#getSleep('2018-08-2') #no sleep data\n",
    "#print(getActivities('2018-06-29'))\n",
    "#print(getSleep('2018-08-28')) #no sleep data\n",
    "\n",
    "#auth2_client.activities(date='2018-09-05')\n",
    "#auth2_client_new.sleep(date='2018-09-04')\n",
    "#fetchAllData(datetime.datetime.strptime('2018-09-04', DATE_FORMAT))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data fetch layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Connecting database...\n",
      "Database connected!\n"
     ]
    }
   ],
   "source": [
    "database = connectAndLoadDb()\n",
    "\n",
    "def shouldFetchDataForProvidedDate(providedDate, todaysDate, API_COUNTER):\n",
    "    return (providedDate < todaysDate) and API_COUNTER < 100;\n",
    "\n",
    "def fetchAndAppendToDb(date, database):    \n",
    "    mergedData = fetchAllData(date)\n",
    "    database = database.append(mergedData, ignore_index=True)\n",
    "    return database;\n",
    "\n",
    "def fetchData(database, refetchAll = False):\n",
    "    API_COUNTER = 0\n",
    "    \n",
    "    todaysDate = datetime.datetime.today()\n",
    "    \n",
    "    print(\"Date today is :\" + prettyPrintDate(todaysDate))\n",
    "    \n",
    "    if refetchAll == True:\n",
    "        lastEntryDate = datetime.datetime.strptime('2018-06-23', DATE_FORMAT)\n",
    "    else:\n",
    "        lastEntryDate = getLastEntryDate(database)\n",
    "    \n",
    "    print(\"Last entry in Db is of :\" + prettyPrintDate(lastEntryDate))\n",
    "    print(\"----------------------------------------------\")\n",
    "    \n",
    "    dateToFetch = lastEntryDate + timedelta(days=1)\n",
    "\n",
    "    while shouldFetchDataForProvidedDate(dateToFetch, todaysDate, API_COUNTER):    \n",
    "        database = fetchAndAppendToDb(dateToFetch, database)\n",
    "        dateToFetch = dateToFetch + timedelta(days=1)\n",
    "        API_COUNTER = API_COUNTER+1\n",
    "    \n",
    "    print(\"----------------------------------------------\")\n",
    "    print(\"Data fill completed! 👍👍\")\n",
    "    return database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Date today is :2018-10-17\n",
      "Last entry in Db is of :2018-10-12\n",
      "----------------------------------------------\n",
      "Fetching fitbit data for: 2018-10-13\n",
      "Fetching fitbit data for: 2018-10-14\n",
      "Fetching fitbit data for: 2018-10-15\n",
      "Fetching fitbit data for: 2018-10-16\n",
      "Fetching fitbit data for: 2018-10-17\n",
      "----------------------------------------------\n",
      "Data fill completed! 👍👍\n"
     ]
    }
   ],
   "source": [
    "#database = fetchData(database, refetchAll = True)\n",
    "database = fetchData(database)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing database to filename: data/database_main.xls\n",
      "Database updated with new entries!!\n"
     ]
    }
   ],
   "source": [
    "writeDbToExcelFile(database)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:py3]",
   "language": "python",
   "name": "conda-env-py3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}