Added --since option, closes #3

README.md

@@ -29,6 +29,12 @@ Or as an environment variable:
     $ export FOURSQUARE_TOKEN=XXX
     $ swarm-to-sqlite checkins.db
 
+To retrieve just checkins within the past X hours, days or weeks, use the `--since=` option. For example, to pull only checkins that happened within the last 10 days use:
+
+    $ swarm-to-sqlite checkins.db --token=XXX --since=10d
+
+Use `2w` for two weeks, `10h` for ten hours, `3d` for three days.
+
 In addition to saving the checkins to a database, you can also write them to a JSON file using the `--save` option:
 
     $ swarm-to-sqlite checkins.db --save=checkins.json

swarm_to_sqlite/cli.py

@@ -1,10 +1,26 @@
 import click
 import os
 import json
+import re
 import sqlite_utils
 from .utils import save_checkin, ensure_foreign_keys, create_views, fetch_all_checkins
 
 
+since_re = re.compile("^(\d+)(w|h|d)$")
+def validate_since(ctx, param, value):
+    if value:
+        match = since_re.match(value)
+        if not match:
+            raise click.BadParameter('since need to be in format 3d/2h/1w')
+        num, unit = match.groups()
+        multiplier = {
+            "d": 24 * 60 * 60,
+            "h": 60 * 60,
+            "w": 7 * 24 * 60 * 60,
+        }[unit]
+        return int(num) * multiplier
+
+
 @click.command()
 @click.argument(
     "db_path",
@@ -18,8 +34,11 @@
 @click.option(
     "--save", type=click.File("w"), help="Save checkins to this JSON file on disk"
 )
+@click.option(
+    "--since", type=str, callback=validate_since, help="Look for checkins since 1w/2d/3h ago"
+)
 @click.option("-s", "--silent", is_flag=True, help="Don't show progress bar")
-def cli(db_path, token, load, save, silent):
+def cli(db_path, token, load, save, since, silent):
     "Save Swarm checkins to a SQLite database"
     if token and load:
         raise click.ClickException("Provide either --load or --token")
@@ -30,7 +49,7 @@ def cli(db_path, token, load, save, silent):
         )
 
     if token:
-        checkins = fetch_all_checkins(token, count_first=True)
+        checkins = fetch_all_checkins(token, count_first=True, since_delta=since)
         checkin_count = next(checkins)
     else:
         checkins = json.load(load)

swarm_to_sqlite/utils.py

@@ -1,4 +1,5 @@
 import datetime
+import time
 import requests
 from sqlite_utils.db import AlterError, ForeignKey
 
@@ -185,20 +186,22 @@ def create_views(db):
             pass
 
 
-def fetch_all_checkins(token, count_first=False):
+def fetch_all_checkins(token, count_first=False, since_delta=None):
     # Generator that yields all checkins using the provided OAuth token
     # If count_first is True it first yields the total checkins count
-    beforeTimestamp = None
+    before_timestamp = None
     params = {
         "oauth_token": token,
         "v": "20190101",
         "sort": "newestfirst",
         "limit": "250",
     }
+    if since_delta:
+        params["afterTimestamp"] = int(time.time() - since_delta)
     first = True
     while True:
-        if beforeTimestamp is not None:
-            params["beforeTimestamp"] = beforeTimestamp
+        if before_timestamp is not None:
+            params["beforeTimestamp"] = before_timestamp
         url = "https://api.foursquare.com/v2/users/self/checkins"
         data = requests.get(url, params).json()
         if first:
@@ -209,4 +212,4 @@ def fetch_all_checkins(token, count_first=False):
             break
         for item in data["response"]["checkins"]["items"]:
             yield item
-        beforeTimestamp = item["createdAt"]
+        before_timestamp = item["createdAt"]