Implemented 'fetch' command

.circleci/config.yml

@@ -47,6 +47,7 @@ jobs:
             pip install -e .
             pip install pytest
             pytest -vv
+            pocket-to-sqlite --help
   test-python-install:
     parameters:
       version:
@@ -75,6 +76,7 @@ jobs:
             pip install -e .
             pip install pytest
             pytest -vv
+            pocket-to-sqlite --help
   deploy:
     docker:
       - image: circleci/python:3.6

README.md

@@ -4,7 +4,7 @@
 [![CircleCI](https://circleci.com/gh/dogsheep/pocket-to-sqlite.svg?style=svg)](https://circleci.com/gh/dogsheep/pocket-to-sqlite)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/dogsheep/pocket-to-sqlite/blob/master/LICENSE)
 
-Create a SQLite database containing data from your Pocket account
+Create a SQLite database containing data from your [Pocket](pocket-to-sqlite fetch pocket.db) account.
 
 ## How to install
 
@@ -21,3 +21,11 @@ You will need to first obtain a valid OAuth token for your Pocket account. You c
 
     Once you have signed in there, hit <enter> to continue
     Authentication tokens written to auth.json
+
+Now you can fetch all of your items from Pocket like this:
+
+    $ pocket-to-sqlite fetch pocket.db
+
+## Using with Datasette
+
+The SQLite database produced by this tool is designed to be browsed using [Datasette](https://datasette.readthedocs.io/).

pocket_to_sqlite/cli.py

@@ -2,6 +2,8 @@
 import json
 import urllib.parse
 import requests
+import sqlite_utils
+from . import utils
 
 CONSUMER_KEY = "87988-a6fd295a556dbdb47960ec60"
 
@@ -46,3 +48,27 @@ def auth(auth):
     codes["consumer_key"] = CONSUMER_KEY
     open(auth, "w").write(json.dumps(codes, indent=4) + "\n")
     click.echo("Authentication tokens written to {}".format(auth))
+
+
+@cli.command()
+@click.argument(
+    "db_path",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+@click.option(
+    "-a",
+    "--auth",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
+    default="auth.json",
+    help="Path to auth tokens, defaults to auth.json",
+)
+@click.option("-s", "--silent", is_flag=True, help="Don't show progress bar")
+def fetch(db_path, auth, silent):
+    "Save Pocket data to a SQLite database"
+    auth = json.load(open(auth))
+    items = utils.fetch_all_items(auth)
+    db = sqlite_utils.Database(db_path)
+    # TODO: Progress bar if not silent
+    utils.save_items(items, db)
+    utils.ensure_fts(db)

pocket_to_sqlite/utils.py

@@ -4,33 +4,34 @@
 
 
 def save_items(items, db):
+    items_authors_to_save = []
     for item in items:
         transform(item)
-    authors = item.pop("authors", None)
-    items_authors_to_save = []
-    if authors:
-        authors_to_save = []
-        for details in authors.values():
-            authors_to_save.append(
-                {
-                    "author_id": int(details["author_id"]),
-                    "name": details["name"],
-                    "url": details["url"],
-                }
-            )
-            items_authors_to_save.append(
-                {
-                    "author_id": int(details["author_id"]),
-                    "item_id": int(details["item_id"]),
-                }
-            )
-        db["authors"].upsert_all(authors_to_save, pk="author_id")
+        authors = item.pop("authors", None)
+        if authors:
+            authors_to_save = []
+            for details in authors.values():
+                authors_to_save.append(
+                    {
+                        "author_id": int(details["author_id"]),
+                        "name": details["name"],
+                        "url": details["url"],
+                    }
+                )
+                items_authors_to_save.append(
+                    {
+                        "author_id": int(details["author_id"]),
+                        "item_id": int(details["item_id"]),
+                    }
+                )
+            db["authors"].upsert_all(authors_to_save, pk="author_id")
     db["items"].upsert_all(items, pk="item_id", alter=True)
-    db["items_authors"].upsert_all(
-        items_authors_to_save,
-        pk=("author_id", "item_id"),
-        foreign_keys=("author_id", "item_id"),
-    )
+    if items_authors_to_save:
+        db["items_authors"].upsert_all(
+            items_authors_to_save,
+            pk=("author_id", "item_id"),
+            foreign_keys=("author_id", "item_id"),
+        )
 
 
 def transform(item):
@@ -51,13 +52,27 @@ def transform(item):
         "time_to_read",
         "listen_duration_estimate",
     ):
-        item[key] = int(item[key])
+        if key in item:
+            item[key] = int(item[key])
 
     for key in ("time_read", "time_favorited"):
-        if not item[key]:
+        if key in item and not item[key]:
             item[key] = None
 
 
 def ensure_fts(db):
     if "items_fts" not in db.table_names():
         db["items"].enable_fts(["resolved_title", "excerpt"], create_triggers=True)
+
+
+def fetch_all_items(auth):
+    # TODO: Use pagination, don't attempt to pull all at once
+    data = requests.get(
+        "https://getpocket.com/v3/get",
+        {
+            "consumer_key": auth["consumer_key"],
+            "access_token": auth["access_token"],
+            "detailType": "complete",
+        },
+    ).json()
+    return list(data["list"].values())