From 1143fcc20b253213925f84f3979a27c9834fbbac Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 6 Oct 2019 22:07:37 -0700 Subject: [PATCH] Implemented 'fetch' command --- .circleci/config.yml | 2 ++ README.md | 10 +++++- pocket_to_sqlite/cli.py | 26 +++++++++++++++ pocket_to_sqlite/utils.py | 67 ++++++++++++++++++++++++--------------- 4 files changed, 78 insertions(+), 27 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0b15793..906d1bf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,6 +47,7 @@ jobs: pip install -e . pip install pytest pytest -vv + pocket-to-sqlite --help test-python-install: parameters: version: @@ -75,6 +76,7 @@ jobs: pip install -e . pip install pytest pytest -vv + pocket-to-sqlite --help deploy: docker: - image: circleci/python:3.6 diff --git a/README.md b/README.md index a0dfd2e..fa2979d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![CircleCI](https://circleci.com/gh/dogsheep/pocket-to-sqlite.svg?style=svg)](https://circleci.com/gh/dogsheep/pocket-to-sqlite) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/dogsheep/pocket-to-sqlite/blob/master/LICENSE) -Create a SQLite database containing data from your Pocket account +Create a SQLite database containing data from your [Pocket](pocket-to-sqlite fetch pocket.db) account. ## How to install @@ -21,3 +21,11 @@ You will need to first obtain a valid OAuth token for your Pocket account. You c Once you have signed in there, hit to continue Authentication tokens written to auth.json + +Now you can fetch all of your items from Pocket like this: + + $ pocket-to-sqlite fetch pocket.db + +## Using with Datasette + +The SQLite database produced by this tool is designed to be browsed using [Datasette](https://datasette.readthedocs.io/). diff --git a/pocket_to_sqlite/cli.py b/pocket_to_sqlite/cli.py index 5163606..34fbe37 100644 --- a/pocket_to_sqlite/cli.py +++ b/pocket_to_sqlite/cli.py @@ -2,6 +2,8 @@ import json import urllib.parse import requests +import sqlite_utils +from . import utils CONSUMER_KEY = "87988-a6fd295a556dbdb47960ec60" @@ -46,3 +48,27 @@ def auth(auth): codes["consumer_key"] = CONSUMER_KEY open(auth, "w").write(json.dumps(codes, indent=4) + "\n") click.echo("Authentication tokens written to {}".format(auth)) + + +@cli.command() +@click.argument( + "db_path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.option( + "-a", + "--auth", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + default="auth.json", + help="Path to auth tokens, defaults to auth.json", +) +@click.option("-s", "--silent", is_flag=True, help="Don't show progress bar") +def fetch(db_path, auth, silent): + "Save Pocket data to a SQLite database" + auth = json.load(open(auth)) + items = utils.fetch_all_items(auth) + db = sqlite_utils.Database(db_path) + # TODO: Progress bar if not silent + utils.save_items(items, db) + utils.ensure_fts(db) diff --git a/pocket_to_sqlite/utils.py b/pocket_to_sqlite/utils.py index e6bedba..7e0f901 100644 --- a/pocket_to_sqlite/utils.py +++ b/pocket_to_sqlite/utils.py @@ -4,33 +4,34 @@ def save_items(items, db): + items_authors_to_save = [] for item in items: transform(item) - authors = item.pop("authors", None) - items_authors_to_save = [] - if authors: - authors_to_save = [] - for details in authors.values(): - authors_to_save.append( - { - "author_id": int(details["author_id"]), - "name": details["name"], - "url": details["url"], - } - ) - items_authors_to_save.append( - { - "author_id": int(details["author_id"]), - "item_id": int(details["item_id"]), - } - ) - db["authors"].upsert_all(authors_to_save, pk="author_id") + authors = item.pop("authors", None) + if authors: + authors_to_save = [] + for details in authors.values(): + authors_to_save.append( + { + "author_id": int(details["author_id"]), + "name": details["name"], + "url": details["url"], + } + ) + items_authors_to_save.append( + { + "author_id": int(details["author_id"]), + "item_id": int(details["item_id"]), + } + ) + db["authors"].upsert_all(authors_to_save, pk="author_id") db["items"].upsert_all(items, pk="item_id", alter=True) - db["items_authors"].upsert_all( - items_authors_to_save, - pk=("author_id", "item_id"), - foreign_keys=("author_id", "item_id"), - ) + if items_authors_to_save: + db["items_authors"].upsert_all( + items_authors_to_save, + pk=("author_id", "item_id"), + foreign_keys=("author_id", "item_id"), + ) def transform(item): @@ -51,13 +52,27 @@ def transform(item): "time_to_read", "listen_duration_estimate", ): - item[key] = int(item[key]) + if key in item: + item[key] = int(item[key]) for key in ("time_read", "time_favorited"): - if not item[key]: + if key in item and not item[key]: item[key] = None def ensure_fts(db): if "items_fts" not in db.table_names(): db["items"].enable_fts(["resolved_title", "excerpt"], create_triggers=True) + + +def fetch_all_items(auth): + # TODO: Use pagination, don't attempt to pull all at once + data = requests.get( + "https://getpocket.com/v3/get", + { + "consumer_key": auth["consumer_key"], + "access_token": auth["access_token"], + "detailType": "complete", + }, + ).json() + return list(data["list"].values())