Remove Redis dependency (#12)

* Move all user events to NATS.

* Remove redlock usage.

* Track loading progress in database.

* Add admin view for imports.

Cargo.toml

@@ -54,8 +54,6 @@ percent-encoding = "2.2"
 prometheus = { version = "0.13", features = ["process"] }
 radix_fmt = "1"
 rand = "0.8"
-redis = { version = "0.25.4", features = ["tokio-comp", "aio", "connection-manager"] }
-redlock = { git = "https://github.com/Syfaro/redlock-rs.git" }
 regex = "1"
 reqwest = { version = "0.11", features = ["json", "cookies"] }
 roux = "2"

README.md

@@ -44,10 +44,9 @@ up and running.
 It has a few software dependencies to run:
 
 * PostgreSQL with the [bktree] extension as the primary backing datastore
-* Redis to send events about user actions
+* NATS for pubsub and distributing events
 * [Faktory] for managing background tasks
 * [faktory-cron] to run scheduled jobs (provided in jobs.yaml)
-* NATS for publishing information on newly discovered public images
 
 It also requires credentials or app tokens for the following sites:
 
@@ -60,7 +59,6 @@ It also requires credentials or app tokens for the following sites:
 * S3-like (endpoint, region, bucket, access, and secret key)
 * SMTP (host, username, and password)
 * Telegram (token and domain)
-* Twitter (API token and secret)
 
 [bktree]: https://github.com/fake-name/pg-spgist_hamming
 [faktory]: https://github.com/contribsys/faktory

migrations/20240601011418_loading_accounts.down.sql

@@ -0,0 +1 @@
+DROP TABLE linked_account_import;

migrations/20240601011418_loading_accounts.up.sql

@@ -0,0 +1,10 @@
+CREATE TABLE linked_account_import (
+    linked_account_id uuid PRIMARY KEY REFERENCES linked_account (id) ON DELETE CASCADE,
+    started_at timestamp with time zone NOT NULL DEFAULT current_timestamp,
+    completed_at timestamp with time zone,
+    expected_count integer NOT NULL,
+    expected_ids text[] NOT NULL,
+    loaded_ids text[] NOT NULL DEFAULT array[]::text[]
+);
+
+CREATE INDEX linked_account_import_started_at_idx ON linked_account_import (started_at DESC);

queries/linked_account/lookup_by_id_for_update.sql

@@ -0,0 +1,7 @@
+SELECT
+    *
+FROM
+    linked_account
+WHERE
+    id = $1
+FOR UPDATE;

queries/linked_account_import/admin_list.sql

@@ -0,0 +1,14 @@
+SELECT
+    linked_account_id,
+    linked_account.source_site,
+    started_at,
+    completed_at,
+    expected_count,
+    cardinality(loaded_ids) loaded_count
+FROM
+    linked_account_import
+    JOIN linked_account ON linked_account.id = linked_account_import.linked_account_id
+ORDER BY
+    started_at DESC
+LIMIT
+    100;

queries/linked_account_import/complete.sql

@@ -0,0 +1,6 @@
+UPDATE
+    linked_account_import
+SET
+    completed_at = current_timestamp
+WHERE
+    linked_account_id = $1;

queries/linked_account_import/loaded.sql

@@ -0,0 +1,9 @@
+UPDATE
+    linked_account_import
+SET
+    loaded_ids = array_append(loaded_ids, $2)
+WHERE
+    linked_account_id = $1
+RETURNING
+    expected_count,
+    cardinality(loaded_ids) loaded_count;

queries/linked_account_import/start.sql

@@ -0,0 +1,4 @@
+INSERT INTO
+    linked_account_import (linked_account_id, expected_count, expected_ids)
+VALUES
+    ($1, $2, $3);