Switch to postgres

backend/.gitignore

@@ -7,3 +7,4 @@ storage/
 data/
 !data/pages/.gitkeep
 .pdm.toml
+db/

backend/pyproject.toml

@@ -20,6 +20,7 @@ dependencies = [
     "python-magic>=0.4.27",
     "transcribee-proto @ file:///${PROJECT_ROOT}/../proto",
     "python-frontmatter>=1.0.0",
+    "psycopg2-binary>=2.9.9",
 ]
 requires-python = ">=3.10"
 readme = "./README.md"

backend/transcribee_backend/db/__init__.py

@@ -1,24 +1,17 @@
 import os
 from contextlib import contextmanager
+from pathlib import Path
 
-from sqlalchemy import event
-from sqlalchemy.engine import Engine
 from sqlmodel import Session, create_engine
 
+DEFAULT_SOCKET_PATH = Path(__file__).parent.parent.parent / "db" / "sockets"
 
-@event.listens_for(Engine, "connect")
-def set_sqlite_pragma(dbapi_connection, connection_record):
-    cursor = dbapi_connection.cursor()
-    cursor.execute("PRAGMA foreign_keys=ON")
-    cursor.close()
+DATABASE_URL = os.environ.get(
+    "TRANSCRIBEE_BACKEND_DATABASE_URL",
+    f"postgresql:///transcribee?host={DEFAULT_SOCKET_PATH}",
+)
 
-
-DATABASE_URL = os.environ.get("TRANSCRIBEE_BACKEND_DATABASE_URL", "sqlite:///db.sqlite")
-
-connect_args = {}
-if DATABASE_URL.startswith("sqlite://"):
-    connect_args = {"check_same_thread": False}
-engine = create_engine(DATABASE_URL, connect_args=connect_args)
+engine = create_engine(DATABASE_URL)
 
 
 def get_session():

backend/transcribee_backend/db/migrations/versions/6392770332cd_add_taskattempt.py

@@ -50,8 +50,6 @@ def upgrade_with_autocommit() -> None:
     with op.batch_alter_table("taskattempt", schema=None) as batch_op:
         batch_op.create_index(batch_op.f("ix_taskattempt_id"), ["id"], unique=False)
 
-    op.execute("PRAGMA foreign_keys=OFF")
-
     with op.batch_alter_table("task", schema=None) as batch_op:
         batch_op.add_column(
             sa.Column("current_attempt_id", sqlmodel.sql.sqltypes.GUID(), nullable=True)
@@ -61,10 +59,15 @@ def upgrade_with_autocommit() -> None:
                 "attempt_counter", sa.Integer(), nullable=True, server_default="0"
             )
         )
+        taskstate_enum = sa.Enum(
+            "NEW", "ASSIGNED", "COMPLETED", "FAILED", name="taskstate"
+        )
+        bind = batch_op.get_bind()
+        taskstate_enum.create(bind=bind)
         batch_op.add_column(
             sa.Column(
                 "state",
-                sa.Enum("NEW", "ASSIGNED", "COMPLETED", "FAILED", name="taskstate"),
+                taskstate_enum,
                 nullable=False,
                 server_default="NEW",
             )
@@ -85,8 +88,6 @@ def upgrade_with_autocommit() -> None:
             use_alter=True,
         )
 
-    op.execute("PRAGMA foreign_keys=ON")
-
     Task = sa.table(
         "task",
         sa.column("id", sqlmodel.sql.sqltypes.GUID()),
@@ -154,7 +155,6 @@ def upgrade_with_autocommit() -> None:
             .values(remaining_attempts=settings.task_attempt_limit)
         )
 
-    op.execute("PRAGMA foreign_keys=OFF")
     with op.batch_alter_table("task", schema=None) as batch_op:
         batch_op.drop_column("completed_at")
         batch_op.drop_column("is_completed")
@@ -173,8 +173,6 @@ def upgrade_with_autocommit() -> None:
             "attempt_counter", existing_type=sa.INTEGER(), nullable=False
         )
 
-    op.execute("PRAGMA foreign_keys=ON")
-
 
 def downgrade() -> None:
     raise NotImplementedError()

frontend/src/editor/worker_status.tsx

@@ -48,9 +48,35 @@ export function WorkerStatus({ documentId }: { documentId: string }) {
   return <WorkerStatusWithData data={data} />;
 }
 
-export function WorkerStatusWithData({ data }: { data: Task[] }) {
+function isSuperset<T>(set: Set<T>, subset: Set<T>) {
+  for (const elem of subset) {
+    if (!set.has(elem)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+export function WorkerStatusWithData({ data: unsortedData }: { data: Task[] | undefined }) {
   const systemPrefersDark = useMediaQuery('(prefers-color-scheme: dark)');
 
+  const data: Task[] = [];
+  if (unsortedData !== undefined) {
+    const unsortedDataCopy = [...unsortedData];
+    const seenIds = new Set();
+    while (unsortedDataCopy.length > 0) {
+      const item = unsortedDataCopy.shift();
+      if (!item) break;
+      const dependencyIds = new Set(item.dependencies);
+      if (isSuperset(seenIds, dependencyIds)) {
+        data.push(item);
+        seenIds.add(item.id);
+      } else {
+        unsortedDataCopy.push(item);
+      }
+    }
+  }
+
   const isWorking = data?.some((task) => task.state !== 'COMPLETED');
   const isFailed = data?.some((task) => task.state == 'FAILED');
 

packaging/Procfile

@@ -1,3 +1,4 @@
 backend: ./start_backend.sh
 worker: pdm run -p ../worker/ start --coordinator http://127.0.0.1:8000 --token dev_worker --reload
 frontend: pnpm --prefix ../frontend/ dev --clearScreen false
+db: ./start_db.sh

packaging/dev.sh

@@ -7,7 +7,6 @@
 set -euxo pipefail
 
 ./packaging/install_dependencies.sh
-./packaging/setup_backend.sh
 
 echo -e "\n\n\033[1m# starting application:\033[0m\n"
 overmind start -f packaging/Procfile

packaging/setup_backend.sh

@@ -3,6 +3,6 @@
 set -euxo pipefail
 
 echo -e "\033[1m# setting up backend:\033[0m\n"
-pdm run -p backend/ migrate
-pdm run -p backend/ create_user --user test --pass test
-pdm run -p backend/ create_worker --token dev_worker --name "Development Worker"
+pdm run -p ../backend/ migrate
+pdm run -p ../backend/ create_user --user test --pass test
+pdm run -p ../backend/ create_worker --token dev_worker --name "Development Worker"

packaging/start_backend.sh

@@ -8,4 +8,6 @@ if [ !  -z "${TRANSCRIBEE_DYLD_LIBRARY_PATH:-}" ]; then
     export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH:${DYLD_LIBRARY_PATH:-}
 fi
 
+./setup_backend.sh
+
 pdm run -p ../backend/ dev

packaging/start_db.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env sh
+
+if [ ! -e ../backend/db/data ]; then
+    initdb -D ../backend/db/data
+fi
+
+if [ ! -e ../backend/db/sockets ]; then
+    mkdir ../backend/db/sockets
+fi
+
+trap "pg_ctl -D ../backend/db/data -o \"-h '' -k '$(pwd)/../backend/db/sockets/'\" stop" INT
+pg_ctl -D ../backend/db/data -o "-h '' -k '$(pwd)/../backend/db/sockets/'" start
+
+createdb -h"$(pwd)/../backend/db/sockets/" transcribee || true
+
+sleep infinity

shell.nix

@@ -53,6 +53,9 @@ pkgs.mkShell {
     file
 
     icu.dev
+
+    # Our database
+    postgresql
   ] ++
 
   # accelerates whisper.cpp on M{1,2} Macs