Flatten index to remove expensive joins during invalidation

packages/host/app/lib/sqlite-adapter.ts

@@ -13,6 +13,7 @@ import {
 } from '@cardstack/runtime-common';
 
 export default class SQLiteAdapter implements DBAdapter {
+  readonly kind = 'sqlite';
   private _sqlite: typeof SQLiteWorker | undefined;
   private _dbId: string | undefined;
   private primaryKeys = new Map<string, string>();

packages/host/config/schema/1735668047598_schema.sql → packages/host/config/schema/1735832183444_schema.sql

@@ -24,7 +24,33 @@
    display_names BLOB,
    resource_created_at,
    icon_html TEXT,
-   PRIMARY KEY ( url, realm_version, realm_url ) 
+   PRIMARY KEY ( url, realm_url ) 
+);
+
+ CREATE TABLE IF NOT EXISTS boxel_index_working (
+   url TEXT NOT NULL,
+   file_alias TEXT NOT NULL,
+   type TEXT NOT NULL,
+   realm_version INTEGER NOT NULL,
+   realm_url TEXT NOT NULL,
+   pristine_doc BLOB,
+   search_doc BLOB,
+   error_doc BLOB,
+   deps BLOB,
+   types BLOB,
+   icon_html TEXT,
+   isolated_html TEXT,
+   indexed_at,
+   is_deleted BOOLEAN,
+   source TEXT,
+   transpiled_code TEXT,
+   last_modified,
+   embedded_html BLOB,
+   atom_html TEXT,
+   fitted_html BLOB,
+   display_names BLOB,
+   resource_created_at,
+   PRIMARY KEY ( url, realm_url ) 
 );
 
  CREATE TABLE IF NOT EXISTS realm_meta (

packages/host/tests/unit/index-query-engine-test.ts

@@ -484,15 +484,6 @@ module('Unit | query', function (hooks) {
     });
   });
 
-  test('can get paginated results that are stable during index mutations', async function (assert) {
-    await runSharedTest(indexQueryEngineTests, assert, {
-      indexQueryEngine,
-      dbAdapter,
-      loader,
-      testCards,
-    });
-  });
-
   test(`can filter using 'gt'`, async function (assert) {
     await runSharedTest(indexQueryEngineTests, assert, {
       indexQueryEngine,

packages/host/tests/unit/index-writer-test.ts

@@ -48,14 +48,6 @@ module('Unit | index-writer', function (hooks) {
     });
   });
 
-  test('only invalidates latest version of content', async function (assert) {
-    await runSharedTest(indexWriterTests, assert, {
-      indexWriter,
-      indexQueryEngine,
-      adapter,
-    });
-  });
-
   test('can update an index entry', async function (assert) {
     await runSharedTest(indexWriterTests, assert, {
       indexWriter,

packages/postgres/migrations/1735832183444_add-boxel-index-working.js

@@ -0,0 +1,73 @@
+exports.up = (pgm) => {
+  pgm.createTable('boxel_index_working', {
+    url: { type: 'varchar', notNull: true },
+    file_alias: { type: 'varchar', notNull: true },
+    type: { type: 'varchar', notNull: true },
+    realm_version: { type: 'integer', notNull: true },
+    realm_url: { type: 'varchar', notNull: true },
+    pristine_doc: 'jsonb',
+    search_doc: 'jsonb',
+    error_doc: 'jsonb',
+    deps: 'jsonb',
+    types: 'jsonb',
+    icon_html: 'varchar',
+    isolated_html: 'varchar',
+    indexed_at: 'bigint',
+    is_deleted: 'boolean',
+    source: 'varchar',
+    transpiled_code: 'varchar',
+    last_modified: 'bigint',
+    embedded_html: 'jsonb',
+    atom_html: 'varchar',
+    fitted_html: 'jsonb',
+    display_names: 'jsonb',
+    resource_created_at: 'bigint',
+  });
+  pgm.sql('ALTER TABLE boxel_index_working SET UNLOGGED');
+  pgm.addConstraint('boxel_index_working', 'boxel_index_working_pkey', {
+    primaryKey: ['url', 'realm_url'],
+  });
+  pgm.createIndex('boxel_index_working', ['realm_version']);
+  pgm.createIndex('boxel_index_working', ['realm_url']);
+  pgm.createIndex('boxel_index_working', ['file_alias']);
+  pgm.createIndex('boxel_index_working', ['resource_created_at']);
+  pgm.createIndex('boxel_index_working', ['last_modified']);
+  pgm.createIndex('boxel_index_working', 'type');
+  pgm.createIndex('boxel_index_working', ['url', 'realm_version']);
+  pgm.createIndex('boxel_index_working', 'deps', { method: 'gin' });
+  pgm.createIndex('boxel_index_working', 'types', { method: 'gin' });
+  pgm.createIndex('boxel_index_working', 'fitted_html', { method: 'gin' });
+  pgm.createIndex('boxel_index_working', 'embedded_html', { method: 'gin' });
+  pgm.createIndex('boxel_index_working', 'search_doc', { method: 'gin' });
+
+  pgm.sql('delete from boxel_index');
+  pgm.sql('delete from realm_versions;');
+  pgm.sql('delete from job_reservations');
+  pgm.sql('delete from jobs');
+
+  pgm.dropConstraint('boxel_index', 'boxel_index_pkey');
+  pgm.addConstraint('boxel_index', 'boxel_index_pkey', {
+    primaryKey: ['url', 'realm_url'],
+  });
+};
+
+exports.down = (pgm) => {
+  pgm.dropIndex('boxel_index_working', ['realm_version']);
+  pgm.dropIndex('boxel_index_working', ['realm_url']);
+  pgm.dropIndex('boxel_index_working', ['file_alias']);
+  pgm.dropIndex('boxel_index_working', ['resource_created_at']);
+  pgm.dropIndex('boxel_index_working', ['last_modified']);
+  pgm.dropIndex('boxel_index_working', 'type');
+  pgm.dropIndex('boxel_index_working', ['url', 'realm_version']);
+  pgm.dropIndex('boxel_index_working', 'deps');
+  pgm.dropIndex('boxel_index_working', 'types');
+  pgm.dropIndex('boxel_index_working', 'fitted_html');
+  pgm.dropIndex('boxel_index_working', 'embedded_html');
+  pgm.dropIndex('boxel_index_working', 'search_doc');
+  pgm.dropTable('boxel_index_working', { cascade: true });
+
+  pgm.dropConstraint('boxel_index', 'boxel_index_pkey');
+  pgm.addConstraint('boxel_index', 'boxel_index_pkey', {
+    primaryKey: ['url', 'realm_url', 'realm_version'],
+  });
+};

packages/postgres/pg-adapter.ts

@@ -23,6 +23,7 @@ function config() {
 type Config = ReturnType<typeof config>;
 
 export class PgAdapter implements DBAdapter {
+  readonly kind = 'pg';
   #isClosed = false;
   private pool: Pool;
   private started: Promise<void>;
@@ -124,7 +125,7 @@ export class PgAdapter implements DBAdapter {
 
     let client = await this.pool.connect();
     let query = async (expression: Expression) => {
-      let sql = expressionToSql(expression);
+      let sql = expressionToSql(this.kind, expression);
       log.debug('search: %s trace: %j', sql.text, sql.values);
       let { rows } = await client.query(sql);
       return rows;

packages/realm-server/tests/index-query-engine-test.ts

@@ -424,15 +424,6 @@ module('query', function (hooks) {
     });
   });
 
-  test('can get paginated results that are stable during index mutations', async function (assert) {
-    await runSharedTest(indexQueryEngineTests, assert, {
-      indexQueryEngine,
-      dbAdapter,
-      loader,
-      testCards: await makeTestCards(loader),
-    });
-  });
-
   test(`can filter using 'gt'`, async function (assert) {
     await runSharedTest(indexQueryEngineTests, assert, {
       indexQueryEngine,

packages/realm-server/tests/index-writer-test.ts

@@ -45,14 +45,6 @@ module('index-writer', function (hooks) {
     });
   });
 
-  test('only invalidates latest version of content', async function (assert) {
-    await runSharedTest(indexWriterTests, assert, {
-      indexWriter,
-      indexQueryEngine,
-      adapter,
-    });
-  });
-
   test('can update an index entry', async function (assert) {
     await runSharedTest(indexWriterTests, assert, {
       indexWriter,

packages/runtime-common/db.ts

@@ -12,6 +12,7 @@ export interface ExecuteOptions {
 }
 
 export interface DBAdapter {
+  kind: 'pg' | 'sqlite';
   isClosed: boolean;
   execute: (
     sql: string,

packages/runtime-common/expression.ts

@@ -5,7 +5,13 @@ import flattenDeep from 'lodash/flattenDeep';
 
 import { type CodeRef, type DBAdapter, type TypeCoercion } from './index';
 
-export type Expression = (string | Param | TableValuedEach | TableValuedTree)[];
+export type Expression = (
+  | string
+  | Param
+  | TableValuedEach
+  | TableValuedTree
+  | DBSpecificExpression
+)[];
 
 export type PgPrimitive =
   | number
@@ -16,10 +22,18 @@ export type PgPrimitive =
   | null;
 
 export interface Param {
-  param: PgPrimitive;
+  param?: PgPrimitive;
+  pg?: PgPrimitive;
+  sqlite?: PgPrimitive;
   kind: 'param';
 }
 
+export interface DBSpecificExpression {
+  pg?: PgPrimitive;
+  sqlite?: PgPrimitive;
+  kind: 'db-specific-expression';
+}
+
 export interface FieldQuery {
   type: CodeRef;
   path: string;
@@ -62,6 +76,7 @@ export interface FieldArity {
 export type CardExpression = (
   | string
   | Param
+  | DBSpecificExpression
   | TableValuedEach
   | TableValuedTree
   | FieldQuery
@@ -92,12 +107,50 @@ export function separatedByCommas(expressions: unknown[][]): unknown {
   }, []);
 }
 
-export function param(value: PgPrimitive): Param {
+export function param(value: { pg?: PgPrimitive; sqlite?: PgPrimitive }): Param;
+export function param(value: PgPrimitive): Param;
+export function param(
+  value: PgPrimitive | { pg?: PgPrimitive; sqlite?: PgPrimitive },
+): Param {
+  if (
+    value &&
+    typeof value === 'object' &&
+    ('pg' in value || 'sqlite' in value)
+  ) {
+    return {
+      ...value,
+      kind: 'param',
+    };
+  }
   return { param: value, kind: 'param' };
 }
 
 export function isParam(expression: any): expression is Param {
-  return isPlainObject(expression) && 'param' in expression;
+  return (
+    isPlainObject(expression) &&
+    'kind' in expression &&
+    expression.kind === 'param'
+  );
+}
+
+export function dbExpression({
+  pg,
+  sqlite,
+}: {
+  pg?: PgPrimitive;
+  sqlite?: PgPrimitive;
+}): DBSpecificExpression {
+  return { pg, sqlite, kind: 'db-specific-expression' };
+}
+
+export function isDbExpression(
+  expression: any,
+): expression is DBSpecificExpression {
+  return (
+    isPlainObject(expression) &&
+    'kind' in expression &&
+    expression.kind === 'db-specific-expression'
+  );
 }
 
 export function tableValuedEach(column: string): TableValuedEach {
@@ -226,12 +279,9 @@ export function asExpressions(
   let paramBucket = Object.fromEntries(
     Object.entries(values).map(([col, val]) => [
       col,
-      // TODO: SQLite requires JSON be referenced in a stringified
-      // manner--need to confirm if postgres is ok with this
-
-      // TODO probably we should insert using the json() or jsonb() function in
-      // SQLite, need to check for compatibility in postgres for this function
-      param(opts?.jsonFields?.includes(col) ? stringify(val) : val),
+      param(
+        opts?.jsonFields?.includes(col) ? stringify(val ?? null) : val ?? null,
+      ),
     ]),
   );
   let nameExpressions = Object.keys(paramBucket).map((name) => [name]);
@@ -309,14 +359,17 @@ export async function query(
   query: Expression,
   coerceTypes?: TypeCoercion,
 ) {
-  let sql = await expressionToSql(query);
+  let sql = await expressionToSql(dbAdapter.kind, query);
   return await dbAdapter.execute(sql.text, {
     coerceTypes,
     bind: sql.values,
   });
 }
 
-export function expressionToSql(query: Expression) {
+export function expressionToSql(
+  dbAdapterKind: DBAdapter['kind'],
+  query: Expression,
+) {
   let values: PgPrimitive[] = [];
   let nonce = 0;
   let tableValuedFunctions = new Map<
@@ -328,8 +381,11 @@ export function expressionToSql(query: Expression) {
   >();
   let text = query
     .map((element) => {
-      if (isParam(element)) {
-        values.push(element.param);
+      if (isDbExpression(element)) {
+        return element[dbAdapterKind] ?? '';
+      } else if (isParam(element)) {
+        let value = element[dbAdapterKind] ?? element.param ?? null;
+        values.push(value);
         return `$${values.length}`;
       } else if (typeof element === 'string') {
         return element;
@@ -386,28 +442,6 @@ export function expressionToSql(query: Expression) {
   };
 }
 
-export function realmVersionExpression(opts?: {
-  useWorkInProgressIndex?: boolean;
-  withMaxVersion?: number;
-}) {
-  return [
-    'realm_version =',
-    ...addExplicitParens([
-      'SELECT MAX(i2.realm_version)',
-      'FROM boxel_index i2',
-      'WHERE i2.url = i.url',
-      ...(opts?.withMaxVersion
-        ? ['AND i2.realm_version <=', param(opts?.withMaxVersion)]
-        : !opts?.useWorkInProgressIndex
-        ? // if we are not using the work in progress index then we limit the max
-          // version permitted to the current version for the realm
-          ['AND i2.realm_version <= r.current_version']
-        : // otherwise we choose the highest version in the system
-          []),
-    ]),
-  ] as Expression;
-}
-
 function trimBrackets(pathTraveled: string) {
   return pathTraveled.replace(/\[\]/g, '');
 }