Flatten index to remove expensive joins during invalidation

packages/host/config/schema/1735668047598_schema.sql → packages/host/config/schema/1735832183444_schema.sql

@@ -23,7 +23,32 @@
    fitted_html BLOB,
    display_names BLOB,
    resource_created_at,
-   PRIMARY KEY ( url, realm_version, realm_url ) 
+   PRIMARY KEY ( url, realm_url ) 
+);
+
+ CREATE TABLE IF NOT EXISTS boxel_index_working (
+   url TEXT NOT NULL,
+   file_alias TEXT NOT NULL,
+   type TEXT NOT NULL,
+   realm_version INTEGER NOT NULL,
+   realm_url TEXT NOT NULL,
+   pristine_doc BLOB,
+   search_doc BLOB,
+   error_doc BLOB,
+   deps BLOB,
+   types BLOB,
+   isolated_html TEXT,
+   indexed_at,
+   is_deleted BOOLEAN,
+   source TEXT,
+   transpiled_code TEXT,
+   last_modified,
+   embedded_html BLOB,
+   atom_html TEXT,
+   fitted_html BLOB,
+   display_names BLOB,
+   resource_created_at,
+   PRIMARY KEY ( url, realm_url ) 
 );
 
  CREATE TABLE IF NOT EXISTS realm_meta (

packages/host/tests/unit/index-query-engine-test.ts

@@ -484,15 +484,6 @@ module('Unit | query', function (hooks) {
     });
   });
 
-  test('can get paginated results that are stable during index mutations', async function (assert) {
-    await runSharedTest(indexQueryEngineTests, assert, {
-      indexQueryEngine,
-      dbAdapter,
-      loader,
-      testCards,
-    });
-  });
-
   test(`can filter using 'gt'`, async function (assert) {
     await runSharedTest(indexQueryEngineTests, assert, {
       indexQueryEngine,

packages/host/tests/unit/index-writer-test.ts

@@ -48,14 +48,6 @@ module('Unit | index-writer', function (hooks) {
     });
   });
 
-  test('only invalidates latest version of content', async function (assert) {
-    await runSharedTest(indexWriterTests, assert, {
-      indexWriter,
-      indexQueryEngine,
-      adapter,
-    });
-  });
-
   test('can update an index entry', async function (assert) {
     await runSharedTest(indexWriterTests, assert, {
       indexWriter,

packages/postgres/migrations/1735832183444_add-boxel-index-working.js

@@ -0,0 +1,59 @@
+exports.up = (pgm) => {
+  pgm.createTable('boxel_index_working', {
+    url: { type: 'varchar', notNull: true },
+    file_alias: { type: 'varchar', notNull: true },
+    type: { type: 'varchar', notNull: true },
+    realm_version: { type: 'integer', notNull: true },
+    realm_url: { type: 'varchar', notNull: true },
+    pristine_doc: 'jsonb',
+    search_doc: 'jsonb',
+    error_doc: 'jsonb',
+    deps: 'jsonb',
+    types: 'jsonb',
+    isolated_html: 'varchar',
+    indexed_at: 'bigint',
+    is_deleted: 'boolean',
+    source: 'varchar',
+    transpiled_code: 'varchar',
+    last_modified: 'bigint',
+    embedded_html: 'jsonb',
+    atom_html: 'varchar',
+    fitted_html: 'jsonb',
+    display_names: 'jsonb',
+    resource_created_at: 'bigint',
+  });
+  pgm.sql('ALTER TABLE boxel_index_working SET UNLOGGED');
+  pgm.addConstraint('boxel_index_working', 'boxel_index_working_pkey', {
+    primaryKey: ['url', 'realm_url'],
+  });
+  pgm.createIndex('boxel_index_working', ['realm_version']);
+  pgm.createIndex('boxel_index_working', ['realm_url']);
+  pgm.createIndex('boxel_index_working', ['file_alias']);
+  pgm.createIndex('boxel_index_working', ['resource_created_at']);
+  pgm.createIndex('boxel_index_working', ['last_modified']);
+  pgm.createIndex('boxel_index_working', 'type');
+  pgm.createIndex('boxel_index_working', ['url', 'realm_version']);
+  pgm.createIndex('boxel_index_working', 'deps', { method: 'gin' });
+  pgm.createIndex('boxel_index_working', 'types', { method: 'gin' });
+  pgm.createIndex('boxel_index_working', 'fitted_html', { method: 'gin' });
+  pgm.createIndex('boxel_index_working', 'embedded_html', { method: 'gin' });
+  pgm.createIndex('boxel_index_working', 'search_doc', { method: 'gin' });
+
+  pgm.sql('delete from boxel_index');
+  pgm.sql('delete from realm_versions;');
+  pgm.sql('delete from job_reservations');
+  pgm.sql('delete from jobs');
+
+  pgm.dropConstraint('boxel_index', 'boxel_index_pkey');
+  pgm.addConstraint('boxel_index', 'boxel_index_pkey', {
+    primaryKey: ['url', 'realm_url'],
+  });
+};
+
+exports.down = (pgm) => {
+  pgm.dropTable('boxel_index_working', { cascade: true });
+  pgm.dropConstraint('boxel_index', 'boxel_index_pkey');
+  pgm.addConstraint('boxel_index', 'boxel_index_pkey', {
+    primaryKey: ['url', 'realm_url', 'realm_version'],
+  });
+};

packages/realm-server/tests/index-query-engine-test.ts

@@ -424,15 +424,6 @@ module('query', function (hooks) {
     });
   });
 
-  test('can get paginated results that are stable during index mutations', async function (assert) {
-    await runSharedTest(indexQueryEngineTests, assert, {
-      indexQueryEngine,
-      dbAdapter,
-      loader,
-      testCards: await makeTestCards(loader),
-    });
-  });
-
   test(`can filter using 'gt'`, async function (assert) {
     await runSharedTest(indexQueryEngineTests, assert, {
       indexQueryEngine,

packages/realm-server/tests/index-writer-test.ts

@@ -45,14 +45,6 @@ module('index-writer', function (hooks) {
     });
   });
 
-  test('only invalidates latest version of content', async function (assert) {
-    await runSharedTest(indexWriterTests, assert, {
-      indexWriter,
-      indexQueryEngine,
-      adapter,
-    });
-  });
-
   test('can update an index entry', async function (assert) {
     await runSharedTest(indexWriterTests, assert, {
       indexWriter,

packages/runtime-common/expression.ts

@@ -386,28 +386,6 @@ export function expressionToSql(query: Expression) {
   };
 }
 
-export function realmVersionExpression(opts?: {
-  useWorkInProgressIndex?: boolean;
-  withMaxVersion?: number;
-}) {
-  return [
-    'realm_version =',
-    ...addExplicitParens([
-      'SELECT MAX(i2.realm_version)',
-      'FROM boxel_index i2',
-      'WHERE i2.url = i.url',
-      ...(opts?.withMaxVersion
-        ? ['AND i2.realm_version <=', param(opts?.withMaxVersion)]
-        : !opts?.useWorkInProgressIndex
-        ? // if we are not using the work in progress index then we limit the max
-          // version permitted to the current version for the realm
-          ['AND i2.realm_version <= r.current_version']
-        : // otherwise we choose the highest version in the system
-          []),
-    ]),
-  ] as Expression;
-}
-
 function trimBrackets(pathTraveled: string) {
   return pathTraveled.replace(/\[\]/g, '');
 }

packages/runtime-common/helpers/indexer.ts

@@ -109,6 +109,11 @@ export async function setupIndex(
   client: DBAdapter,
   indexRows: TestIndexRow[],
 ): Promise<void>;
+export async function setupIndex(
+  client: DBAdapter,
+  versionRows: RealmVersionsTable[],
+  indexRows: { working: TestIndexRow[]; production: TestIndexRow[] },
+): Promise<void>;
 export async function setupIndex(
   client: DBAdapter,
   versionRows: RealmVersionsTable[],
@@ -117,17 +122,109 @@ export async function setupIndex(
 export async function setupIndex(
   client: DBAdapter,
   maybeVersionRows: RealmVersionsTable[] | TestIndexRow[] = [],
-  indexRows?: TestIndexRow[],
+  maybeWorkingProductionRows?:
+    | TestIndexRow[]
+    | { working: TestIndexRow[]; production: TestIndexRow[] },
 ): Promise<void> {
   let versionRows: RealmVersionsTable[];
-  if (!indexRows) {
+  let workingRows: TestIndexRow[] = [];
+  let productionRows: TestIndexRow[] = [];
+  if (!maybeWorkingProductionRows) {
     versionRows = [{ realm_url: testRealmURL, current_version: 1 }];
-    indexRows = maybeVersionRows as TestIndexRow[];
+    workingRows = maybeVersionRows as TestIndexRow[];
+    productionRows = maybeVersionRows as TestIndexRow[];
   } else {
     versionRows = maybeVersionRows as RealmVersionsTable[];
+    if (Array.isArray(maybeWorkingProductionRows)) {
+      workingRows = maybeWorkingProductionRows as TestIndexRow[];
+      productionRows = maybeWorkingProductionRows as TestIndexRow[];
+    } else {
+      workingRows = maybeWorkingProductionRows.working;
+      productionRows = maybeWorkingProductionRows.production;
+    }
   }
   let now = Date.now();
-  let indexedCardsExpressions = await Promise.all(
+  let workingIndexedCardsExpressions = await indexedCardsExpressions({
+    indexRows: workingRows,
+    now,
+    client,
+  });
+  let productionIndexedCardsExpressions = await indexedCardsExpressions({
+    indexRows: productionRows,
+    now,
+    client,
+  });
+  let versionExpressions = versionRows.map((r) => asExpressions(r));
+
+  if (workingIndexedCardsExpressions.length > 0) {
+    await query(
+      client,
+      [
+        `INSERT INTO boxel_index_working`,
+        ...addExplicitParens(
+          separatedByCommas(workingIndexedCardsExpressions[0].nameExpressions),
+        ),
+        'VALUES',
+        ...separatedByCommas(
+          workingIndexedCardsExpressions.map((row) =>
+            addExplicitParens(separatedByCommas(row.valueExpressions)),
+          ),
+        ),
+      ] as Expression,
+      coerceTypes,
+    );
+  }
+  if (productionIndexedCardsExpressions.length > 0) {
+    await query(
+      client,
+      [
+        `INSERT INTO boxel_index`,
+        ...addExplicitParens(
+          separatedByCommas(
+            productionIndexedCardsExpressions[0].nameExpressions,
+          ),
+        ),
+        'VALUES',
+        ...separatedByCommas(
+          productionIndexedCardsExpressions.map((row) =>
+            addExplicitParens(separatedByCommas(row.valueExpressions)),
+          ),
+        ),
+      ] as Expression,
+      coerceTypes,
+    );
+  }
+
+  if (versionExpressions.length > 0) {
+    await query(
+      client,
+      [
+        `INSERT INTO realm_versions`,
+        ...addExplicitParens(
+          separatedByCommas(versionExpressions[0].nameExpressions),
+        ),
+        'VALUES',
+        ...separatedByCommas(
+          versionExpressions.map((row) =>
+            addExplicitParens(separatedByCommas(row.valueExpressions)),
+          ),
+        ),
+      ] as Expression,
+      coerceTypes,
+    );
+  }
+}
+
+async function indexedCardsExpressions({
+  indexRows,
+  now,
+  client,
+}: {
+  indexRows: TestIndexRow[];
+  now: number;
+  client: DBAdapter;
+}) {
+  return await Promise.all(
     indexRows.map(async (r) => {
       let row: Pick<RelaxedBoxelIndexTable, 'url'> &
         Partial<Omit<RelaxedBoxelIndexTable, 'url'>>;
@@ -182,43 +279,4 @@ export async function setupIndex(
       });
     }),
   );
-  let versionExpressions = versionRows.map((r) => asExpressions(r));
-
-  if (indexedCardsExpressions.length > 0) {
-    await query(
-      client,
-      [
-        `INSERT INTO boxel_index`,
-        ...addExplicitParens(
-          separatedByCommas(indexedCardsExpressions[0].nameExpressions),
-        ),
-        'VALUES',
-        ...separatedByCommas(
-          indexedCardsExpressions.map((row) =>
-            addExplicitParens(separatedByCommas(row.valueExpressions)),
-          ),
-        ),
-      ] as Expression,
-      coerceTypes,
-    );
-  }
-
-  if (versionExpressions.length > 0) {
-    await query(
-      client,
-      [
-        `INSERT INTO realm_versions`,
-        ...addExplicitParens(
-          separatedByCommas(versionExpressions[0].nameExpressions),
-        ),
-        'VALUES',
-        ...separatedByCommas(
-          versionExpressions.map((row) =>
-            addExplicitParens(separatedByCommas(row.valueExpressions)),
-          ),
-        ),
-      ] as Expression,
-      coerceTypes,
-    );
-  }
 }