From fe7bf4168be2b0e786cfb40ea0d05d35dbf4253a Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Mon, 6 Jan 2025 15:24:45 -0500 Subject: [PATCH 01/12] Flatten index to remove expensive joins during invalidation --- ...98_schema.sql => 1735832183444_schema.sql} | 27 +- .../tests/unit/index-query-engine-test.ts | 9 - packages/host/tests/unit/index-writer-test.ts | 8 - .../1735832183444_add-boxel-index-working.js | 59 ++++ .../tests/index-query-engine-test.ts | 9 - .../realm-server/tests/index-writer-test.ts | 8 - packages/runtime-common/expression.ts | 22 -- packages/runtime-common/helpers/indexer.ts | 144 ++++++--- packages/runtime-common/index-query-engine.ts | 47 +-- packages/runtime-common/index-writer.ts | 72 ++--- .../tests/index-query-engine-test.ts | 293 ++++-------------- .../runtime-common/tests/index-writer-test.ts | 137 ++------ 12 files changed, 310 insertions(+), 525 deletions(-) rename packages/host/config/schema/{1735668047598_schema.sql => 1735832183444_schema.sql} (67%) create mode 100644 packages/postgres/migrations/1735832183444_add-boxel-index-working.js diff --git a/packages/host/config/schema/1735668047598_schema.sql b/packages/host/config/schema/1735832183444_schema.sql similarity index 67% rename from packages/host/config/schema/1735668047598_schema.sql rename to packages/host/config/schema/1735832183444_schema.sql index 5577b1a87b..f46c27c89c 100644 --- a/packages/host/config/schema/1735668047598_schema.sql +++ b/packages/host/config/schema/1735832183444_schema.sql @@ -23,7 +23,32 @@ fitted_html BLOB, display_names BLOB, resource_created_at, - PRIMARY KEY ( url, realm_version, realm_url ) + PRIMARY KEY ( url, realm_url ) +); + + CREATE TABLE IF NOT EXISTS boxel_index_working ( + url TEXT NOT NULL, + file_alias TEXT NOT NULL, + type TEXT NOT NULL, + realm_version INTEGER NOT NULL, + realm_url TEXT NOT NULL, + pristine_doc BLOB, + search_doc BLOB, + error_doc BLOB, + deps BLOB, + types BLOB, + isolated_html TEXT, + indexed_at, + is_deleted BOOLEAN, + source TEXT, + transpiled_code TEXT, + last_modified, + embedded_html BLOB, + atom_html TEXT, + fitted_html BLOB, + display_names BLOB, + resource_created_at, + PRIMARY KEY ( url, realm_url ) ); CREATE TABLE IF NOT EXISTS realm_meta ( diff --git a/packages/host/tests/unit/index-query-engine-test.ts b/packages/host/tests/unit/index-query-engine-test.ts index 7c276a3c39..4aa84a4677 100644 --- a/packages/host/tests/unit/index-query-engine-test.ts +++ b/packages/host/tests/unit/index-query-engine-test.ts @@ -484,15 +484,6 @@ module('Unit | query', function (hooks) { }); }); - test('can get paginated results that are stable during index mutations', async function (assert) { - await runSharedTest(indexQueryEngineTests, assert, { - indexQueryEngine, - dbAdapter, - loader, - testCards, - }); - }); - test(`can filter using 'gt'`, async function (assert) { await runSharedTest(indexQueryEngineTests, assert, { indexQueryEngine, diff --git a/packages/host/tests/unit/index-writer-test.ts b/packages/host/tests/unit/index-writer-test.ts index e0492c5075..e893a8b3ff 100644 --- a/packages/host/tests/unit/index-writer-test.ts +++ b/packages/host/tests/unit/index-writer-test.ts @@ -48,14 +48,6 @@ module('Unit | index-writer', function (hooks) { }); }); - test('only invalidates latest version of content', async function (assert) { - await runSharedTest(indexWriterTests, assert, { - indexWriter, - indexQueryEngine, - adapter, - }); - }); - test('can update an index entry', async function (assert) { await runSharedTest(indexWriterTests, assert, { indexWriter, diff --git a/packages/postgres/migrations/1735832183444_add-boxel-index-working.js b/packages/postgres/migrations/1735832183444_add-boxel-index-working.js new file mode 100644 index 0000000000..31afcd9999 --- /dev/null +++ b/packages/postgres/migrations/1735832183444_add-boxel-index-working.js @@ -0,0 +1,59 @@ +exports.up = (pgm) => { + pgm.createTable('boxel_index_working', { + url: { type: 'varchar', notNull: true }, + file_alias: { type: 'varchar', notNull: true }, + type: { type: 'varchar', notNull: true }, + realm_version: { type: 'integer', notNull: true }, + realm_url: { type: 'varchar', notNull: true }, + pristine_doc: 'jsonb', + search_doc: 'jsonb', + error_doc: 'jsonb', + deps: 'jsonb', + types: 'jsonb', + isolated_html: 'varchar', + indexed_at: 'bigint', + is_deleted: 'boolean', + source: 'varchar', + transpiled_code: 'varchar', + last_modified: 'bigint', + embedded_html: 'jsonb', + atom_html: 'varchar', + fitted_html: 'jsonb', + display_names: 'jsonb', + resource_created_at: 'bigint', + }); + pgm.sql('ALTER TABLE boxel_index_working SET UNLOGGED'); + pgm.addConstraint('boxel_index_working', 'boxel_index_working_pkey', { + primaryKey: ['url', 'realm_url'], + }); + pgm.createIndex('boxel_index_working', ['realm_version']); + pgm.createIndex('boxel_index_working', ['realm_url']); + pgm.createIndex('boxel_index_working', ['file_alias']); + pgm.createIndex('boxel_index_working', ['resource_created_at']); + pgm.createIndex('boxel_index_working', ['last_modified']); + pgm.createIndex('boxel_index_working', 'type'); + pgm.createIndex('boxel_index_working', ['url', 'realm_version']); + pgm.createIndex('boxel_index_working', 'deps', { method: 'gin' }); + pgm.createIndex('boxel_index_working', 'types', { method: 'gin' }); + pgm.createIndex('boxel_index_working', 'fitted_html', { method: 'gin' }); + pgm.createIndex('boxel_index_working', 'embedded_html', { method: 'gin' }); + pgm.createIndex('boxel_index_working', 'search_doc', { method: 'gin' }); + + pgm.sql('delete from boxel_index'); + pgm.sql('delete from realm_versions;'); + pgm.sql('delete from job_reservations'); + pgm.sql('delete from jobs'); + + pgm.dropConstraint('boxel_index', 'boxel_index_pkey'); + pgm.addConstraint('boxel_index', 'boxel_index_pkey', { + primaryKey: ['url', 'realm_url'], + }); +}; + +exports.down = (pgm) => { + pgm.dropTable('boxel_index_working', { cascade: true }); + pgm.dropConstraint('boxel_index', 'boxel_index_pkey'); + pgm.addConstraint('boxel_index', 'boxel_index_pkey', { + primaryKey: ['url', 'realm_url', 'realm_version'], + }); +}; diff --git a/packages/realm-server/tests/index-query-engine-test.ts b/packages/realm-server/tests/index-query-engine-test.ts index 4f15632200..23ea660a68 100644 --- a/packages/realm-server/tests/index-query-engine-test.ts +++ b/packages/realm-server/tests/index-query-engine-test.ts @@ -424,15 +424,6 @@ module('query', function (hooks) { }); }); - test('can get paginated results that are stable during index mutations', async function (assert) { - await runSharedTest(indexQueryEngineTests, assert, { - indexQueryEngine, - dbAdapter, - loader, - testCards: await makeTestCards(loader), - }); - }); - test(`can filter using 'gt'`, async function (assert) { await runSharedTest(indexQueryEngineTests, assert, { indexQueryEngine, diff --git a/packages/realm-server/tests/index-writer-test.ts b/packages/realm-server/tests/index-writer-test.ts index fe20bb5fa2..cbe5255a65 100644 --- a/packages/realm-server/tests/index-writer-test.ts +++ b/packages/realm-server/tests/index-writer-test.ts @@ -45,14 +45,6 @@ module('index-writer', function (hooks) { }); }); - test('only invalidates latest version of content', async function (assert) { - await runSharedTest(indexWriterTests, assert, { - indexWriter, - indexQueryEngine, - adapter, - }); - }); - test('can update an index entry', async function (assert) { await runSharedTest(indexWriterTests, assert, { indexWriter, diff --git a/packages/runtime-common/expression.ts b/packages/runtime-common/expression.ts index 66d11e29bc..452249003d 100644 --- a/packages/runtime-common/expression.ts +++ b/packages/runtime-common/expression.ts @@ -386,28 +386,6 @@ export function expressionToSql(query: Expression) { }; } -export function realmVersionExpression(opts?: { - useWorkInProgressIndex?: boolean; - withMaxVersion?: number; -}) { - return [ - 'realm_version =', - ...addExplicitParens([ - 'SELECT MAX(i2.realm_version)', - 'FROM boxel_index i2', - 'WHERE i2.url = i.url', - ...(opts?.withMaxVersion - ? ['AND i2.realm_version <=', param(opts?.withMaxVersion)] - : !opts?.useWorkInProgressIndex - ? // if we are not using the work in progress index then we limit the max - // version permitted to the current version for the realm - ['AND i2.realm_version <= r.current_version'] - : // otherwise we choose the highest version in the system - []), - ]), - ] as Expression; -} - function trimBrackets(pathTraveled: string) { return pathTraveled.replace(/\[\]/g, ''); } diff --git a/packages/runtime-common/helpers/indexer.ts b/packages/runtime-common/helpers/indexer.ts index bee5d5b8fd..ee2d7fdd4b 100644 --- a/packages/runtime-common/helpers/indexer.ts +++ b/packages/runtime-common/helpers/indexer.ts @@ -109,6 +109,11 @@ export async function setupIndex( client: DBAdapter, indexRows: TestIndexRow[], ): Promise; +export async function setupIndex( + client: DBAdapter, + versionRows: RealmVersionsTable[], + indexRows: { working: TestIndexRow[]; production: TestIndexRow[] }, +): Promise; export async function setupIndex( client: DBAdapter, versionRows: RealmVersionsTable[], @@ -117,17 +122,109 @@ export async function setupIndex( export async function setupIndex( client: DBAdapter, maybeVersionRows: RealmVersionsTable[] | TestIndexRow[] = [], - indexRows?: TestIndexRow[], + maybeWorkingProductionRows?: + | TestIndexRow[] + | { working: TestIndexRow[]; production: TestIndexRow[] }, ): Promise { let versionRows: RealmVersionsTable[]; - if (!indexRows) { + let workingRows: TestIndexRow[] = []; + let productionRows: TestIndexRow[] = []; + if (!maybeWorkingProductionRows) { versionRows = [{ realm_url: testRealmURL, current_version: 1 }]; - indexRows = maybeVersionRows as TestIndexRow[]; + workingRows = maybeVersionRows as TestIndexRow[]; + productionRows = maybeVersionRows as TestIndexRow[]; } else { versionRows = maybeVersionRows as RealmVersionsTable[]; + if (Array.isArray(maybeWorkingProductionRows)) { + workingRows = maybeWorkingProductionRows as TestIndexRow[]; + productionRows = maybeWorkingProductionRows as TestIndexRow[]; + } else { + workingRows = maybeWorkingProductionRows.working; + productionRows = maybeWorkingProductionRows.production; + } } let now = Date.now(); - let indexedCardsExpressions = await Promise.all( + let workingIndexedCardsExpressions = await indexedCardsExpressions({ + indexRows: workingRows, + now, + client, + }); + let productionIndexedCardsExpressions = await indexedCardsExpressions({ + indexRows: productionRows, + now, + client, + }); + let versionExpressions = versionRows.map((r) => asExpressions(r)); + + if (workingIndexedCardsExpressions.length > 0) { + await query( + client, + [ + `INSERT INTO boxel_index_working`, + ...addExplicitParens( + separatedByCommas(workingIndexedCardsExpressions[0].nameExpressions), + ), + 'VALUES', + ...separatedByCommas( + workingIndexedCardsExpressions.map((row) => + addExplicitParens(separatedByCommas(row.valueExpressions)), + ), + ), + ] as Expression, + coerceTypes, + ); + } + if (productionIndexedCardsExpressions.length > 0) { + await query( + client, + [ + `INSERT INTO boxel_index`, + ...addExplicitParens( + separatedByCommas( + productionIndexedCardsExpressions[0].nameExpressions, + ), + ), + 'VALUES', + ...separatedByCommas( + productionIndexedCardsExpressions.map((row) => + addExplicitParens(separatedByCommas(row.valueExpressions)), + ), + ), + ] as Expression, + coerceTypes, + ); + } + + if (versionExpressions.length > 0) { + await query( + client, + [ + `INSERT INTO realm_versions`, + ...addExplicitParens( + separatedByCommas(versionExpressions[0].nameExpressions), + ), + 'VALUES', + ...separatedByCommas( + versionExpressions.map((row) => + addExplicitParens(separatedByCommas(row.valueExpressions)), + ), + ), + ] as Expression, + coerceTypes, + ); + } +} + +async function indexedCardsExpressions({ + indexRows, + now, + client, +}: { + indexRows: TestIndexRow[]; + now: number; + client: DBAdapter; +}) { + return await Promise.all( indexRows.map(async (r) => { let row: Pick & Partial>; @@ -182,43 +279,4 @@ export async function setupIndex( }); }), ); - let versionExpressions = versionRows.map((r) => asExpressions(r)); - - if (indexedCardsExpressions.length > 0) { - await query( - client, - [ - `INSERT INTO boxel_index`, - ...addExplicitParens( - separatedByCommas(indexedCardsExpressions[0].nameExpressions), - ), - 'VALUES', - ...separatedByCommas( - indexedCardsExpressions.map((row) => - addExplicitParens(separatedByCommas(row.valueExpressions)), - ), - ), - ] as Expression, - coerceTypes, - ); - } - - if (versionExpressions.length > 0) { - await query( - client, - [ - `INSERT INTO realm_versions`, - ...addExplicitParens( - separatedByCommas(versionExpressions[0].nameExpressions), - ), - 'VALUES', - ...separatedByCommas( - versionExpressions.map((row) => - addExplicitParens(separatedByCommas(row.valueExpressions)), - ), - ), - ] as Expression, - coerceTypes, - ); - } } diff --git a/packages/runtime-common/index-query-engine.ts b/packages/runtime-common/index-query-engine.ts index f94e80aaa4..a8e99dabec 100644 --- a/packages/runtime-common/index-query-engine.ts +++ b/packages/runtime-common/index-query-engine.ts @@ -29,7 +29,6 @@ import { fieldArity, tableValuedFunctionsPlaceholder, query, - realmVersionExpression, } from './expression'; import { type Query, @@ -53,7 +52,6 @@ import { RealmMetaTable, type BoxelIndexTable, type CardTypeSummary, - type RealmVersionsTable, } from './index-structure'; import { isScopedCSSRequest } from 'glimmer-scoped-css'; @@ -137,13 +135,9 @@ export interface PrerenderedCard { export interface QueryResultsMeta { // TODO SQLite doesn't let us use cursors in the classic sense so we need to - // keep track of page size and index number--note it is possible for the index - // to mutate between pages. Perhaps consider querying a specific realm version - // (and only cleanup realm versions when making generations) so we can see - // consistent paginated results... + // keep track of page size and index number page: { total: number; - realmVersion: number; }; } @@ -181,15 +175,13 @@ export class IndexQueryEngine { ): Promise { let rows = (await this.#query([ `SELECT i.* - FROM boxel_index as i - INNER JOIN realm_versions r ON i.realm_url = r.realm_url + FROM ${tableFromOpts(opts)} as i WHERE`, ...every([ any([ [`i.url =`, param(url.href)], [`i.file_alias =`, param(url.href)], ]), - realmVersionExpression(opts), any([ ['i.type =', param('module')], ['i.type =', param('error')], @@ -239,15 +231,13 @@ export class IndexQueryEngine { ): Promise { let result = (await this.#query([ `SELECT i.*, embedded_html, fitted_html`, - `FROM boxel_index as i - INNER JOIN realm_versions r ON i.realm_url = r.realm_url + `FROM ${tableFromOpts(opts)} as i WHERE`, ...every([ any([ [`i.url =`, param(url.href)], [`i.file_alias =`, param(url.href)], ]), - realmVersionExpression(opts), any([ ['i.type =', param('instance')], ['i.type =', param('error')], @@ -334,19 +324,9 @@ export class IndexQueryEngine { meta: QueryResultsMeta; results: Partial[]; }> { - let version: number; - if (page?.realmVersion) { - version = page.realmVersion; - } else { - let currentRealmVersion = await this.fetchCurrentRealmVersion(realmURL); - version = opts?.useWorkInProgressIndex - ? currentRealmVersion + 1 - : currentRealmVersion; - } let conditions: CardExpression[] = [ ['i.realm_url = ', param(realmURL.href)], ['is_deleted = FALSE OR is_deleted IS NULL'], - realmVersionExpression({ withMaxVersion: version }), ]; if (opts.includeErrors) { @@ -379,8 +359,7 @@ export class IndexQueryEngine { let everyCondition = every(conditions); let query = [ ...selectClauseExpression, - `FROM boxel_index AS i ${tableValuedFunctionsPlaceholder}`, - `INNER JOIN realm_versions r ON i.realm_url = r.realm_url`, + `FROM ${tableFromOpts(opts)} AS i ${tableValuedFunctionsPlaceholder}`, 'WHERE', ...everyCondition, 'GROUP BY url', @@ -390,7 +369,6 @@ export class IndexQueryEngine { let queryCount = [ 'SELECT COUNT(DISTINCT url) AS total', `FROM boxel_index AS i ${tableValuedFunctionsPlaceholder}`, - `INNER JOIN realm_versions r ON i.realm_url = r.realm_url`, 'WHERE', ...everyCondition, ]; @@ -403,7 +381,7 @@ export class IndexQueryEngine { return { results, meta: { - page: { total: Number(totalResults[0].total), realmVersion: version }, + page: { total: Number(totalResults[0].total) }, }, }; } @@ -562,17 +540,6 @@ export class IndexQueryEngine { return (results[0]?.value ?? []) as unknown as CardTypeSummary[]; } - private async fetchCurrentRealmVersion(realmURL: URL) { - let [{ current_version }] = (await this.#query([ - 'SELECT current_version FROM realm_versions WHERE realm_url =', - param(realmURL.href), - ])) as Pick[]; - if (current_version == null) { - throw new Error(`No current version found for realm ${realmURL.href}`); - } - return current_version; - } - private filterCondition(filter: Filter, onRef: CodeRef): CardExpression { if ('type' in filter) { return this.typeCondition(filter.type); @@ -1130,6 +1097,10 @@ function assertIndexEntrySource(obj: T): Omit< }; } +function tableFromOpts(opts: WIPOptions | undefined) { + return opts?.useWorkInProgressIndex ? 'boxel_index_working' : 'boxel_index'; +} + function assertNever(value: never) { return new Error(`should never happen ${value}`); } diff --git a/packages/runtime-common/index-writer.ts b/packages/runtime-common/index-writer.ts index cd0a378271..287ce7cde9 100644 --- a/packages/runtime-common/index-writer.ts +++ b/packages/runtime-common/index-writer.ts @@ -19,7 +19,6 @@ import { any, query, upsert, - realmVersionExpression, } from './expression'; import { type SerializedError } from './error'; import { type DBAdapter } from './db'; @@ -96,7 +95,6 @@ export class Batch { readonly ready: Promise; #invalidations = new Set(); #dbAdapter: DBAdapter; - private isNewGeneration = false; private declare realmVersion: number; constructor( @@ -122,11 +120,8 @@ export class Batch { let results = (await this.#query([ `SELECT i.url, i.type, i.last_modified FROM boxel_index as i - INNER JOIN realm_versions r ON i.realm_url = r.realm_url WHERE i.realm_url =`, param(this.realmURL.href), - 'AND', - ...realmVersionExpression({ useWorkInProgressIndex: false }), ] as Expression)) as Pick< BoxelIndexTable, 'url' | 'type' | 'last_modified' @@ -222,8 +217,8 @@ export class Batch { await this.#query([ ...upsert( - 'boxel_index', - 'boxel_index_pkey', + 'boxel_index_working', + 'boxel_index_working_pkey', nameExpressions, valueExpressions, ), @@ -231,9 +226,11 @@ export class Batch { } async done(): Promise<{ totalIndexEntries: number }> { + await this.#query(['BEGIN']); await this.updateRealmMeta(); await this.applyBatchUpdates(); await this.pruneObsoleteEntries(); + await this.#query(['COMMIT']); let totalIndexEntries = await this.numberOfIndexEntries(); return { totalIndexEntries }; @@ -247,14 +244,12 @@ export class Batch { let [entry] = (await this.#query([ `SELECT i.*`, `FROM boxel_index as i - INNER JOIN realm_versions r ON i.realm_url = r.realm_url WHERE`, ...every([ any([ [`i.url =`, param(url.href)], [`i.file_alias =`, param(url.href)], ]), - realmVersionExpression(), ]), ] as Expression)) as unknown as BoxelIndexTable[]; if (!entry) { @@ -281,13 +276,11 @@ export class Batch { let [{ total }] = (await this.#query([ `SELECT count(i.url) as total FROM boxel_index as i - INNER JOIN realm_versions r ON i.realm_url = r.realm_url WHERE`, ...every([ ['i.realm_url =', param(this.realmURL.href)], ['i.type != ', param('error')], ['i.is_deleted != true'], - realmVersionExpression({ useWorkInProgressIndex: true }), ]), ] as Expression)) as { total: string }[]; return parseInt(total); @@ -296,17 +289,12 @@ export class Batch { private async updateRealmMeta() { let results = await this.#query([ `SELECT CAST(count(i.url) AS INTEGER) as total, i.display_names->>0 as display_name, i.types->>0 as code_ref - FROM boxel_index as i - INNER JOIN realm_versions r ON i.realm_url = r.realm_url + FROM boxel_index_working as i WHERE`, ...every([ ['i.realm_url =', param(this.realmURL.href)], ['i.type = ', param('instance')], ['i.types IS NOT NULL'], - realmVersionExpression({ - useWorkInProgressIndex: true, - withMaxVersion: this.realmVersion, - }), ]), `GROUP BY i.display_names->>0, i.types->>0`, ] as Expression); @@ -348,6 +336,24 @@ export class Batch { valueExpressions, ), ]); + + let columns = (await this.#dbAdapter.getColumnNames('boxel_index')).map( + (c) => [c], + ); + let names = flattenDeep(columns); + await this.#query([ + 'INSERT INTO boxel_index', + ...addExplicitParens(separatedByCommas(columns)), + 'SELECT', + ...separatedByCommas(columns), + 'FROM boxel_index_working', + 'WHERE url in', + ...addExplicitParens( + separatedByCommas([...this.#invalidations].map((i) => [param(i)])), + ), + 'ON CONFLICT ON CONSTRAINT boxel_index_pkey DO UPDATE SET', + ...separatedByCommas(names.map((name) => [`${name}=EXCLUDED.${name}`])), + ] as Expression); } private async pruneObsoleteEntries() { @@ -359,17 +365,6 @@ export class Batch { ['realm_url =', param(this.realmURL.href)], ]), ] as Expression); - - if (this.isNewGeneration) { - await this.#query([ - `DELETE FROM boxel_index`, - 'WHERE', - ...every([ - ['realm_version <', param(this.realmVersion)], - ['realm_url =', param(this.realmURL.href)], - ]), - ] as Expression); - } } private async setNextRealmVersion() { @@ -434,13 +429,13 @@ export class Batch { let names = flattenDeep(columns); await this.#query([ - 'INSERT INTO boxel_index', + 'INSERT INTO boxel_index_working', ...addExplicitParens(separatedByCommas(columns)), 'VALUES', ...separatedByCommas( rows.map((value) => addExplicitParens(separatedByCommas(value))), ), - 'ON CONFLICT ON CONSTRAINT boxel_index_pkey DO UPDATE SET', + 'ON CONFLICT ON CONSTRAINT boxel_index_working_pkey DO UPDATE SET', ...separatedByCommas(names.map((name) => [`${name}=EXCLUDED.${name}`])), ] as Expression); @@ -450,7 +445,6 @@ export class Batch { private async itemsThatReference( resolvedPath: string, - realmVersion: number, ): Promise< { url: string; alias: string; type: 'instance' | 'module' | 'error' }[] > { @@ -469,13 +463,18 @@ export class Batch { // realm_version to keep the result set stable across pages rows = (await this.#query([ 'SELECT i.url, i.file_alias, i.type', - 'FROM boxel_index as i', + 'FROM boxel_index_working as i', + // TODO: If this query is still slow we'll need to consider dropping + // this join and writing a pg specific query here--as this query is + // currently written in a less performant manner in order to be + // compatible with SQLite. make sure to measure performance in hosted + // env first before sending for team review. Should strive to be less + // than 50ms (currently about 600-700ms) when this is run against a + // reasonably sized index. 'CROSS JOIN LATERAL jsonb_array_elements_text(i.deps) as deps_array_element', - 'INNER JOIN realm_versions r ON i.realm_url = r.realm_url', 'WHERE', ...every([ [`deps_array_element =`, param(resolvedPath)], - realmVersionExpression({ withMaxVersion: realmVersion }), // css is a subset of modules, so there won't by any references that // are css entries that aren't already represented by a module entry [`i.type != 'css'`], @@ -506,10 +505,7 @@ export class Batch { return []; } visited.add(resolvedPath); - let childInvalidations = await this.itemsThatReference( - resolvedPath, - this.realmVersion, - ); + let childInvalidations = await this.itemsThatReference(resolvedPath); let realmPath = new RealmPaths(this.realmURL); let invalidationsInThisRealm = childInvalidations.filter((c) => realmPath.inRealm(new URL(c.url)), diff --git a/packages/runtime-common/tests/index-query-engine-test.ts b/packages/runtime-common/tests/index-query-engine-test.ts index 6f146a79ad..a107f9fb5d 100644 --- a/packages/runtime-common/tests/index-query-engine-test.ts +++ b/packages/runtime-common/tests/index-query-engine-test.ts @@ -8,12 +8,11 @@ import { type Loader, type ResolvedCodeRef, DBAdapter, - IndexWriter, } from '../index'; import { serializeCard } from '../helpers/indexer'; import { testRealmURL } from '../helpers/const'; import { type SharedTests } from '../helpers'; -import { type TestIndexRow, setupIndex, getTypes } from '../helpers/indexer'; +import { setupIndex, getTypes } from '../helpers/indexer'; import { type CardDef } from 'https://cardstack.com/base/card-api'; import { cardSrc } from '../etc/test-fixtures'; @@ -1356,28 +1355,36 @@ const tests = Object.freeze({ await setupIndex( dbAdapter, [{ realm_url: testRealmURL, current_version: 1 }], - [ - { - card: mango, - data: { realm_version: 1, search_doc: { name: 'Mango' } }, - }, - { - card: vangogh, - data: { realm_version: 1, search_doc: { name: 'Van Gogh' } }, - }, - { - card: vangogh, - data: { realm_version: 2, search_doc: { name: 'Mango' } }, - }, - { - card: ringo, - data: { realm_version: 1, search_doc: { name: 'Mango' } }, - }, - { - card: ringo, - data: { realm_version: 2, search_doc: { name: 'Ringo' } }, - }, - ], + { + working: [ + { + card: mango, + data: { realm_version: 1, search_doc: { name: 'Mango' } }, + }, + { + card: vangogh, + data: { realm_version: 2, search_doc: { name: 'Mango' } }, + }, + { + card: ringo, + data: { realm_version: 2, search_doc: { name: 'Ringo' } }, + }, + ], + production: [ + { + card: mango, + data: { realm_version: 1, search_doc: { name: 'Mango' } }, + }, + { + card: vangogh, + data: { realm_version: 1, search_doc: { name: 'Van Gogh' } }, + }, + { + card: ringo, + data: { realm_version: 1, search_doc: { name: 'Mango' } }, + }, + ], + }, ); let type = await personCardType(testCards); @@ -1394,11 +1401,6 @@ const tests = Object.freeze({ ); assert.strictEqual(meta.page.total, 2, 'the total results meta is correct'); - assert.strictEqual( - meta.page.realmVersion, - 2, - 'the realm version queried is correct', - ); assert.deepEqual( getIds(results), [mango.id, vangogh.id], @@ -1414,24 +1416,36 @@ const tests = Object.freeze({ await setupIndex( dbAdapter, [{ realm_url: testRealmURL, current_version: 1 }], - [ - { - card: mango, - data: { realm_version: 1, search_doc: { name: 'Mango' } }, - }, - { - card: vangogh, - data: { realm_version: 1, search_doc: { name: 'Van Gogh' } }, - }, - { - card: vangogh, - data: { realm_version: 2, search_doc: { name: 'Mango' } }, - }, - { - card: ringo, - data: { realm_version: 1, search_doc: { name: 'Ringo' } }, - }, - ], + { + working: [ + { + card: mango, + data: { realm_version: 1, search_doc: { name: 'Mango' } }, + }, + { + card: vangogh, + data: { realm_version: 2, search_doc: { name: 'Mango' } }, + }, + { + card: ringo, + data: { realm_version: 1, search_doc: { name: 'Ringo' } }, + }, + ], + production: [ + { + card: mango, + data: { realm_version: 1, search_doc: { name: 'Mango' } }, + }, + { + card: vangogh, + data: { realm_version: 1, search_doc: { name: 'Van Gogh' } }, + }, + { + card: ringo, + data: { realm_version: 1, search_doc: { name: 'Ringo' } }, + }, + ], + }, ); let type = await personCardType(testCards); @@ -1447,11 +1461,6 @@ const tests = Object.freeze({ ); assert.strictEqual(meta.page.total, 1, 'the total results meta is correct'); - assert.strictEqual( - meta.page.realmVersion, - 1, - 'the realm version queried is correct', - ); assert.deepEqual(getIds(results), [mango.id], 'results are correct'); }, @@ -1650,179 +1659,6 @@ const tests = Object.freeze({ } }, - 'can get paginated results that are stable during index mutations': async ( - assert, - { indexQueryEngine, dbAdapter, loader, testCards }, - ) => { - let { mango } = testCards; - let Card = mango.constructor as typeof CardDef; - let testData: TestIndexRow[] = []; - for (let i = 0; i < 10; i++) { - testData.push({ - card: new Card({ id: `${testRealmURL}mango${i}` }), - data: { search_doc: { name: `Mango-${i}` } }, - }); - } - - await setupIndex(dbAdapter, testData); - - // page 1 - let type = await personCardType(testCards); - let { cards: results, meta } = await indexQueryEngine.search( - new URL(testRealmURL), - { - page: { number: 0, size: 3 }, - sort: [ - { - on: type, - by: 'name', - direction: 'desc', - }, - ], - filter: { - on: type, - contains: { name: 'Mango' }, - }, - }, - loader, - ); - - let { - page: { total, realmVersion }, - } = meta; - assert.strictEqual(total, 10, 'the total results meta is correct'); - assert.strictEqual(realmVersion, 1, 'the query realm version is correct'); - assert.deepEqual(getIds(results), [ - `${testRealmURL}mango9`, - `${testRealmURL}mango8`, - `${testRealmURL}mango7`, - ]); - - { - // page 2 - let { cards: results, meta } = await indexQueryEngine.search( - new URL(testRealmURL), - { - // providing the realm version received from the 1st page's meta keeps - // the result set stable while we page over it - page: { number: 1, size: 3, realmVersion }, - sort: [ - { - on: type, - by: 'name', - direction: 'desc', - }, - ], - filter: { - on: type, - contains: { name: 'Mango' }, - }, - }, - loader, - ); - assert.strictEqual( - meta.page.total, - 10, - 'the total results meta is correct', - ); - assert.strictEqual( - meta.page.realmVersion, - 1, - 'the query realm version is correct', - ); - assert.deepEqual(getIds(results), [ - `${testRealmURL}mango6`, - `${testRealmURL}mango5`, - `${testRealmURL}mango4`, - ]); - } - - // mutate the index - let batch = await new IndexWriter(dbAdapter).createBatch( - new URL(testRealmURL), - ); - await batch.invalidate(new URL(`${testRealmURL}mango3.json`)); - await batch.done(); - - { - // page 3 - let { cards: results, meta } = await indexQueryEngine.search( - new URL(testRealmURL), - { - // providing the realm version received from the 1st page's meta keeps - // the result set stable while we page over it - page: { number: 2, size: 3, realmVersion }, - sort: [ - { - on: type, - by: 'name', - direction: 'desc', - }, - ], - filter: { - on: type, - contains: { name: 'Mango' }, - }, - }, - loader, - ); - assert.strictEqual( - meta.page.total, - 10, - 'the total results meta is correct', - ); - assert.strictEqual( - meta.page.realmVersion, - 1, - 'the query realm version is correct', - ); - assert.deepEqual(getIds(results), [ - `${testRealmURL}mango3`, // this is actually removed in the current index - `${testRealmURL}mango2`, - `${testRealmURL}mango1`, - ]); - } - - // assert that a new search against the current index no longer contains the - // removed card - { - let { cards: results, meta } = await indexQueryEngine.search( - new URL(testRealmURL), - { - sort: [ - { - on: type, - by: 'name', - direction: 'desc', - }, - ], - filter: { - on: type, - contains: { name: 'Mango' }, - }, - }, - loader, - ); - - let { - page: { total, realmVersion }, - } = meta; - assert.strictEqual(total, 9, 'the total results meta is correct'); - assert.strictEqual(realmVersion, 2, 'the query realm version is correct'); - assert.deepEqual(getIds(results), [ - `${testRealmURL}mango9`, - `${testRealmURL}mango8`, - `${testRealmURL}mango7`, - `${testRealmURL}mango6`, - `${testRealmURL}mango5`, - `${testRealmURL}mango4`, - `${testRealmURL}mango2`, - `${testRealmURL}mango1`, - `${testRealmURL}mango0`, - ]); - } - }, - "can filter using 'gt'": async ( assert, { indexQueryEngine, dbAdapter, loader, testCards }, @@ -2723,7 +2559,6 @@ const tests = Object.freeze({ 3, 'meta total results meta is correct', ); - assert.strictEqual(meta.page.realmVersion, 1, 'realm version is correct'); assert.strictEqual( prerenderedCards.length, 3, @@ -2816,12 +2651,6 @@ const tests = Object.freeze({ )); assert.strictEqual(meta.page.total, 1, 'the total results meta is correct'); - assert.strictEqual( - meta.page.realmVersion, - 1, - 'the realm version is correct', - ); - assert.strictEqual(prerenderedCards[0].url, `${testRealmURL}donald.json`); assert.strictEqual(prerenderedCards[0].html, 'Donald'); // Atom template }, diff --git a/packages/runtime-common/tests/index-writer-test.ts b/packages/runtime-common/tests/index-writer-test.ts index 685c712038..cb47ee5dfb 100644 --- a/packages/runtime-common/tests/index-writer-test.ts +++ b/packages/runtime-common/tests/index-writer-test.ts @@ -82,21 +82,8 @@ const tests = Object.freeze({ `${testRealmURL}4.json`, ]); - let originalEntries = await adapter.execute( - 'SELECT url, realm_url, is_deleted FROM boxel_index WHERE realm_version = 1 ORDER BY url COLLATE "POSIX"', - { coerceTypes: { is_deleted: 'BOOLEAN' } }, - ); - assert.deepEqual( - originalEntries, - [1, 2, 3, 4, 5].map((i) => ({ - url: `${testRealmURL}${i}.json`, - realm_url: testRealmURL, - is_deleted: null, - })), - 'the "production" version of the index entries are unchanged', - ); let invalidatedEntries = await adapter.execute( - 'SELECT url, realm_url, is_deleted FROM boxel_index WHERE realm_version = 2 ORDER BY url COLLATE "POSIX"', + 'SELECT url, realm_url, is_deleted FROM boxel_index_working WHERE realm_version = 2 ORDER BY url COLLATE "POSIX"', { coerceTypes: { is_deleted: 'BOOLEAN' } }, ); assert.deepEqual( @@ -109,7 +96,7 @@ const tests = Object.freeze({ 'the "work-in-progress" version of the index entries have been marked as deleted', ); let otherRealms = await adapter.execute( - `SELECT url, realm_url, realm_version, is_deleted FROM boxel_index WHERE realm_url != '${testRealmURL}'`, + `SELECT url, realm_url, realm_version, is_deleted FROM boxel_index_working WHERE realm_url != '${testRealmURL}'`, { coerceTypes: { is_deleted: 'BOOLEAN' } }, ); assert.deepEqual( @@ -249,77 +236,6 @@ const tests = Object.freeze({ assert.deepEqual(invalidations, [`${testRealmURL}person.gts`]); }, - 'only invalidates latest version of content': async ( - assert, - { indexWriter, adapter }, - ) => { - await setupIndex( - adapter, - [{ realm_url: testRealmURL, current_version: 2 }], - [ - { - url: `${testRealmURL}1.json`, - realm_version: 1, - realm_url: testRealmURL, - deps: [`${testRealmURL}2.json`], - }, - { - url: `${testRealmURL}2.json`, - realm_version: 1, - realm_url: testRealmURL, - deps: [`${testRealmURL}4.json`], - }, - { - url: `${testRealmURL}2.json`, - realm_version: 2, - realm_url: testRealmURL, - deps: [], - }, - { - url: `${testRealmURL}3.json`, - realm_version: 1, - realm_url: testRealmURL, - deps: [`${testRealmURL}2.json`], - }, - { - url: `${testRealmURL}4.json`, - realm_version: 1, - realm_url: testRealmURL, - deps: [], - }, - { - url: `${testRealmURL}5.json`, - realm_version: 1, - realm_url: testRealmURL, - deps: [`${testRealmURL}4.json`], - }, - ], - ); - - let batch = await indexWriter.createBatch(new URL(testRealmURL)); - let invalidations = await batch.invalidate( - new URL(`${testRealmURL}4.json`), - ); - - assert.deepEqual(invalidations.sort(), [ - `${testRealmURL}4.json`, - `${testRealmURL}5.json`, - ]); - let invalidatedEntries = await adapter.execute( - 'SELECT url, realm_url, is_deleted FROM boxel_index WHERE realm_version = 3 ORDER BY url COLLATE "POSIX"', - { coerceTypes: { is_deleted: 'BOOLEAN' } }, - ); - assert.deepEqual( - invalidatedEntries, - [4, 5].map((i) => ({ - url: `${testRealmURL}${i}.json`, - realm_url: testRealmURL, - is_deleted: true, - })), - 'the "work-in-progress" version of the index entries have been marked as deleted', - ); - }, - 'can update an index entry': async (assert, { indexWriter, adapter }) => { await setupIndex( adapter, @@ -382,8 +298,8 @@ const tests = Object.freeze({ ].map((i) => internalKeyFor(i, new URL(testRealmURL))), }); - let versions = await adapter.execute( - `SELECT realm_version, pristine_doc, search_doc, deps, types FROM boxel_index WHERE url = $1 ORDER BY realm_version`, + let [liveVersion] = await adapter.execute( + `SELECT realm_version, pristine_doc, search_doc, deps, types FROM boxel_index WHERE url = $1`, { bind: [`${testRealmURL}1.json`], coerceTypes: { @@ -394,13 +310,7 @@ const tests = Object.freeze({ }, }, ); - assert.strictEqual( - versions.length, - 2, - 'correct number of versions exist for the entry before finishing the batch', - ); - let [liveVersion, wipVersion] = versions; assert.deepEqual( liveVersion, { @@ -426,6 +336,19 @@ const tests = Object.freeze({ }, 'live version of the doc has not changed', ); + + let [wipVersion] = await adapter.execute( + `SELECT realm_version, pristine_doc, search_doc, deps, types FROM boxel_index_working WHERE url = $1`, + { + bind: [`${testRealmURL}1.json`], + coerceTypes: { + pristine_doc: 'JSON', + search_doc: 'JSON', + deps: 'JSON', + types: 'JSON', + }, + }, + ); assert.deepEqual( wipVersion, { @@ -456,8 +379,8 @@ const tests = Object.freeze({ await batch.done(); - versions = await adapter.execute( - `SELECT realm_version, pristine_doc, search_doc, deps, types FROM boxel_index WHERE url = $1 ORDER BY realm_version`, + let [finalVersion] = await adapter.execute( + `SELECT realm_version, pristine_doc, search_doc, deps, types FROM boxel_index WHERE url = $1`, { bind: [`${testRealmURL}1.json`], coerceTypes: { @@ -468,13 +391,6 @@ const tests = Object.freeze({ }, }, ); - assert.strictEqual( - versions.length, - 2, - 'correct number of versions exist for the entry after finishing the batch', - ); - - let [_, finalVersion] = versions; assert.deepEqual( finalVersion, { @@ -965,21 +881,8 @@ const tests = Object.freeze({ indexRows.map((r) => r.url), ); - let originalEntries = (await adapter.execute( - 'SELECT url, realm_url, is_deleted FROM boxel_index WHERE realm_version = 1 ORDER BY url COLLATE "POSIX"', - { coerceTypes: { is_deleted: 'BOOLEAN' } }, - )) as Pick[]; - assert.deepEqual( - originalEntries, - indexRows.map((indexRow) => ({ - url: indexRow.url, - realm_url: indexRow.realm_url, - is_deleted: null, - })) as Pick[], - 'the "production" version of the index entries are unchanged', - ); let invalidatedEntries = (await adapter.execute( - 'SELECT url, realm_url, is_deleted FROM boxel_index WHERE realm_version = 2 ORDER BY url COLLATE "POSIX"', + 'SELECT url, realm_url, is_deleted FROM boxel_index_working WHERE realm_version = 2 ORDER BY url COLLATE "POSIX"', { coerceTypes: { is_deleted: 'BOOLEAN' } }, )) as Pick[]; assert.deepEqual( From 59ff11e1f81a5653a9ce7c3daafa6afdbc5417b7 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Mon, 6 Jan 2025 15:50:14 -0500 Subject: [PATCH 02/12] when no invalidations after batch is done then skip writing to prod table --- packages/runtime-common/index-writer.ts | 36 +++++++++++++------------ 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/packages/runtime-common/index-writer.ts b/packages/runtime-common/index-writer.ts index 287ce7cde9..507d78399e 100644 --- a/packages/runtime-common/index-writer.ts +++ b/packages/runtime-common/index-writer.ts @@ -337,23 +337,25 @@ export class Batch { ), ]); - let columns = (await this.#dbAdapter.getColumnNames('boxel_index')).map( - (c) => [c], - ); - let names = flattenDeep(columns); - await this.#query([ - 'INSERT INTO boxel_index', - ...addExplicitParens(separatedByCommas(columns)), - 'SELECT', - ...separatedByCommas(columns), - 'FROM boxel_index_working', - 'WHERE url in', - ...addExplicitParens( - separatedByCommas([...this.#invalidations].map((i) => [param(i)])), - ), - 'ON CONFLICT ON CONSTRAINT boxel_index_pkey DO UPDATE SET', - ...separatedByCommas(names.map((name) => [`${name}=EXCLUDED.${name}`])), - ] as Expression); + if (this.#invalidations.size > 0) { + let columns = (await this.#dbAdapter.getColumnNames('boxel_index')).map( + (c) => [c], + ); + let names = flattenDeep(columns); + await this.#query([ + 'INSERT INTO boxel_index', + ...addExplicitParens(separatedByCommas(columns)), + 'SELECT', + ...separatedByCommas(columns), + 'FROM boxel_index_working', + 'WHERE url in', + ...addExplicitParens( + separatedByCommas([...this.#invalidations].map((i) => [param(i)])), + ), + 'ON CONFLICT ON CONSTRAINT boxel_index_pkey DO UPDATE SET', + ...separatedByCommas(names.map((name) => [`${name}=EXCLUDED.${name}`])), + ] as Expression); + } } private async pruneObsoleteEntries() { From bbbaa5728ae75344c04a4d6039acafd09ed02f57 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Mon, 6 Jan 2025 16:04:02 -0500 Subject: [PATCH 03/12] remove error doc when index entry has recovered from error --- packages/runtime-common/index-writer.ts | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/packages/runtime-common/index-writer.ts b/packages/runtime-common/index-writer.ts index 507d78399e..ead7a489f2 100644 --- a/packages/runtime-common/index-writer.ts +++ b/packages/runtime-common/index-writer.ts @@ -169,6 +169,7 @@ export class Batch { source: entry.source, last_modified: entry.lastModified, resource_created_at: entry.resourceCreatedAt, + error_doc: null, } : entry.type === 'module' ? { @@ -181,6 +182,7 @@ export class Batch { entry.source, new RealmPaths(this.realmURL).local(url), ), + error_doc: null, } : { types: entry.types, @@ -337,6 +339,7 @@ export class Batch { ), ]); + let working = await this.#query(['select * from boxel_index_working']); if (this.#invalidations.size > 0) { let columns = (await this.#dbAdapter.getColumnNames('boxel_index')).map( (c) => [c], @@ -348,14 +351,24 @@ export class Batch { 'SELECT', ...separatedByCommas(columns), 'FROM boxel_index_working', - 'WHERE url in', - ...addExplicitParens( - separatedByCommas([...this.#invalidations].map((i) => [param(i)])), - ), + 'WHERE', + ...every([ + ['realm_url =', param(this.realmURL.href)], + [ + 'url in', + ...addExplicitParens( + separatedByCommas( + [...this.#invalidations].map((i) => [param(i)]), + ), + ), + ], + ]), 'ON CONFLICT ON CONSTRAINT boxel_index_pkey DO UPDATE SET', ...separatedByCommas(names.map((name) => [`${name}=EXCLUDED.${name}`])), ] as Expression); } + let production = await this.#query(['select * from boxel_index']); + let foo = 'bar'; } private async pruneObsoleteEntries() { From 0c5765daf2c0f46d1d2d606456a0494ea4d034d1 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Mon, 6 Jan 2025 16:10:55 -0500 Subject: [PATCH 04/12] cleanup debug --- packages/runtime-common/index-writer.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/runtime-common/index-writer.ts b/packages/runtime-common/index-writer.ts index ead7a489f2..ac91d0ee03 100644 --- a/packages/runtime-common/index-writer.ts +++ b/packages/runtime-common/index-writer.ts @@ -339,7 +339,6 @@ export class Batch { ), ]); - let working = await this.#query(['select * from boxel_index_working']); if (this.#invalidations.size > 0) { let columns = (await this.#dbAdapter.getColumnNames('boxel_index')).map( (c) => [c], @@ -367,8 +366,6 @@ export class Batch { ...separatedByCommas(names.map((name) => [`${name}=EXCLUDED.${name}`])), ] as Expression); } - let production = await this.#query(['select * from boxel_index']); - let foo = 'bar'; } private async pruneObsoleteEntries() { From 67e46bc3f66df555941755c8322e989621f5112d Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Mon, 6 Jan 2025 16:40:31 -0500 Subject: [PATCH 05/12] clean up realm meta query --- packages/runtime-common/index-query-engine.ts | 2 -- packages/runtime-common/index-writer.ts | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/runtime-common/index-query-engine.ts b/packages/runtime-common/index-query-engine.ts index a8e99dabec..44b993ae01 100644 --- a/packages/runtime-common/index-query-engine.ts +++ b/packages/runtime-common/index-query-engine.ts @@ -531,8 +531,6 @@ export class IndexQueryEngine { let results = (await this.#query([ `SELECT value FROM realm_meta rm - INNER JOIN realm_versions rv - ON rm.realm_url = rv.realm_url AND rm.realm_version = rv.current_version WHERE`, ...every([['rm.realm_url =', param(realmURL.href)]]), ] as Expression)) as Pick[]; diff --git a/packages/runtime-common/index-writer.ts b/packages/runtime-common/index-writer.ts index ac91d0ee03..5ef3ba3cf2 100644 --- a/packages/runtime-common/index-writer.ts +++ b/packages/runtime-common/index-writer.ts @@ -297,6 +297,7 @@ export class Batch { ['i.realm_url =', param(this.realmURL.href)], ['i.type = ', param('instance')], ['i.types IS NOT NULL'], + ['i.is_deleted != true'], ]), `GROUP BY i.display_names->>0, i.types->>0`, ] as Expression); From 9afc6b34bb505b0cd6598ce4e3e208690cf727c4 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Mon, 6 Jan 2025 17:38:12 -0500 Subject: [PATCH 06/12] better realm_meta test --- packages/runtime-common/index-writer.ts | 2 +- packages/runtime-common/tests/index-writer-test.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/runtime-common/index-writer.ts b/packages/runtime-common/index-writer.ts index 5ef3ba3cf2..01239731b1 100644 --- a/packages/runtime-common/index-writer.ts +++ b/packages/runtime-common/index-writer.ts @@ -297,7 +297,7 @@ export class Batch { ['i.realm_url =', param(this.realmURL.href)], ['i.type = ', param('instance')], ['i.types IS NOT NULL'], - ['i.is_deleted != true'], + any([['i.is_deleted = false'], ['i.is_deleted IS NULL']]), ]), `GROUP BY i.display_names->>0, i.types->>0`, ] as Expression); diff --git a/packages/runtime-common/tests/index-writer-test.ts b/packages/runtime-common/tests/index-writer-test.ts index cb47ee5dfb..748605e0ad 100644 --- a/packages/runtime-common/tests/index-writer-test.ts +++ b/packages/runtime-common/tests/index-writer-test.ts @@ -1136,7 +1136,7 @@ const tests = Object.freeze({ }); let results = await adapter.execute( - `SELECT value FROM realm_meta r INNER JOIN realm_versions rv ON r.realm_url = rv.realm_url WHERE r.realm_url = $1 AND r.realm_version = 1`, + `SELECT value FROM realm_meta r WHERE r.realm_url = $1`, { bind: [testRealmURL], coerceTypes: { @@ -1153,7 +1153,7 @@ const tests = Object.freeze({ await batch.done(); results = await adapter.execute( - `SELECT value FROM realm_meta r INNER JOIN realm_versions rv ON r.realm_url = rv.realm_url WHERE r.realm_url = $1`, + `SELECT value FROM realm_meta r WHERE r.realm_url = $1`, { bind: [testRealmURL], coerceTypes: { @@ -1254,7 +1254,7 @@ const tests = Object.freeze({ await batch.done(); results = await adapter.execute( - `SELECT value FROM realm_meta r INNER JOIN realm_versions rv ON r.realm_url = rv.realm_url WHERE r.realm_url = $1`, + `SELECT value FROM realm_meta r WHERE r.realm_url = $1`, { bind: [testRealmURL], coerceTypes: { From 2fb34f18a4a0afa5749986a8311f105c8cc8bccf Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Tue, 7 Jan 2025 12:49:21 -0500 Subject: [PATCH 07/12] Use more optimized pg expression in invalidation --- packages/host/app/lib/sqlite-adapter.ts | 1 + packages/postgres/pg-adapter.ts | 3 +- packages/runtime-common/db.ts | 1 + packages/runtime-common/expression.ts | 105 +++++++++++++++--- packages/runtime-common/index-query-engine.ts | 2 + packages/runtime-common/index-writer.ts | 24 ++-- 6 files changed, 109 insertions(+), 27 deletions(-) diff --git a/packages/host/app/lib/sqlite-adapter.ts b/packages/host/app/lib/sqlite-adapter.ts index 4f1d88ba74..aaf81bdd8f 100644 --- a/packages/host/app/lib/sqlite-adapter.ts +++ b/packages/host/app/lib/sqlite-adapter.ts @@ -13,6 +13,7 @@ import { } from '@cardstack/runtime-common'; export default class SQLiteAdapter implements DBAdapter { + readonly kind = 'sqlite'; private _sqlite: typeof SQLiteWorker | undefined; private _dbId: string | undefined; private primaryKeys = new Map(); diff --git a/packages/postgres/pg-adapter.ts b/packages/postgres/pg-adapter.ts index 819e9d238f..a012d7609a 100644 --- a/packages/postgres/pg-adapter.ts +++ b/packages/postgres/pg-adapter.ts @@ -23,6 +23,7 @@ function config() { type Config = ReturnType; export class PgAdapter implements DBAdapter { + readonly kind = 'pg'; #isClosed = false; private pool: Pool; private started: Promise; @@ -124,7 +125,7 @@ export class PgAdapter implements DBAdapter { let client = await this.pool.connect(); let query = async (expression: Expression) => { - let sql = expressionToSql(expression); + let sql = expressionToSql(this.kind, expression); log.debug('search: %s trace: %j', sql.text, sql.values); let { rows } = await client.query(sql); return rows; diff --git a/packages/runtime-common/db.ts b/packages/runtime-common/db.ts index 7bdc19f65e..26706481da 100644 --- a/packages/runtime-common/db.ts +++ b/packages/runtime-common/db.ts @@ -12,6 +12,7 @@ export interface ExecuteOptions { } export interface DBAdapter { + kind: 'pg' | 'sqlite'; isClosed: boolean; execute: ( sql: string, diff --git a/packages/runtime-common/expression.ts b/packages/runtime-common/expression.ts index 452249003d..a308d5efe1 100644 --- a/packages/runtime-common/expression.ts +++ b/packages/runtime-common/expression.ts @@ -5,7 +5,13 @@ import flattenDeep from 'lodash/flattenDeep'; import { type CodeRef, type DBAdapter, type TypeCoercion } from './index'; -export type Expression = (string | Param | TableValuedEach | TableValuedTree)[]; +export type Expression = ( + | string + | Param + | TableValuedEach + | TableValuedTree + | DBSpecificExpression +)[]; export type PgPrimitive = | number @@ -16,10 +22,18 @@ export type PgPrimitive = | null; export interface Param { - param: PgPrimitive; + param?: PgPrimitive; + pg?: PgPrimitive; + sqlite?: PgPrimitive; kind: 'param'; } +export interface DBSpecificExpression { + pg?: PgPrimitive; + sqlite?: PgPrimitive; + kind: 'db-specific-expression'; +} + export interface FieldQuery { type: CodeRef; path: string; @@ -62,6 +76,7 @@ export interface FieldArity { export type CardExpression = ( | string | Param + | DBSpecificExpression | TableValuedEach | TableValuedTree | FieldQuery @@ -92,12 +107,50 @@ export function separatedByCommas(expressions: unknown[][]): unknown { }, []); } -export function param(value: PgPrimitive): Param { +export function param(value: { pg?: PgPrimitive; sqlite?: PgPrimitive }): Param; +export function param(value: PgPrimitive): Param; +export function param( + value: PgPrimitive | { pg?: PgPrimitive; sqlite?: PgPrimitive }, +): Param { + if ( + value && + typeof value === 'object' && + ('pg' in value || 'sqlite' in value) + ) { + return { + ...value, + kind: 'param', + }; + } return { param: value, kind: 'param' }; } export function isParam(expression: any): expression is Param { - return isPlainObject(expression) && 'param' in expression; + return ( + isPlainObject(expression) && + 'kind' in expression && + expression.kind === 'param' + ); +} + +export function dbExpression({ + pg, + sqlite, +}: { + pg?: PgPrimitive; + sqlite?: PgPrimitive; +}): DBSpecificExpression { + return { pg, sqlite, kind: 'db-specific-expression' }; +} + +export function isDbExpression( + expression: any, +): expression is DBSpecificExpression { + return ( + isPlainObject(expression) && + 'kind' in expression && + expression.kind === 'db-specific-expression' + ); } export function tableValuedEach(column: string): TableValuedEach { @@ -226,12 +279,9 @@ export function asExpressions( let paramBucket = Object.fromEntries( Object.entries(values).map(([col, val]) => [ col, - // TODO: SQLite requires JSON be referenced in a stringified - // manner--need to confirm if postgres is ok with this - - // TODO probably we should insert using the json() or jsonb() function in - // SQLite, need to check for compatibility in postgres for this function - param(opts?.jsonFields?.includes(col) ? stringify(val) : val), + param( + opts?.jsonFields?.includes(col) ? stringify(val ?? null) : val ?? null, + ), ]), ); let nameExpressions = Object.keys(paramBucket).map((name) => [name]); @@ -309,14 +359,17 @@ export async function query( query: Expression, coerceTypes?: TypeCoercion, ) { - let sql = await expressionToSql(query); + let sql = await expressionToSql(dbAdapter.kind, query); return await dbAdapter.execute(sql.text, { coerceTypes, bind: sql.values, }); } -export function expressionToSql(query: Expression) { +export function expressionToSql( + dbAdapterKind: DBAdapter['kind'], + query: Expression, +) { let values: PgPrimitive[] = []; let nonce = 0; let tableValuedFunctions = new Map< @@ -328,9 +381,31 @@ export function expressionToSql(query: Expression) { >(); let text = query .map((element) => { - if (isParam(element)) { - values.push(element.param); - return `$${values.length}`; + if (isDbExpression(element)) { + if (dbAdapterKind === 'pg') { + return element.pg ?? ''; + } else if (dbAdapterKind === 'sqlite') { + return element.sqlite ?? ''; + } else { + throw assertNever(dbAdapterKind); + } + } else if (isParam(element)) { + if (dbAdapterKind === 'pg' && 'pg' in element && element.pg != null) { + values.push(element.pg); + return `$${values.length}`; + } else if ( + dbAdapterKind === 'sqlite' && + 'sqlite' in element && + element.sqlite != null + ) { + values.push(element.sqlite); + return `$${values.length}`; + } else if ('param' in element && element.param !== undefined) { + values.push(element.param); + return `$${values.length}`; + } else { + throw new Error(`unexpected param value: ${JSON.stringify(element)}`); + } } else if (typeof element === 'string') { return element; } else if (element.kind === 'table-valued-tree') { diff --git a/packages/runtime-common/index-query-engine.ts b/packages/runtime-common/index-query-engine.ts index 44b993ae01..d7a8f2782f 100644 --- a/packages/runtime-common/index-query-engine.ts +++ b/packages/runtime-common/index-query-engine.ts @@ -29,6 +29,7 @@ import { fieldArity, tableValuedFunctionsPlaceholder, query, + isDbExpression, } from './expression'; import { type Query, @@ -715,6 +716,7 @@ export class IndexQueryEngine { query.map((element) => { if ( isParam(element) || + isDbExpression(element) || typeof element === 'string' || element.kind === 'table-valued-each' || element.kind === 'table-valued-tree' diff --git a/packages/runtime-common/index-writer.ts b/packages/runtime-common/index-writer.ts index 01239731b1..80a94f21bc 100644 --- a/packages/runtime-common/index-writer.ts +++ b/packages/runtime-common/index-writer.ts @@ -19,6 +19,7 @@ import { any, query, upsert, + dbExpression, } from './expression'; import { type SerializedError } from './error'; import { type DBAdapter } from './db'; @@ -472,22 +473,23 @@ export class Batch { do { // SQLite does not support cursors when used in the worker thread since // the API for using cursors cannot be serialized over the postMessage - // boundary. so we use a handcrafted paging approach that leverages - // realm_version to keep the result set stable across pages + // boundary. so we use a handcrafted paging approach rows = (await this.#query([ 'SELECT i.url, i.file_alias, i.type', 'FROM boxel_index_working as i', - // TODO: If this query is still slow we'll need to consider dropping - // this join and writing a pg specific query here--as this query is - // currently written in a less performant manner in order to be - // compatible with SQLite. make sure to measure performance in hosted - // env first before sending for team review. Should strive to be less - // than 50ms (currently about 600-700ms) when this is run against a - // reasonably sized index. - 'CROSS JOIN LATERAL jsonb_array_elements_text(i.deps) as deps_array_element', + dbExpression({ + sqlite: + 'CROSS JOIN LATERAL jsonb_array_elements_text(i.deps) as deps_array_element', + }), 'WHERE', ...every([ - [`deps_array_element =`, param(resolvedPath)], + [ + dbExpression({ + sqlite: `deps_array_element =`, + pg: `COALESCE(i.deps, '[]'::jsonb) @>`, + }), + param({ sqlite: resolvedPath, pg: `["${resolvedPath}"]` }), + ], // css is a subset of modules, so there won't by any references that // are css entries that aren't already represented by a module entry [`i.type != 'css'`], From 8878d550c49f6a1ed185335f16909434686f9f9b Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Tue, 7 Jan 2025 13:11:20 -0500 Subject: [PATCH 08/12] include dropIndex in migration down script --- .../1735832183444_add-boxel-index-working.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/postgres/migrations/1735832183444_add-boxel-index-working.js b/packages/postgres/migrations/1735832183444_add-boxel-index-working.js index 31afcd9999..8476371505 100644 --- a/packages/postgres/migrations/1735832183444_add-boxel-index-working.js +++ b/packages/postgres/migrations/1735832183444_add-boxel-index-working.js @@ -51,7 +51,20 @@ exports.up = (pgm) => { }; exports.down = (pgm) => { + pgm.dropIndex('boxel_index_working', ['realm_version']); + pgm.dropIndex('boxel_index_working', ['realm_url']); + pgm.dropIndex('boxel_index_working', ['file_alias']); + pgm.dropIndex('boxel_index_working', ['resource_created_at']); + pgm.dropIndex('boxel_index_working', ['last_modified']); + pgm.dropIndex('boxel_index_working', 'type'); + pgm.dropIndex('boxel_index_working', ['url', 'realm_version']); + pgm.dropIndex('boxel_index_working', 'deps'); + pgm.dropIndex('boxel_index_working', 'types'); + pgm.dropIndex('boxel_index_working', 'fitted_html'); + pgm.dropIndex('boxel_index_working', 'embedded_html'); + pgm.dropIndex('boxel_index_working', 'search_doc'); pgm.dropTable('boxel_index_working', { cascade: true }); + pgm.dropConstraint('boxel_index', 'boxel_index_pkey'); pgm.addConstraint('boxel_index', 'boxel_index_pkey', { primaryKey: ['url', 'realm_url', 'realm_version'], From f7e66515fce23238ed5d2adff828e62f5b6c065a Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Tue, 7 Jan 2025 13:21:31 -0500 Subject: [PATCH 09/12] simplify expressionToSql logic --- packages/runtime-common/expression.ts | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/packages/runtime-common/expression.ts b/packages/runtime-common/expression.ts index a308d5efe1..1a215e3e99 100644 --- a/packages/runtime-common/expression.ts +++ b/packages/runtime-common/expression.ts @@ -382,30 +382,14 @@ export function expressionToSql( let text = query .map((element) => { if (isDbExpression(element)) { - if (dbAdapterKind === 'pg') { - return element.pg ?? ''; - } else if (dbAdapterKind === 'sqlite') { - return element.sqlite ?? ''; - } else { - throw assertNever(dbAdapterKind); - } + return element[dbAdapterKind] ?? ''; } else if (isParam(element)) { - if (dbAdapterKind === 'pg' && 'pg' in element && element.pg != null) { - values.push(element.pg); - return `$${values.length}`; - } else if ( - dbAdapterKind === 'sqlite' && - 'sqlite' in element && - element.sqlite != null - ) { - values.push(element.sqlite); - return `$${values.length}`; - } else if ('param' in element && element.param !== undefined) { - values.push(element.param); - return `$${values.length}`; - } else { + let value = element[dbAdapterKind] ?? element.param; + if (value === undefined) { throw new Error(`unexpected param value: ${JSON.stringify(element)}`); } + values.push(value); + return `$${values.length}`; } else if (typeof element === 'string') { return element; } else if (element.kind === 'table-valued-tree') { From 9a5f11c38b502e79ab13d86578daa2eafb4c740b Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Tue, 7 Jan 2025 13:23:59 -0500 Subject: [PATCH 10/12] better error message --- packages/runtime-common/expression.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/runtime-common/expression.ts b/packages/runtime-common/expression.ts index 1a215e3e99..2f21b89e42 100644 --- a/packages/runtime-common/expression.ts +++ b/packages/runtime-common/expression.ts @@ -386,7 +386,7 @@ export function expressionToSql( } else if (isParam(element)) { let value = element[dbAdapterKind] ?? element.param; if (value === undefined) { - throw new Error(`unexpected param value: ${JSON.stringify(element)}`); + throw new Error(`undefined param value: ${JSON.stringify(element)}`); } values.push(value); return `$${values.length}`; From d60ecc2cba7db7913596b8440dd9ea1270989ecc Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Tue, 7 Jan 2025 13:59:01 -0500 Subject: [PATCH 11/12] better handling for undefined db params --- packages/runtime-common/expression.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/runtime-common/expression.ts b/packages/runtime-common/expression.ts index 2f21b89e42..3ceaf3dd91 100644 --- a/packages/runtime-common/expression.ts +++ b/packages/runtime-common/expression.ts @@ -384,10 +384,7 @@ export function expressionToSql( if (isDbExpression(element)) { return element[dbAdapterKind] ?? ''; } else if (isParam(element)) { - let value = element[dbAdapterKind] ?? element.param; - if (value === undefined) { - throw new Error(`undefined param value: ${JSON.stringify(element)}`); - } + let value = element[dbAdapterKind] ?? element.param ?? null; values.push(value); return `$${values.length}`; } else if (typeof element === 'string') { From f12d6956cc1e34f6772a7d95fc4275865379c667 Mon Sep 17 00:00:00 2001 From: FadhlanR Date: Wed, 8 Jan 2025 10:28:05 +0700 Subject: [PATCH 12/12] Show card type icon of CardsGird's filter list (#1971) --- packages/base/cards-grid.gts | 10 ++++-- .../src/components/filter-list/index.gts | 32 ++++++++++++++++--- .../host/app/components/card-prerender.gts | 6 ++-- packages/host/app/lib/current-run.ts | 9 ++++++ packages/host/app/lib/isolated-render.gts | 4 +-- packages/host/app/services/render-service.ts | 15 +++++++-- .../config/schema/1735668047598_schema.sql | 1 + ...734686216941_prerendered-card-type-icon.js | 13 ++++++++ .../realm-server/tests/realm-server-test.ts | 6 +++- packages/runtime-common/card-document.ts | 1 + packages/runtime-common/helpers/index.ts | 2 ++ packages/runtime-common/index-structure.ts | 2 ++ packages/runtime-common/index-writer.ts | 4 ++- .../runtime-common/tests/index-writer-test.ts | 28 ++++++++++++++-- 14 files changed, 115 insertions(+), 18 deletions(-) create mode 100644 packages/postgres/migrations/1734686216941_prerendered-card-type-icon.js diff --git a/packages/base/cards-grid.gts b/packages/base/cards-grid.gts index 10038734b0..c3ba4e64b8 100644 --- a/packages/base/cards-grid.gts +++ b/packages/base/cards-grid.gts @@ -306,7 +306,7 @@ class Isolated extends Component { - filters: { displayName: string; icon: IconComponent; query: any }[] = + filters: { displayName: string; icon: IconComponent | string; query: any }[] = new TrackedArray([ { displayName: 'All Cards', @@ -424,7 +424,11 @@ class Isolated extends Component { } let cardTypeSummaries = (await response.json()).data as { id: string; - attributes: { displayName: string; total: number }; + attributes: { + displayName: string; + total: number; + iconHTML: string | null; + }; }[]; let excludedCardTypeIds = [ `${baseRealm.url}card-api/CardDef`, @@ -441,7 +445,7 @@ class Isolated extends Component { const lastIndex = summary.id.lastIndexOf('/'); this.filters.push({ displayName: summary.attributes.displayName, - icon: Captions, + icon: summary.attributes.iconHTML ?? Captions, query: { filter: { type: { diff --git a/packages/boxel-ui/addon/src/components/filter-list/index.gts b/packages/boxel-ui/addon/src/components/filter-list/index.gts index 961a08047e..0345c44ced 100644 --- a/packages/boxel-ui/addon/src/components/filter-list/index.gts +++ b/packages/boxel-ui/addon/src/components/filter-list/index.gts @@ -10,11 +10,13 @@ export interface FilterListIconSignature { export type FilterListIcon = ComponentLike; +import { htmlSafe } from '@ember/template'; + import { cn, eq } from '../../helpers.ts'; export type Filter = { displayName: string; - icon: FilterListIcon; + icon: FilterListIcon | string; }; interface Signature { @@ -39,9 +41,16 @@ export default class FilterList extends Component { class={{cn 'filter-list__button' selected=(eq @activeFilter filter)}} {{on 'click' (fn this.onChanged filter)}} data-test-boxel-filter-list-button={{filter.displayName}} - >{{filter.displayName}} + > + {{#if (isIconString filter.icon)}} + {{htmlSafe + (addClassToSVG filter.icon 'filter-list__icon') + }}{{filter.displayName}} + {{else}} + {{filter.displayName}}{{/if}} + {{/each}} } + +function addClassToSVG(svgString: string, className: string) { + return svgString + .replace(/]*)\sclass="([^"]*)"/, `]*)>/, ``); +} + +function isIconString(icon: FilterListIcon | string): icon is string { + return typeof icon === 'string'; +} diff --git a/packages/host/app/components/card-prerender.gts b/packages/host/app/components/card-prerender.gts index cfd63c3a4d..c2c9fb1ed0 100644 --- a/packages/host/app/components/card-prerender.gts +++ b/packages/host/app/components/card-prerender.gts @@ -105,7 +105,8 @@ export default class CardPrerender extends Component { realmURL, reader, indexWriter, - renderCard: this.renderService.renderCard.bind(this.renderService), + renderCard: this.renderService.renderCard, + render: this.renderService.render, }); setOwner(currentRun, getOwner(this)!); @@ -127,7 +128,8 @@ export default class CardPrerender extends Component { reader, indexWriter, ignoreData: { ...ignoreData }, - renderCard: this.renderService.renderCard.bind(this.renderService), + renderCard: this.renderService.renderCard, + render: this.renderService.render, }); setOwner(currentRun, getOwner(this)!); let current = await CurrentRun.incremental(currentRun, { diff --git a/packages/host/app/lib/current-run.ts b/packages/host/app/lib/current-run.ts index e3d2d6f8c3..70522c33fd 100644 --- a/packages/host/app/lib/current-run.ts +++ b/packages/host/app/lib/current-run.ts @@ -41,6 +41,7 @@ import { serializableError, type SerializedError, } from '@cardstack/runtime-common/error'; +import { cardTypeIcon } from '@cardstack/runtime-common/helpers'; import { RealmPaths, LocalPath } from '@cardstack/runtime-common/paths'; import { isIgnored } from '@cardstack/runtime-common/realm-index-updater'; import { type Reader, type Stats } from '@cardstack/runtime-common/worker'; @@ -50,6 +51,7 @@ import type * as CardAPI from 'https://cardstack.com/base/card-api'; import { type RenderCard, + type Render, IdentityContextWithErrors, } from '../services/render-service'; @@ -82,6 +84,7 @@ export class CurrentRun { #realmPaths: RealmPaths; #ignoreData: Record; #renderCard: RenderCard; + #render: Render; #realmURL: URL; #realmInfo?: RealmInfo; readonly stats: Stats = { @@ -100,12 +103,14 @@ export class CurrentRun { indexWriter, ignoreData = {}, renderCard, + render, }: { realmURL: URL; reader: Reader; indexWriter: IndexWriter; ignoreData?: Record; renderCard: RenderCard; + render: Render; }) { this.#indexWriter = indexWriter; this.#realmPaths = new RealmPaths(realmURL); @@ -113,6 +118,7 @@ export class CurrentRun { this.#realmURL = realmURL; this.#ignoreData = ignoreData; this.#renderCard = renderCard; + this.#render = render; } static async fromScratch(current: CurrentRun): Promise { @@ -413,6 +419,7 @@ export class CurrentRun { let cardType: typeof CardDef | undefined; let isolatedHtml: string | undefined; let atomHtml: string | undefined; + let iconHTML: string | undefined; let card: CardDef | undefined; let embeddedHtml: Record | undefined; let fittedHtml: Record | undefined; @@ -473,6 +480,7 @@ export class CurrentRun { }), ), ); + iconHTML = unwrap(sanitizeHTML(this.#render(cardTypeIcon(card)))); cardType = Reflect.getPrototypeOf(card)?.constructor as typeof CardDef; let data = api.serializeCard(card, { includeComputeds: true }); // prepare the document for index serialization @@ -552,6 +560,7 @@ export class CurrentRun { atomHtml, embeddedHtml, fittedHtml, + iconHTML, lastModified, resourceCreatedAt, types: typesMaybeError.types.map(({ refURL }) => refURL), diff --git a/packages/host/app/lib/isolated-render.gts b/packages/host/app/lib/isolated-render.gts index 28ca079157..c612132512 100644 --- a/packages/host/app/lib/isolated-render.gts +++ b/packages/host/app/lib/isolated-render.gts @@ -16,7 +16,7 @@ import type { SimpleElement } from '@simple-dom/interface'; interface Signature { Args: { - format: Format; + format?: Format; }; } @@ -24,7 +24,7 @@ export function render( C: ComponentLike, element: SimpleElement, owner: Owner, - format: Format, + format?: Format, ): void { // this needs to be a template-only component because the way we're invoking it // just grabs the template and would drop any associated class. diff --git a/packages/host/app/services/render-service.ts b/packages/host/app/services/render-service.ts index 30d2539f87..f8d1872ccf 100644 --- a/packages/host/app/services/render-service.ts +++ b/packages/host/app/services/render-service.ts @@ -2,6 +2,7 @@ import { getOwner } from '@ember/application'; import type Owner from '@ember/owner'; import Service, { service } from '@ember/service'; +import { ComponentLike } from '@glint/template'; import Serializer from '@simple-dom/serializer'; import voidMap from '@simple-dom/void-map'; @@ -58,6 +59,7 @@ interface RenderCardParams { componentCodeRef?: CodeRef; } export type RenderCard = (params: RenderCardParams) => Promise; +export type Render = (component: ComponentLike) => string; const maxRenderThreshold = 10000; export default class RenderService extends Service { @@ -69,7 +71,7 @@ export default class RenderService extends Service { renderError: Error | undefined; owner: Owner = getOwner(this)!; - async renderCard(params: RenderCardParams): Promise { + renderCard = async (params: RenderCardParams): Promise => { let { card, visit, @@ -127,7 +129,16 @@ export default class RenderService extends Service { let serializer = new Serializer(voidMap); let html = serializer.serialize(element); return parseCardHtml(html); - } + }; + + render = (component: ComponentLike): string => { + let element = getIsolatedRenderElement(this.document); + render(component, element, this.owner); + + let serializer = new Serializer(voidMap); + let html = serializer.serialize(element); + return parseCardHtml(html); + }; // TODO delete me private async resolveField( diff --git a/packages/host/config/schema/1735668047598_schema.sql b/packages/host/config/schema/1735668047598_schema.sql index 5577b1a87b..8375812c17 100644 --- a/packages/host/config/schema/1735668047598_schema.sql +++ b/packages/host/config/schema/1735668047598_schema.sql @@ -23,6 +23,7 @@ fitted_html BLOB, display_names BLOB, resource_created_at, + icon_html TEXT, PRIMARY KEY ( url, realm_version, realm_url ) ); diff --git a/packages/postgres/migrations/1734686216941_prerendered-card-type-icon.js b/packages/postgres/migrations/1734686216941_prerendered-card-type-icon.js new file mode 100644 index 0000000000..83964f952e --- /dev/null +++ b/packages/postgres/migrations/1734686216941_prerendered-card-type-icon.js @@ -0,0 +1,13 @@ +/* eslint-disable camelcase */ + +exports.shorthands = undefined; + +exports.up = (pgm) => { + pgm.addColumns('boxel_index', { + icon_html: 'varchar', + }); +}; + +exports.down = (pgm) => { + pgm.dropColumns('boxel_index', ['icon_html']); +}; diff --git a/packages/realm-server/tests/realm-server-test.ts b/packages/realm-server/tests/realm-server-test.ts index f892a1f5ca..405058b1a1 100644 --- a/packages/realm-server/tests/realm-server-test.ts +++ b/packages/realm-server/tests/realm-server-test.ts @@ -4055,7 +4055,8 @@ module('Realm Server', function (hooks) { let response = await request .get('/_types') .set('Accept', 'application/json'); - + let iconHTML = + ''; assert.strictEqual(response.status, 200, 'HTTP 200 status'); assert.deepEqual(response.body, { data: [ @@ -4065,6 +4066,7 @@ module('Realm Server', function (hooks) { attributes: { displayName: 'Friend', total: 2, + iconHTML, }, }, { @@ -4073,6 +4075,7 @@ module('Realm Server', function (hooks) { attributes: { displayName: 'Home', total: 1, + iconHTML, }, }, { @@ -4081,6 +4084,7 @@ module('Realm Server', function (hooks) { attributes: { displayName: 'Person', total: 3, + iconHTML, }, }, ], diff --git a/packages/runtime-common/card-document.ts b/packages/runtime-common/card-document.ts index 8386d478af..c8f55767f7 100644 --- a/packages/runtime-common/card-document.ts +++ b/packages/runtime-common/card-document.ts @@ -356,6 +356,7 @@ export function makeCardTypeSummaryDoc(summaries: CardTypeSummary[]) { attributes: { displayName: summary.display_name, total: summary.total, + iconHTML: summary.icon_html, }, })); diff --git a/packages/runtime-common/helpers/index.ts b/packages/runtime-common/helpers/index.ts index f9d7199a4a..c00d4fd6d4 100644 --- a/packages/runtime-common/helpers/index.ts +++ b/packages/runtime-common/helpers/index.ts @@ -1,5 +1,7 @@ import { parse } from 'date-fns'; +export * from './card-type-display-name'; + export interface SharedTests { [testName: string]: (assert: Assert, args: T) => Promise; } diff --git a/packages/runtime-common/index-structure.ts b/packages/runtime-common/index-structure.ts index 097f45b321..e8986ec03d 100644 --- a/packages/runtime-common/index-structure.ts +++ b/packages/runtime-common/index-structure.ts @@ -25,6 +25,7 @@ export interface BoxelIndexTable { fitted_html: Record | null; isolated_html: string | null; atom_html: string | null; + icon_html: string | null; indexed_at: string | null; // pg represents big integers as strings in javascript last_modified: string | null; // pg represents big integers as strings in javascript resource_created_at: string | null; // pg represents big integers as strings in javascript @@ -40,6 +41,7 @@ export interface CardTypeSummary { code_ref: string; display_name: string; total: number; + icon_html: string; } export interface RealmMetaTable { diff --git a/packages/runtime-common/index-writer.ts b/packages/runtime-common/index-writer.ts index cd0a378271..96a430534b 100644 --- a/packages/runtime-common/index-writer.ts +++ b/packages/runtime-common/index-writer.ts @@ -72,6 +72,7 @@ export interface InstanceEntry { embeddedHtml?: Record; fittedHtml?: Record; atomHtml?: string; + iconHTML?: string; types: string[]; displayNames: string[]; deps: Set; @@ -168,6 +169,7 @@ export class Batch { embedded_html: entry.embeddedHtml, fitted_html: entry.fittedHtml, atom_html: entry.atomHtml, + icon_html: entry.iconHTML, deps: [...entry.deps], types: entry.types, display_names: entry.displayNames, @@ -295,7 +297,7 @@ export class Batch { private async updateRealmMeta() { let results = await this.#query([ - `SELECT CAST(count(i.url) AS INTEGER) as total, i.display_names->>0 as display_name, i.types->>0 as code_ref + `SELECT CAST(count(i.url) AS INTEGER) as total, i.display_names->>0 as display_name, i.types->>0 as code_ref, MAX(i.icon_html) as icon_html FROM boxel_index as i INNER JOIN realm_versions r ON i.realm_url = r.realm_url WHERE`, diff --git a/packages/runtime-common/tests/index-writer-test.ts b/packages/runtime-common/tests/index-writer-test.ts index 685c712038..b090782a1a 100644 --- a/packages/runtime-common/tests/index-writer-test.ts +++ b/packages/runtime-common/tests/index-writer-test.ts @@ -556,6 +556,7 @@ const tests = Object.freeze({ ), isolated_html: `
Isolated HTML
`, atom_html: `Atom HTML`, + icon_html: 'test icon', }, ], ); @@ -611,6 +612,7 @@ const tests = Object.freeze({ last_modified: String(modified), resource_created_at: String(modified), is_deleted: null, + icon_html: 'test icon', }, 'the error entry includes last known good state of instance', ); @@ -665,6 +667,7 @@ const tests = Object.freeze({ last_modified: null, resource_created_at: null, is_deleted: false, + icon_html: null, }, 'the error entry does not include last known good state of instance', ); @@ -1170,6 +1173,7 @@ const tests = Object.freeze({ assert, { indexWriter, adapter }, ) => { + let iconHTML = 'test icon'; await setupIndex( adapter, [{ realm_url: testRealmURL, current_version: 1 }], @@ -1197,6 +1201,7 @@ const tests = Object.freeze({ types: [{ module: `./person`, name: 'Person' }, baseCardRef].map( (i) => internalKeyFor(i, new URL(testRealmURL)), ), + icon_html: iconHTML, }, ], ); @@ -1230,6 +1235,7 @@ const tests = Object.freeze({ { module: `./person`, name: 'Person' }, baseCardRef, ].map((i) => internalKeyFor(i, new URL(testRealmURL))), + iconHTML, }); let results = await adapter.execute( @@ -1265,7 +1271,12 @@ const tests = Object.freeze({ 'correct length of query result after indexing is done', ); let value = results[0].value as [ - { code_ref: string; display_name: string; total: number }, + { + code_ref: string; + display_name: string; + icon_html: string; + total: number; + }, ]; assert.strictEqual( value.length, @@ -1279,11 +1290,13 @@ const tests = Object.freeze({ total: 1, code_ref: `${testRealmURL}fancy-person/FancyPerson`, display_name: 'Fancy Person', + icon_html: iconHTML, }, { total: 1, code_ref: `${testRealmURL}person/Person`, display_name: 'Person', + icon_html: iconHTML, }, ], 'correct card type summary after indexing is done', @@ -1318,6 +1331,7 @@ const tests = Object.freeze({ { module: `./person`, name: 'Person' }, baseCardRef, ].map((i) => internalKeyFor(i, new URL(testRealmURL))), + iconHTML, }); let resource4: CardResource = { id: `${testRealmURL}4`, @@ -1347,6 +1361,7 @@ const tests = Object.freeze({ { module: `./card-api`, name: 'CardDef' }, baseCardRef, ].map((i) => internalKeyFor(i, new URL(testRealmURL))), + iconHTML, }); await batch.done(); @@ -1365,13 +1380,19 @@ const tests = Object.freeze({ 'correct length of query result after indexing is done', ); value = results[0].value as [ - { code_ref: string; display_name: string; total: number }, + { + code_ref: string; + display_name: string; + total: number; + icon_html: string; + }, ]; assert.strictEqual( value.length, 3, 'correct length of card type summary after indexing is done', ); + assert.deepEqual( value, [ @@ -1379,16 +1400,19 @@ const tests = Object.freeze({ total: 2, code_ref: `${testRealmURL}fancy-person/FancyPerson`, display_name: 'Fancy Person', + icon_html: iconHTML, }, { total: 1, code_ref: `${testRealmURL}person/Person`, display_name: 'Person', + icon_html: iconHTML, }, { total: 1, code_ref: `${testRealmURL}pet/Pet`, display_name: 'Pet', + icon_html: iconHTML, }, ], 'correct card type summary after indexing is done',