diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5ddd691fe2..f9bd7cbb60 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -142,11 +142,11 @@ jobs: - name: Fail if generated icons have been changed without underlying raw icon changing run: git diff --exit-code - host-test-in-memory-index: - name: Host Tests - in-memory index + host-test: + name: Host Tests runs-on: ubuntu-latest concurrency: - group: boxel-host-test-in-memory-index${{ github.head_ref || github.run_id }} + group: boxel-host-test${{ github.head_ref || github.run_id }} cancel-in-progress: true steps: - uses: actions/checkout@v4 @@ -175,43 +175,7 @@ jobs: if: always() with: junit_files: junit/host.xml - check_name: Host Tests (In-Memory Index) Test Results - - host-test-db-index: - name: Host Tests - db index - runs-on: ubuntu-latest - concurrency: - group: boxel-host-test-db-index${{ github.head_ref || github.run_id }} - cancel-in-progress: true - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/init - - name: Build boxel-ui - run: PG_INDEXER=true pnpm build - working-directory: packages/boxel-ui/addon - - name: Build host dist/ for fastboot - run: PG_INDEXER=true pnpm build - env: - NODE_OPTIONS: --max_old_space_size=4096 - working-directory: packages/host - - name: Start realm servers - run: pnpm start:all & - working-directory: packages/realm-server - - name: create realm users - run: pnpm register-realm-users - working-directory: packages/matrix - - name: host test suite - # don't run the db index host tests in percy until we remove the feature flag - run: pnpm test:wait-for-servers - env: - PERCY_TOKEN: ${{ secrets.PERCY_TOKEN_HOST }} - working-directory: packages/host - - name: Publish test results - uses: EnricoMi/publish-unit-test-result-action@v2.9.0 - if: always() - with: - junit_files: junit/host.xml - check_name: Host Tests (DB Index) Test Results + check_name: Host Tests Test Results matrix-client-test: name: Matrix Client Tests @@ -327,11 +291,11 @@ jobs: https://api.github.com/repos/$REPOSITORY/statuses/$HEAD_SHA \ -d '{"context":"Matrix Playwright tests report","description":"","target_url":"'"$PLAYWRIGHT_REPORT_URL"'","state":"success"}' - realm-server-in-memory-index-test: - name: Realm Server Tests - in-memory index + realm-server-test: + name: Realm Server Tests runs-on: ubuntu-latest concurrency: - group: realm-server-in-memory-index-test-${{ github.head_ref || github.run_id }} + group: realm-server-test-${{ github.head_ref || github.run_id }} cancel-in-progress: true steps: - uses: actions/checkout@v4 @@ -357,36 +321,6 @@ jobs: run: pnpm test:dom working-directory: packages/realm-server - realm-server-db-index-test: - name: Realm Server Tests - db index - runs-on: ubuntu-latest - concurrency: - group: realm-server-db-index-test-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/init - - name: Build boxel-ui - run: pnpm build - working-directory: packages/boxel-ui/addon - - name: Build host dist/ for fastboot - run: pnpm build - env: - NODE_OPTIONS: --max_old_space_size=4096 - working-directory: packages/host - - name: Start realm servers - run: PG_INDEXER=true pnpm start:all & - working-directory: packages/realm-server - - name: create realm users - run: pnpm register-realm-users - working-directory: packages/matrix - - name: realm server test suite - run: PG_INDEXER=true pnpm test:wait-for-servers - working-directory: packages/realm-server - - name: realm server DOM tests - run: PG_INDEXER=true pnpm test:dom - working-directory: packages/realm-server - change-check: name: Check which packages changed if: github.ref == 'refs/heads/main' @@ -424,9 +358,8 @@ jobs: needs: - change-check - boxel-ui-test - # don't forget to change this after we remove the feature flag - - host-test-in-memory-index - - realm-server-in-memory-index-test + - host-test + - realm-server-test uses: ./.github/workflows/manual-deploy.yml secrets: inherit with: diff --git a/README.md b/README.md index 649620bf9e..57e5f6ce56 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ If you wish to drop the development databases you can execute: pnpm drop-all-dbs ``` -You can then run `PGDATABASE=boxel_dev pnpm migrate up` (with `PGDATABASE` set accordingly) or just start the realm server (`PG_INDEXER=true pnpm start:all`) to create the database again. +You can then run `PGDATABASE=boxel_dev pnpm migrate up` (with `PGDATABASE` set accordingly) or just start the realm server (`pnpm start:all`) to create the database again. To interact with your local database directly you can use psql: ``` diff --git a/packages/base/card-api.gts b/packages/base/card-api.gts index 8050825a26..9f03eec2c6 100644 --- a/packages/base/card-api.gts +++ b/packages/base/card-api.gts @@ -323,10 +323,6 @@ export interface Field< component(model: Box): BoxComponent; getter(instance: BaseDef): BaseInstanceType; queryableValue(value: any, stack: BaseDef[]): SearchT; - // TODO remove this after feature flag is removed - queryMatcher( - innerMatcher: (innerValue: any) => boolean | null, - ): (value: SearchT) => boolean | null; handleNotLoadedError( instance: BaseInstanceType, e: NotLoaded, @@ -457,22 +453,6 @@ class ContainsMany }); } - queryMatcher( - innerMatcher: (innerValue: any) => boolean | null, - ): (value: any[] | null) => boolean | null { - return (value) => { - if (Array.isArray(value) && value.length === 0) { - return innerMatcher(null); - } - return ( - Array.isArray(value) && - value.some((innerValue) => { - return innerMatcher(innerValue); - }) - ); - }; - } - serialize( values: BaseInstanceType[], doc: JSONAPISingleResourceDocument, @@ -676,12 +656,6 @@ class Contains implements Field { return this.card[queryableValue](instance, stack); } - queryMatcher( - innerMatcher: (innerValue: any) => boolean | null, - ): (value: any) => boolean | null { - return (value) => innerMatcher(value); - } - serialize( value: InstanceType, doc: JSONAPISingleResourceDocument, @@ -835,12 +809,6 @@ class LinksTo implements Field { return this.card[queryableValue](instance, stack); } - queryMatcher( - innerMatcher: (innerValue: any) => boolean | null, - ): (value: any) => boolean | null { - return (value) => innerMatcher(value); - } - serialize( value: InstanceType, doc: JSONAPISingleResourceDocument, @@ -1183,22 +1151,6 @@ class LinksToMany }); } - queryMatcher( - innerMatcher: (innerValue: any) => boolean | null, - ): (value: any[] | null) => boolean | null { - return (value) => { - if (Array.isArray(value) && value.length === 0) { - return innerMatcher(null); - } - return ( - Array.isArray(value) && - value.some((innerValue) => { - return innerMatcher(innerValue); - }) - ); - }; - } - serialize( values: BaseInstanceType[] | null | undefined, doc: JSONAPISingleResourceDocument, diff --git a/packages/base/cards-grid.gts b/packages/base/cards-grid.gts index b59746f7ad..c6d32499f8 100644 --- a/packages/base/cards-grid.gts +++ b/packages/base/cards-grid.gts @@ -183,9 +183,8 @@ class Isolated extends Component { { filter: { not: { - type: { - module: `${baseRealm.url}cards-grid`, - name: 'CardsGrid', + eq: { + _cardType: 'Cards Grid', }, }, }, diff --git a/packages/host/app/components/card-prerender.gts b/packages/host/app/components/card-prerender.gts index 34fa89abab..d156a5d5c2 100644 --- a/packages/host/app/components/card-prerender.gts +++ b/packages/host/app/components/card-prerender.gts @@ -6,13 +6,12 @@ import { didCancel, enqueueTask, restartableTask } from 'ember-concurrency'; import { type Indexer } from '@cardstack/runtime-common'; import type { LocalPath } from '@cardstack/runtime-common/paths'; +import { readFileAsText as _readFileAsText } from '@cardstack/runtime-common/stream'; import { - type EntrySetter, + type IndexResults, type Reader, - type RunState, type RunnerOpts, -} from '@cardstack/runtime-common/search-index'; -import { readFileAsText as _readFileAsText } from '@cardstack/runtime-common/stream'; +} from '@cardstack/runtime-common/worker'; import { CurrentRun } from '../lib/current-run'; @@ -50,10 +49,10 @@ export default class CardPrerender extends Component { } } - private async fromScratch(realmURL: URL): Promise { + private async fromScratch(realmURL: URL): Promise { try { - let state = await this.doFromScratch.perform(realmURL); - return state; + let results = await this.doFromScratch.perform(realmURL); + return results; } catch (e: any) { if (!didCancel(e)) { throw e; @@ -65,17 +64,17 @@ export default class CardPrerender extends Component { } private async incremental( - prev: RunState, url: URL, + realmURL: URL, operation: 'delete' | 'update', - onInvalidation?: (invalidatedURLs: URL[]) => void, - ): Promise { + ignoreData: Record, + ): Promise { try { let state = await this.doIncremental.perform( - prev, url, + realmURL, operation, - onInvalidation, + ignoreData, ); return state; } catch (e: any) { @@ -98,7 +97,7 @@ export default class CardPrerender extends Component { }); private doFromScratch = enqueueTask(async (realmURL: URL) => { - let { reader, entrySetter, indexer } = this.getRunnerParams(); + let { reader, indexer } = this.getRunnerParams(); await this.resetLoaderInFastboot.perform(); let current = await CurrentRun.fromScratch( new CurrentRun({ @@ -106,7 +105,6 @@ export default class CardPrerender extends Component { loader: this.loaderService.loader, reader, indexer, - entrySetter, renderCard: this.renderService.renderCard.bind(this.renderService), }), ); @@ -116,23 +114,22 @@ export default class CardPrerender extends Component { private doIncremental = enqueueTask( async ( - prev: RunState, url: URL, + realmURL: URL, operation: 'delete' | 'update', - onInvalidation?: (invalidatedURLs: URL[]) => void, + ignoreData: Record, ) => { - let { reader, entrySetter, indexer } = this.getRunnerParams(); + let { reader, indexer } = this.getRunnerParams(); await this.resetLoaderInFastboot.perform(); let current = await CurrentRun.incremental({ url, + realmURL, operation, - prev, reader, + ignoreData, indexer, loader: this.loaderService.loader, - entrySetter, renderCard: this.renderService.renderCard.bind(this.renderService), - onInvalidation, }); this.renderService.indexRunDeferred?.fulfill(); return current; @@ -149,9 +146,7 @@ export default class CardPrerender extends Component { private getRunnerParams(): { reader: Reader; - entrySetter: EntrySetter; - // TODO make this required after feature flag removed - indexer?: Indexer; + indexer: Indexer; } { let self = this; function readFileAsText( @@ -172,7 +167,6 @@ export default class CardPrerender extends Component { } return { reader: getRunnerOpts(optsId).reader, - entrySetter: getRunnerOpts(optsId).entrySetter, indexer: getRunnerOpts(optsId).indexer, }; } else { @@ -183,7 +177,6 @@ export default class CardPrerender extends Component { ), readFileAsText, }, - entrySetter: this.localIndexer.setEntry.bind(this.localIndexer), indexer: this.localIndexer.indexer, }; } diff --git a/packages/host/app/config/environment.d.ts b/packages/host/app/config/environment.d.ts index 8e830fefcf..2fa7fba39c 100644 --- a/packages/host/app/config/environment.d.ts +++ b/packages/host/app/config/environment.d.ts @@ -28,7 +28,5 @@ declare const config: { loginMessageTimeoutMs: number; minSaveTaskDurationMs: number; sqlSchema: string; - featureFlags?: { - 'pg-indexer'?: boolean; - }; + featureFlags?: {}; }; diff --git a/packages/host/app/lib/current-run.ts b/packages/host/app/lib/current-run.ts index 98d4f0c84c..c12fb38345 100644 --- a/packages/host/app/lib/current-run.ts +++ b/packages/host/app/lib/current-run.ts @@ -1,6 +1,7 @@ +import { cached } from '@glimmer/tracking'; + import ignore, { type Ignore } from 'ignore'; -import flatMap from 'lodash/flatMap'; import isEqual from 'lodash/isEqual'; import merge from 'lodash/merge'; @@ -17,19 +18,17 @@ import { SupportedMimeType, Indexer, Batch, - type CodeRef, - type RealmInfo, -} from '@cardstack/runtime-common'; -import { - type SingleCardDocument, - type Relationship, -} from '@cardstack/runtime-common/card-document'; -import { loadCard, identifyCard, isBaseDef, moduleFrom, -} from '@cardstack/runtime-common/code-ref'; + type SearchEntryWithErrors, + type CodeRef, + type RealmInfo, + type IndexResults, + type SingleCardDocument, + type Relationship, +} from '@cardstack/runtime-common'; import { Deferred } from '@cardstack/runtime-common/deferred'; import { CardError, @@ -38,17 +37,8 @@ import { type SerializedError, } from '@cardstack/runtime-common/error'; import { RealmPaths, LocalPath } from '@cardstack/runtime-common/paths'; -import { - isIgnored, - type Reader, - type EntrySetter, - type RunState, - type Stats, - type Module, - type SearchEntryWithErrors, - type ModuleWithErrors, -} from '@cardstack/runtime-common/search-index'; -import { URLMap } from '@cardstack/runtime-common/url-map'; +import { isIgnored } from '@cardstack/runtime-common/search-index'; +import { type Reader, type Stats } from '@cardstack/runtime-common/worker'; import { CardDef, @@ -61,27 +51,30 @@ import { type RenderCard } from '../services/render-service'; const log = logger('current-run'); +interface Module { + url: string; + consumes: string[]; +} + +type ModuleWithErrors = + | { type: 'module'; module: Module } + | { type: 'error'; moduleURL: string; error: SerializedError }; + type TypesWithErrors = | { type: 'types'; types: string[] } | { type: 'error'; error: SerializedError }; export class CurrentRun { - #invalidations: string[] = []; - #instances: URLMap; #modules = new Map(); #moduleWorkingCache = new Map>(); #typesCache = new WeakMap>(); #indexingInstances = new Map>(); #reader: Reader; - // TODO make this required after feature flag removed - #indexer: Indexer | undefined; - // TODO make this required after feature flag removed + #indexer: Indexer; #batch: Batch | undefined; #realmPaths: RealmPaths; - #ignoreMap: URLMap; #ignoreData: Record; #loader: Loader; - #entrySetter: EntrySetter; #renderCard: RenderCard; #realmURL: URL; #realmInfo?: RealmInfo; @@ -96,37 +89,27 @@ export class CurrentRun { realmURL, reader, indexer, - instances = new URLMap(), - modules = new Map(), - ignoreMap = new URLMap(), - ignoreData = new Object(null) as Record, + ignoreData = {}, loader, - entrySetter, renderCard, }: { realmURL: URL; reader: Reader; - indexer?: Indexer; - instances?: URLMap; - modules?: Map; - ignoreMap?: URLMap; + indexer: Indexer; ignoreData?: Record; loader: Loader; - entrySetter: EntrySetter; renderCard: RenderCard; }) { this.#indexer = indexer; this.#realmPaths = new RealmPaths(realmURL); this.#reader = reader; this.#realmURL = realmURL; - this.#instances = instances; - this.#modules = modules; - this.#ignoreMap = ignoreMap; this.#ignoreData = ignoreData; - this.#loader = loader; - this.#entrySetter = entrySetter; this.#renderCard = renderCard; - + // we are intentionally using the same loader as the LoaderService.loader + // because our WIP header needs to be used anywhere the loader service's + // loader is being used (specifically in the render-service). + this.#loader = loader; this.loader.prependURLHandlers([ async (req) => { if (this.#isIndexing) { @@ -137,115 +120,80 @@ export class CurrentRun { ]); } - static async fromScratch(current: CurrentRun) { + static async fromScratch(current: CurrentRun): Promise { await current.whileIndexing(async () => { let start = Date.now(); log.debug(`starting from scratch indexing`); - (globalThis as any).__currentRunLoader = current.loader; - if (isDbIndexerEnabled()) { - current.#batch = await current.indexer.createBatch(current.realmURL); - current.#invalidations = []; - await current.batch.makeNewGeneration(); - } + // (globalThis as any).__currentRunLoader = current.loader; + current.#batch = await current.#indexer.createBatch(current.realmURL); + // current.#invalidations = []; + await current.batch.makeNewGeneration(); await current.visitDirectory(current.realmURL); - if (isDbIndexerEnabled()) { - await current.batch.done(); - } - (globalThis as any).__currentRunLoader = undefined; + await current.batch.done(); + // (globalThis as any).__currentRunLoader = undefined; log.debug(`completed from scratch indexing in ${Date.now() - start}ms`); }); - return current; + let { stats, ignoreData } = current; + return { invalidations: [], stats, ignoreData }; } static async incremental({ url, + realmURL, operation, - prev, + ignoreData, reader, loader, - entrySetter, renderCard, indexer, - onInvalidation, }: { url: URL; + realmURL: URL; operation: 'update' | 'delete'; - prev: RunState; + ignoreData: Record; reader: Reader; loader: Loader; - entrySetter: EntrySetter; renderCard: RenderCard; - indexer?: Indexer; - // TODO remove this after we remove the feature flag. this handler happens - // outside of the index job - onInvalidation?: (invalidatedURLs: URL[]) => void; - }) { + indexer: Indexer; + }): Promise { let start = Date.now(); log.debug(`starting from incremental indexing for ${url.href}`); - (globalThis as any).__currentRunLoader = loader; - let instances = new URLMap(prev.instances); - let ignoreMap = new URLMap(prev.ignoreMap); - let ignoreData = { ...prev.ignoreData }; - let invalidations: URL[] = []; - let modules = new Map(prev.modules); - - if (!isDbIndexerEnabled()) { - instances.remove(new URL(url.href.replace(/\.json$/, ''))); - invalidations = flatMap(invalidate(url, modules, instances), (u) => - // we only ever want to visit our own URL in the update case so we'll do - // that explicitly - u !== url.href && u !== trimExecutableExtension(url).href - ? [new URL(u)] - : [], - ); - } let current = new this({ - realmURL: prev.realmURL, + realmURL, reader, indexer, - instances, - modules, - ignoreMap, - ignoreData, + ignoreData: { ...ignoreData }, loader, - entrySetter, renderCard, }); - if (isDbIndexerEnabled()) { - current.#batch = await current.indexer.createBatch(current.realmURL); - invalidations = (await current.batch.invalidate(url)).map( - (href) => new URL(href), - ); - current.#invalidations = [...invalidations].map((url) => url.href); - } + current.#batch = await current.#indexer.createBatch(current.realmURL); + let invalidations = (await current.batch.invalidate(url)).map( + (href) => new URL(href), + ); await current.whileIndexing(async () => { - if (operation === 'update') { - await current.tryToVisit(url); - } for (let invalidation of invalidations) { - await current.tryToVisit(invalidation); + if (operation === 'delete' && invalidation.href === url.href) { + // file is deleted, there is nothing to visit + } else { + await current.tryToVisit(invalidation); + } } - if (isDbIndexerEnabled()) { - await current.batch.done(); - } + await current.batch.done(); - (globalThis as any).__currentRunLoader = undefined; log.debug( `completed incremental indexing for ${url.href} in ${ Date.now() - start }ms`, ); - if (onInvalidation) { - let urls = [...new Set([url, ...invalidations].map((u) => u.href))].map( - (href) => new URL(href.replace(/\.json$/, '')), - ); - onInvalidation(urls); - } }); - return current; + return { + invalidations: [...invalidations].map((url) => url.href), + ignoreData: current.#ignoreData, + stats: current.stats, + }; } private async tryToVisit(url: URL) { @@ -266,16 +214,6 @@ export class CurrentRun { this.#isIndexing = false; } - // TODO we can get rid of this after the feature flag is removed. this is just - // some type sugar so we don't have to check to see if the indexer exists - // since ultimately it will be required. - private get indexer() { - if (!this.#indexer) { - throw new Error(`Indexer is missing`); - } - return this.#indexer; - } - private get batch() { if (!this.#batch) { throw new Error('Batch is missing'); @@ -283,22 +221,10 @@ export class CurrentRun { return this.#batch; } - get instances() { - return this.#instances; - } - - get invalidations() { - return [...this.#invalidations]; - } - get modules() { return this.#modules; } - get ignoreMap() { - return this.#ignoreMap; - } - get ignoreData() { return this.#ignoreData; } @@ -311,12 +237,21 @@ export class CurrentRun { return this.#loader; } + @cached + private get ignoreMap() { + let ignoreMap = new Map(); + for (let [url, contents] of Object.entries(this.#ignoreData)) { + ignoreMap.set(url, ignore().add(contents)); + } + return ignoreMap; + } + private async visitDirectory(url: URL): Promise { let ignorePatterns = await this.#reader.readFileAsText( this.#realmPaths.local(new URL('.gitignore', url)), ); if (ignorePatterns && ignorePatterns.content) { - this.#ignoreMap.set(url, ignore().add(ignorePatterns.content)); + this.ignoreMap.set(url.href, ignore().add(ignorePatterns.content)); this.#ignoreData[url.href] = ignorePatterns.content; } @@ -324,7 +259,7 @@ export class CurrentRun { this.#realmPaths.local(url), )) { let innerURL = this.#realmPaths.fileURL(innerPath); - if (isIgnored(this.#realmURL, this.#ignoreMap, innerURL)) { + if (isIgnored(this.#realmURL, this.ignoreMap, innerURL)) { continue; } if (kind === 'file') { @@ -340,7 +275,7 @@ export class CurrentRun { url: URL, identityContext?: IdentityContextType, ): Promise { - if (isIgnored(this.#realmURL, this.#ignoreMap, url)) { + if (isIgnored(this.#realmURL, this.ignoreMap, url)) { return; } let start = Date.now(); @@ -404,17 +339,15 @@ export class CurrentRun { let deps = await ( await this.loader.getConsumedModules(url.href) ).filter((u) => u !== url.href); - if (isDbIndexerEnabled()) { - await this.batch.updateEntry(new URL(url), { - type: 'error', - error: { - status: 500, - detail: `encountered error loading module "${url.href}": ${err.message}`, - additionalErrors: null, - deps, - }, - }); - } + await this.batch.updateEntry(new URL(url), { + type: 'error', + error: { + status: 500, + detail: `encountered error loading module "${url.href}": ${err.message}`, + additionalErrors: null, + deps, + }, + }); this.#modules.set(url.href, { type: 'error', moduleURL: url.href, @@ -464,7 +397,7 @@ export class CurrentRun { let doc: SingleCardDocument | undefined; let searchData: Record | undefined; let cardType: typeof CardDef | undefined; - let html: string | undefined; + let isolatedHtml: string | undefined; try { let api = await this.#loader.import( `${baseRealm.url}card-api`, @@ -480,7 +413,7 @@ export class CurrentRun { let res = { ...resource, ...{ id: instanceURL.href } }; //Realm info may be used by a card to render field values. - //Example: catalog-etry-card + //Example: catalog-entry-card merge(res, { meta: { realmInfo: this.#realmInfo, @@ -496,7 +429,7 @@ export class CurrentRun { identityContext, }, ); - html = await this.#renderCard({ + isolatedHtml = await this.#renderCard({ card, format: 'isolated', visit: this.visitFile.bind(this), @@ -545,14 +478,17 @@ export class CurrentRun { typesMaybeError = await this.getTypes(cardType); } if (searchData && doc && typesMaybeError?.type === 'types') { - await this.setInstance(instanceURL, { + await this.updateEntry(instanceURL, { type: 'entry', entry: { resource: doc.data, searchData, - html, + isolatedHtml, types: typesMaybeError.types, - deps: new Set(await this.loader.getConsumedModules(moduleURL)), + deps: new Set([ + moduleURL, + ...(await this.loader.getConsumedModules(moduleURL)), + ]), }, }); } else if (uncaughtError || typesMaybeError?.type === 'error') { @@ -581,18 +517,13 @@ export class CurrentRun { log.warn( `encountered error indexing card instance ${path}: ${error.error.detail}`, ); - await this.setInstance(instanceURL, error); + await this.updateEntry(instanceURL, error); } deferred.fulfill(); } - private async setInstance(instanceURL: URL, entry: SearchEntryWithErrors) { - if (isDbIndexerEnabled()) { - await this.batch.updateEntry(assertURLEndsWithJSON(instanceURL), entry); - } else { - this.#instances.set(instanceURL, entry); - this.#entrySetter(instanceURL, entry); - } + private async updateEntry(instanceURL: URL, entry: SearchEntryWithErrors) { + await this.batch.updateEntry(assertURLEndsWithJSON(instanceURL), entry); if (entry.type === 'entry') { this.stats.instancesIndexed++; } else { @@ -636,16 +567,14 @@ export class CurrentRun { url, consumes, }; - if (isDbIndexerEnabled()) { - await this.batch.updateEntry(new URL(url), { - type: 'module', - module: { - deps: new Set( - consumes.map((d) => trimExecutableExtension(new URL(d)).href), - ), - }, - }); - } + await this.batch.updateEntry(new URL(url), { + type: 'module', + module: { + deps: new Set( + consumes.map((d) => trimExecutableExtension(new URL(d)).href), + ), + }, + }); this.#modules.set(url, { type: 'module', module }); deferred.fulfill(module); } @@ -691,122 +620,6 @@ export class CurrentRun { } } -function invalidate( - url: URL, - modules: Map, - instances: URLMap, - invalidations: string[] = [], - visited: Set = new Set(), -): string[] { - if (visited.has(url.href)) { - return []; - } - - let invalidationSet = new Set(invalidations); - // invalidate any instances whose deps come from the URL or whose error depends on the URL - let invalidatedInstances = [...instances] - .filter(([instanceURL, item]) => { - if (item.type === 'error') { - for (let errorDep of item.error.deps ?? []) { - if ( - errorDep === url.href || - errorDep === trimExecutableExtension(url).href - ) { - instances.remove(instanceURL); // note this is a side-effect - return true; - } - } - } else { - if ( - item.entry.deps.has(url.href) || - item.entry.deps.has(trimExecutableExtension(url).href) - ) { - instances.remove(instanceURL); // note this is a side-effect - return true; - } - } - return false; - }) - .map(([u]) => `${u.href}.json`); - for (let invalidation of invalidatedInstances) { - invalidationSet.add(invalidation); - } - - for (let [key, maybeError] of [...modules]) { - if (maybeError.type === 'error') { - // invalidate any errored modules that come from the URL - let errorModule = maybeError.moduleURL; - if ( - errorModule === url.href || - errorModule === trimExecutableExtension(url).href - ) { - modules.delete(key); - invalidationSet.add(errorModule); - } - - // invalidate any modules in an error state whose errorReference comes - // from the URL - for (let maybeDef of maybeError.error.deps ?? []) { - if ( - maybeDef === url.href || - maybeDef === trimExecutableExtension(url).href - ) { - for (let invalidation of invalidate( - new URL(errorModule), - modules, - instances, - [...invalidationSet], - new Set([...visited, url.href]), - )) { - invalidationSet.add(invalidation); - } - // no need to test the other error refs, we have already decided to - // invalidate this URL - break; - } - } - continue; - } - - let { module } = maybeError; - // invalidate any modules that come from the URL - if ( - module.url === url.href || - module.url === trimExecutableExtension(url).href - ) { - modules.delete(key); - invalidationSet.add(module.url); - } - - // invalidate any modules that consume the URL - for (let importURL of module.consumes) { - if ( - importURL === url.href || - importURL === trimExecutableExtension(url).href - ) { - for (let invalidation of invalidate( - new URL(module.url), - modules, - instances, - [...invalidationSet], - new Set([...visited, url.href]), - )) { - invalidationSet.add(invalidation); - } - // no need to test the other imports, we have already decided to - // invalidate this URL - break; - } - } - } - - return [...invalidationSet]; -} - -function isDbIndexerEnabled() { - return Boolean((globalThis as any).__enablePgIndexer?.()); -} - function assertURLEndsWithJSON(url: URL): URL { if (!url.href.endsWith('.json')) { return new URL(`${url}.json`); diff --git a/packages/host/app/lib/setup-globals.ts b/packages/host/app/lib/setup-globals.ts index 75a7c56c2a..0f7fcd3071 100644 --- a/packages/host/app/lib/setup-globals.ts +++ b/packages/host/app/lib/setup-globals.ts @@ -8,10 +8,3 @@ import ENV from '@cardstack/host/config/environment'; (globalThis as any)._logDefinitions ?? makeLogDefinitions(ENV.logLevels); (globalThis as any).Buffer = Buffer; - -// we use globalThis for this particular feature flag so that we can control it -// within a fastboot context as well -(globalThis as any).__enablePgIndexer = - typeof (globalThis as any).__enablePgIndexer === 'function' - ? (globalThis as any).__enablePgIndexer - : () => ENV.featureFlags?.['pg-indexer']; diff --git a/packages/host/app/services/local-indexer.ts b/packages/host/app/services/local-indexer.ts index 9ba7d02113..441f7dbecf 100644 --- a/packages/host/app/services/local-indexer.ts +++ b/packages/host/app/services/local-indexer.ts @@ -1,21 +1,22 @@ import Service from '@ember/service'; -import { type Indexer, type RealmAdapter } from '@cardstack/runtime-common'; import { - SearchEntryWithErrors, - type RunState, -} from '@cardstack/runtime-common/search-index'; + type IndexResults, + type Indexer, + type RealmAdapter, +} from '@cardstack/runtime-common'; // Tests inject an implementation of this service to help perform indexing // for the test-realm-adapter export default class LocalIndexer extends Service { setup( - _fromScratch: (realmURL: URL) => Promise, + _fromScratch: (realmURL: URL) => Promise, _incremental: ( - prev: RunState, url: URL, + realmURL: URL, operation: 'update' | 'delete', - ) => Promise, + ignoreData: Record, + ) => Promise, ) {} get adapter(): RealmAdapter { return {} as RealmAdapter; @@ -23,7 +24,6 @@ export default class LocalIndexer extends Service { get indexer(): Indexer { return {} as Indexer; } - async setEntry(_url: URL, _entry: SearchEntryWithErrors) {} } declare module '@ember/service' { diff --git a/packages/host/config/environment.js b/packages/host/config/environment.js index a87a773e84..413088de64 100644 --- a/packages/host/config/environment.js +++ b/packages/host/config/environment.js @@ -56,9 +56,7 @@ module.exports = function (environment) { environment === 'test' ? 'http://test-realm/test/' : process.env.OWN_REALM_URL || 'http://localhost:4200/', - featureFlags: { - 'pg-indexer': process.env.PG_INDEXER, - }, + featureFlags: {}, }; if (environment === 'development') { diff --git a/packages/host/tests/acceptance/interact-submode-test.gts b/packages/host/tests/acceptance/interact-submode-test.gts index c2cd2b9876..e702f2df33 100644 --- a/packages/host/tests/acceptance/interact-submode-test.gts +++ b/packages/host/tests/acceptance/interact-submode-test.gts @@ -11,7 +11,7 @@ import { setupApplicationTest } from 'ember-qunit'; import window from 'ember-window-mock'; import { setupWindowMock } from 'ember-window-mock/test-support'; -import { module, test } from 'qunit'; +import { module, test, skip } from 'qunit'; import stringify from 'safe-stable-stringify'; import { FieldContainer, GridContainer } from '@cardstack/boxel-ui/components'; @@ -1445,7 +1445,8 @@ module('Acceptance | interact submode tests', function (hooks) { assert.dom('[data-test-operator-mode-stack]').exists({ count: 2 }); }); - test('Clicking search panel (without left and right buttons activated) replaces all cards in the rightmost stack', async function (assert) { + // skipping FLaky test: CS-6845 + skip('Clicking search panel (without left and right buttons activated) replaces all cards in the rightmost stack', async function (assert) { // creates a recent search window.localStorage.setItem( 'recent-cards', diff --git a/packages/host/tests/acceptance/operator-mode-acceptance-test.gts b/packages/host/tests/acceptance/operator-mode-acceptance-test.gts index 8409607152..116096e8ef 100644 --- a/packages/host/tests/acceptance/operator-mode-acceptance-test.gts +++ b/packages/host/tests/acceptance/operator-mode-acceptance-test.gts @@ -12,7 +12,7 @@ import { setupApplicationTest } from 'ember-qunit'; import window from 'ember-window-mock'; import { setupWindowMock } from 'ember-window-mock/test-support'; -import { module, test } from 'qunit'; +import { module, test, skip } from 'qunit'; import { FieldContainer } from '@cardstack/boxel-ui/components'; @@ -398,6 +398,7 @@ module('Acceptance | operator mode tests', function (hooks) { assert.dom('[data-test-stack-card-index="0"]').exists(); // Index card opens in the stack await waitFor(`[data-test-cards-grid-item="${testRealmURL}Pet/mango"]`); + assert .dom(`[data-test-cards-grid-item="${testRealmURL}Pet/mango"]`) .exists(); @@ -407,6 +408,14 @@ module('Acceptance | operator mode tests', function (hooks) { assert .dom(`[data-test-cards-grid-item="${testRealmURL}Person/fadhlan"]`) .exists(); + assert + .dom(`[data-test-cards-grid-item="${testRealmURL}index"]`) + .doesNotExist('grid cards do not show other grid cards'); + // this was an unspelled, but very valid assertion that percy is making + // that I'm now making concrete + assert + .dom(`[data-test-cards-grid-item="${testRealmURL}grid"]`) + .doesNotExist('grid cards do not show other grid cards'); // this asserts that cards that throw errors during search // query deserialization (boom.json) are handled gracefully assert @@ -526,7 +535,8 @@ module('Acceptance | operator mode tests', function (hooks) { assert.dom('[data-test-profile-icon]').hasText('J'); // From display name "John" }); - test('can open code submode when card or field has no embedded template', async function (assert) { + // Flaky test: CS-6841 + skip('can open code submode when card or field has no embedded template', async function (assert) { await visitOperatorMode({ stacks: [ [ diff --git a/packages/host/tests/helpers/index.gts b/packages/host/tests/helpers/index.gts index 16639ccb9e..5e8b659db8 100644 --- a/packages/host/tests/helpers/index.gts +++ b/packages/host/tests/helpers/index.gts @@ -13,9 +13,13 @@ import { RealmPermissions, Deferred, Worker, + RunnerOptionsManager, type RealmInfo, type TokenClaims, type Indexer, + type RunnerRegistration, + type IndexRunner, + type IndexResults, } from '@cardstack/runtime-common'; import { @@ -26,15 +30,6 @@ import { Loader } from '@cardstack/runtime-common/loader'; import { Realm } from '@cardstack/runtime-common/realm'; -import { - RunnerOptionsManager, - type RunState, - type RunnerRegistration, - type EntrySetter, - type SearchEntryWithErrors, - type IndexRunner, -} from '@cardstack/runtime-common/search-index'; - import CardPrerender from '@cardstack/host/components/card-prerender'; import ENV from '@cardstack/host/config/environment'; import SQLiteAdapter from '@cardstack/host/lib/sqlite-adapter'; @@ -200,39 +195,37 @@ class MockLocalIndexer extends Service { url = new URL(testRealmURL); #adapter: RealmAdapter | undefined; #indexer: Indexer | undefined; - #entrySetter: EntrySetter | undefined; - #fromScratch: ((realmURL: URL) => Promise) | undefined; + #fromScratch: ((realmURL: URL) => Promise) | undefined; #incremental: | (( - prev: RunState, url: URL, + realmURL: URL, operation: 'update' | 'delete', - ) => Promise) + ignoreData: Record, + ) => Promise) | undefined; setup( - fromScratch: (realmURL: URL) => Promise, + fromScratch: (realmURL: URL) => Promise, incremental: ( - prev: RunState, url: URL, + realmURL: URL, operation: 'update' | 'delete', - ) => Promise, + ignoreData: Record, + ) => Promise, ) { this.#fromScratch = fromScratch; this.#incremental = incremental; } async configureRunner( registerRunner: RunnerRegistration, - entrySetter: EntrySetter, adapter: RealmAdapter, - // TODO make this required after feature flag is removed - indexer?: Indexer, + indexer: Indexer, ) { if (!this.#fromScratch || !this.#incremental) { throw new Error( `fromScratch/incremental not registered with MockLocalIndexer`, ); } - this.#entrySetter = entrySetter; this.#adapter = adapter; this.#indexer = indexer; await registerRunner( @@ -240,20 +233,16 @@ class MockLocalIndexer extends Service { this.#incremental.bind(this), ); } - async setEntry(url: URL, entry: SearchEntryWithErrors) { - if (!this.#entrySetter) { - throw new Error(`entrySetter not registered with MockLocalIndexer`); - } - this.#entrySetter(url, entry); - } get adapter() { if (!this.#adapter) { throw new Error(`adapter has not been set on MockLocalIndexer`); } return this.#adapter; } - // TODO make this throw when no indexer after feature flag removed get indexer() { + if (!this.#indexer) { + throw new Error(`indexer not registered with MockLocalIndexer`); + } return this.#indexer; } } @@ -531,29 +520,22 @@ async function setupTestRealm({ let adapter = new TestRealmAdapter(contents, new URL(realmURL)); let indexRunner: IndexRunner = async (optsId) => { - let { registerRunner, entrySetter, indexer } = - runnerOptsMgr.getOptions(optsId); - await localIndexer.configureRunner( - registerRunner, - entrySetter, - adapter, - indexer, - ); + let { registerRunner, indexer } = runnerOptsMgr.getOptions(optsId); + await localIndexer.configureRunner(registerRunner, adapter, indexer); }; let dbAdapter = await getDbAdapter(); realm = new Realm({ url: realmURL, adapter, - indexRunner, - runnerOptsMgr, getIndexHTML: async () => `Intentionally empty index.html (these tests will not exercise this capability)`, matrix: testMatrix, permissions, realmSecretSeed: testRealmSecretSeed, virtualNetwork, - ...((globalThis as any).__enablePgIndexer?.() ? { dbAdapter, queue } : {}), + dbAdapter, + queue, onIndexer: async (indexer) => { let worker = new Worker({ realmURL: new URL(realmURL!), @@ -561,7 +543,7 @@ async function setupTestRealm({ queue, realmAdapter: adapter, runnerOptsManager: runnerOptsMgr, - loader: virtualNetwork.createLoader(), + loader: realm.loaderTemplate, indexRunner, }); await worker.run(); diff --git a/packages/host/tests/integration/components/card-copy-test.gts b/packages/host/tests/integration/components/card-copy-test.gts index 55b7a33f1a..63ff2ff7a6 100644 --- a/packages/host/tests/integration/components/card-copy-test.gts +++ b/packages/host/tests/integration/components/card-copy-test.gts @@ -5,7 +5,7 @@ import GlimmerComponent from '@glimmer/component'; import { setupRenderingTest } from 'ember-qunit'; import { setupWindowMock } from 'ember-window-mock/test-support'; import flatMap from 'lodash/flatMap'; -import { module, test } from 'qunit'; +import { module, test, skip } from 'qunit'; import { validate as uuidValidate } from 'uuid'; import { @@ -797,7 +797,8 @@ module('Integration | card-copy', function (hooks) { ); }); - test('can copy a card that has a relative link to card in source realm', async function (assert) { + // Skip flaky test: CS-6846 + skip('can copy a card that has a relative link to card in source realm', async function (assert) { assert.expect(15); await setCardInOperatorModeState( [`${testRealmURL}index`], diff --git a/packages/host/tests/integration/components/card-delete-test.gts b/packages/host/tests/integration/components/card-delete-test.gts index 792a2fd6fe..5f66ab6024 100644 --- a/packages/host/tests/integration/components/card-delete-test.gts +++ b/packages/host/tests/integration/components/card-delete-test.gts @@ -3,7 +3,7 @@ import GlimmerComponent from '@glimmer/component'; import { setupRenderingTest } from 'ember-qunit'; import { setupWindowMock } from 'ember-window-mock/test-support'; -import { module, test } from 'qunit'; +import { module, test, skip } from 'qunit'; import { baseRealm } from '@cardstack/runtime-common'; import { Loader } from '@cardstack/runtime-common/loader'; @@ -607,7 +607,8 @@ module('Integration | card-delete', function (hooks) { assert.strictEqual(notFound, undefined, 'file ref does not exist'); }); - test('can delete a card that is a recent item', async function (assert) { + // Flaky test: CS-6843 + skip('can delete a card that is a recent item', async function (assert) { assert.expect(6); let expectedEvents = [ { @@ -675,7 +676,8 @@ module('Integration | card-delete', function (hooks) { .doesNotExist('recent item removed'); }); - test('can delete a card that is a selected item', async function (assert) { + // Flaky test: CS-6843 + skip('can delete a card that is a selected item', async function (assert) { assert.expect(6); let expectedEvents = [ { diff --git a/packages/host/tests/integration/components/operator-mode-test.gts b/packages/host/tests/integration/components/operator-mode-test.gts index 8199773dc8..928af88e3a 100644 --- a/packages/host/tests/integration/components/operator-mode-test.gts +++ b/packages/host/tests/integration/components/operator-mode-test.gts @@ -3080,7 +3080,8 @@ module('Integration | operator-mode', function (hooks) { assert.dom(`[data-test-create-new-card-button]`).isNotVisible(); }); - test(`displays recently accessed card`, async function (assert) { + // Flaky test: CS-6842 + skip(`displays recently accessed card`, async function (assert) { await setCardInOperatorModeState(`${testRealmURL}grid`); await renderComponent( class TestDriver extends GlimmerComponent { @@ -3184,8 +3185,8 @@ module('Integration | operator-mode', function (hooks) { assert.dom(`[data-test-search-label]`).containsText('Searching for “Ma”'); await waitFor(`[data-test-search-sheet-search-result]`); - assert.dom(`[data-test-search-label]`).containsText('2 Results for “Ma”'); - assert.dom(`[data-test-search-sheet-search-result]`).exists({ count: 2 }); + assert.dom(`[data-test-search-label]`).containsText('3 Results for “Ma”'); + assert.dom(`[data-test-search-sheet-search-result]`).exists({ count: 3 }); assert.dom(`[data-test-search-result="${testRealmURL}Pet/mango"]`).exists(); assert .dom(`[data-test-search-result="${testRealmURL}Author/mark"]`) @@ -3194,9 +3195,11 @@ module('Integration | operator-mode', function (hooks) { await click(`[data-test-search-sheet-cancel-button]`); await focus(`[data-test-search-field]`); - await typeIn(`[data-test-search-field]`, 'Mar'); + await typeIn(`[data-test-search-field]`, 'Mark J'); await waitFor(`[data-test-search-sheet-search-result]`); - assert.dom(`[data-test-search-label]`).containsText('1 Result for “Mar”'); + assert + .dom(`[data-test-search-label]`) + .containsText('1 Result for “Mark J”'); //Ensures that there is no cards when reopen the search sheet await click(`[data-test-search-sheet-cancel-button]`); diff --git a/packages/host/tests/integration/realm-test.ts b/packages/host/tests/integration/realm-test.ts index 4a796d4a97..bab768871d 100644 --- a/packages/host/tests/integration/realm-test.ts +++ b/packages/host/tests/integration/realm-test.ts @@ -669,7 +669,7 @@ module('Integration | realm', function (hooks) { ), }), ); - await Promise.all([realm.flushOperations(), realm.flushUpdateEvents()]); + await realm.flushUpdateEvents(); return await response; }, }); @@ -1977,7 +1977,7 @@ module('Integration | realm', function (hooks) { }, }), ); - await Promise.all([realm.flushOperations(), realm.flushUpdateEvents()]); + await realm.flushUpdateEvents(); return await response; }, }); @@ -2092,10 +2092,7 @@ module('Integration | realm', function (hooks) { body: cardSrc, }), ); - await Promise.all([ - realm.flushOperations(), - realm.flushUpdateEvents(), - ]); + await realm.flushUpdateEvents(); return await response; }, }); @@ -2161,7 +2158,7 @@ module('Integration | realm', function (hooks) { }, }), ); - await Promise.all([realm.flushOperations(), realm.flushUpdateEvents()]); + await realm.flushUpdateEvents(); assert.strictEqual((await response).status, 302, 'file exists'); response = realm.handle( @@ -2172,7 +2169,7 @@ module('Integration | realm', function (hooks) { }, }), ); - await Promise.all([realm.flushOperations(), realm.flushUpdateEvents()]); + await realm.flushUpdateEvents(); return await response; }, }); diff --git a/packages/host/tests/integration/search-index-test.gts b/packages/host/tests/integration/search-index-test.gts index 4d66068bda..2f8d1a6dcc 100644 --- a/packages/host/tests/integration/search-index-test.gts +++ b/packages/host/tests/integration/search-index-test.gts @@ -2,7 +2,7 @@ import { RenderingTestContext } from '@ember/test-helpers'; import GlimmerComponent from '@glimmer/component'; import { setupRenderingTest } from 'ember-qunit'; -import { module, test, skip } from 'qunit'; +import { module, test } from 'qunit'; import { baseRealm, @@ -32,7 +32,6 @@ const paths = new RealmPaths(new URL(testRealmURL)); const testModuleRealm = 'http://localhost:4202/test/'; let loader: Loader; -let isDbIndexingEnabled = (globalThis as any).__enablePgIndexer(); module(`Integration | search-index`, function (hooks) { setupRenderingTest(hooks); @@ -3668,63 +3667,59 @@ posts/ignore-me.json } }); - !isDbIndexingEnabled - ? skip( - `can filter on an array of primitive fields inside a containsMany using 'eq'`, - ) - : test(`can filter on an array of primitive fields inside a containsMany using 'eq'`, async function (assert) { - { - let { data: matching } = await indexer.search({ - filter: { - on: { - module: `${testModuleRealm}booking`, - name: 'Booking', - }, - eq: { sponsors: 'Nintendo' }, - }, - }); - assert.deepEqual( - matching.map((m) => m.id), - [`${paths.url}booking1`], - 'eq on sponsors', - ); - } - { - let { data: matching } = await indexer.search({ - filter: { - on: { - module: `${testModuleRealm}booking`, - name: 'Booking', - }, - eq: { sponsors: 'Playstation' }, - }, - }); - assert.strictEqual( - matching.length, - 0, - 'eq on nonexisting value in sponsors', - ); - } - { - let { data: matching } = await indexer.search({ - filter: { - on: { - module: `${testModuleRealm}booking`, - name: 'Booking', - }, - eq: { - 'hosts.firstName': 'Arthur', - sponsors: null, - }, - }, - }); - assert.deepEqual( - matching.map((m) => m.id), - [`${paths.url}booking2`], - 'eq on hosts.firstName and null sponsors', - ); - } + test(`can filter on an array of primitive fields inside a containsMany using 'eq'`, async function (assert) { + { + let { data: matching } = await indexer.search({ + filter: { + on: { + module: `${testModuleRealm}booking`, + name: 'Booking', + }, + eq: { sponsors: 'Nintendo' }, + }, }); + assert.deepEqual( + matching.map((m) => m.id), + [`${paths.url}booking1`], + 'eq on sponsors', + ); + } + { + let { data: matching } = await indexer.search({ + filter: { + on: { + module: `${testModuleRealm}booking`, + name: 'Booking', + }, + eq: { sponsors: 'Playstation' }, + }, + }); + assert.strictEqual( + matching.length, + 0, + 'eq on nonexisting value in sponsors', + ); + } + { + let { data: matching } = await indexer.search({ + filter: { + on: { + module: `${testModuleRealm}booking`, + name: 'Booking', + }, + eq: { + 'hosts.firstName': 'Arthur', + sponsors: null, + }, + }, + }); + assert.deepEqual( + matching.map((m) => m.id), + [`${paths.url}booking2`], + 'eq on hosts.firstName and null sponsors', + ); + } + }); test('can negate a filter', async function (assert) { let { data: matching } = await indexer.search({ @@ -3979,54 +3974,48 @@ posts/ignore-me.json ); }); - // There is actually a sorting bug here in our original in-memory based - // index that was revealed when we started using the DB based index. - // probably not worth fixing as we are about to remove the in-memory based - // index shortly. - !isDbIndexingEnabled - ? skip("can sort on multiple paths in combination with 'any' filter") - : test(`can sort on multiple paths in combination with 'any' filter`, async function (assert) { - let { data: matching } = await indexer.search({ - sort: [ - { - by: 'author.lastName', - on: { module: `${testModuleRealm}book`, name: 'Book' }, + test(`can sort on multiple paths in combination with 'any' filter`, async function (assert) { + let { data: matching } = await indexer.search({ + sort: [ + { + by: 'author.lastName', + on: { module: `${testModuleRealm}book`, name: 'Book' }, + }, + { + by: 'author.firstName', + on: { module: `${testModuleRealm}book`, name: 'Book' }, + direction: 'desc', + }, + ], + filter: { + any: [ + { + type: { + module: `${testModuleRealm}book`, + name: 'Book', }, - { - by: 'author.firstName', - on: { module: `${testModuleRealm}book`, name: 'Book' }, - direction: 'desc', + }, + { + type: { + module: `${testModuleRealm}article`, + name: 'Article', }, - ], - filter: { - any: [ - { - type: { - module: `${testModuleRealm}book`, - name: 'Book', - }, - }, - { - type: { - module: `${testModuleRealm}article`, - name: 'Article', - }, - }, - ], }, - }); - assert.deepEqual( - matching.map((m) => m.id), - [ - `${paths.url}books/2`, // Ab Van Gogh - `${paths.url}books/1`, // Ab Mango - `${paths.url}books/3`, // Ag Jackie - `${paths.url}cards/2`, // De Darrin - `${paths.url}card-2`, // Jo Cardy - `${paths.url}card-1`, // St Cardy - ], - ); - }); + ], + }, + }); + assert.deepEqual( + matching.map((m) => m.id), + [ + `${paths.url}books/2`, // Ab Van Gogh + `${paths.url}books/1`, // Ab Mango + `${paths.url}books/3`, // Ag Jackie + `${paths.url}cards/2`, // De Darrin + `${paths.url}card-2`, // Jo Cardy + `${paths.url}card-1`, // St Cardy + ], + ); + }); test(`can sort on multiple paths in combination with 'every' filter`, async function (assert) { let { data: matching } = await indexer.search({ diff --git a/packages/realm-server/fastboot.ts b/packages/realm-server/fastboot.ts index de1bf9fd2b..ebf27c6b83 100644 --- a/packages/realm-server/fastboot.ts +++ b/packages/realm-server/fastboot.ts @@ -5,7 +5,7 @@ import { instantiateFastBoot } from './fastboot-from-deployed'; import { type IndexRunner, type RunnerOpts, -} from '@cardstack/runtime-common/search-index'; +} from '@cardstack/runtime-common/worker'; import { JSDOM } from 'jsdom'; import { type ErrorReporter } from '@cardstack/runtime-common/realm'; @@ -29,9 +29,6 @@ export async function makeFastBootIndexRunner( buildSandboxGlobals(defaultGlobals: any) { return Object.assign({}, defaultGlobals, { __boxelErrorReporter: globalWithErrorReporter.__boxelErrorReporter, - // the fastboot instance is shared across all tests so we use a - // function to return the feature flag since this can change between tests - __enablePgIndexer: (globalThis as any).__enablePgIndexer, URL: globalThis.URL, Request: globalThis.Request, Response: globalThis.Response, @@ -49,7 +46,6 @@ export async function makeFastBootIndexRunner( (defaultGlobals: any) => { return Object.assign({}, defaultGlobals, { __boxelErrorReporter: globalWithErrorReporter.__boxelErrorReporter, - __enablePgIndexer: (globalThis as any).__enablePgIndexer, URL: globalThis.URL, Request: globalThis.Request, Response: globalThis.Response, diff --git a/packages/realm-server/main.ts b/packages/realm-server/main.ts index 10eca1abcb..53f7f663ba 100644 --- a/packages/realm-server/main.ts +++ b/packages/realm-server/main.ts @@ -4,13 +4,13 @@ import { Worker, VirtualNetwork, logger, + RunnerOptionsManager, } from '@cardstack/runtime-common'; import { NodeAdapter } from './node-realm'; import yargs from 'yargs'; import { RealmServer } from './server'; import { resolve, join } from 'path'; import { makeFastBootIndexRunner } from './fastboot'; -import { RunnerOptionsManager } from '@cardstack/runtime-common/search-index'; import { readFileSync } from 'fs-extra'; import { shimExternals } from './lib/externals'; import { type RealmPermissions as RealmPermissionsInterface } from '@cardstack/runtime-common/realm'; @@ -37,11 +37,6 @@ if (process.env.REALM_SENTRY_DSN) { ); } -if (process.env.PG_INDEXER) { - console.log('enabling db-based indexing'); -} -(globalThis as any).__enablePgIndexer = () => Boolean(process.env.PG_INDEXER); - const REALM_SECRET_SEED = process.env.REALM_SECRET_SEED; if (!REALM_SECRET_SEED) { console.error( @@ -162,13 +157,9 @@ if (distURL) { (async () => { let realms: Realm[] = []; - let dbAdapter: PgAdapter | undefined; - let queue: PgQueue | undefined; - if (process.env.PG_INDEXER) { - dbAdapter = new PgAdapter(); - queue = new PgQueue(dbAdapter); - await dbAdapter.startClient(); - } + let dbAdapter = new PgAdapter(); + let queue = new PgQueue(dbAdapter); + await dbAdapter.startClient(); for (let [i, path] of paths.entries()) { let url = hrefs[i][0]; @@ -199,30 +190,31 @@ if (distURL) { { url, adapter: realmAdapter, - indexRunner: getRunner, - runnerOptsMgr: manager, getIndexHTML: async () => readFileSync(join(distPath, 'index.html')).toString(), matrix: { url: new URL(matrixURL), username, password }, realmSecretSeed: REALM_SECRET_SEED, permissions: realmPermissions.users, virtualNetwork, - // TODO remove this guard after the feature flag is removed - ...(dbAdapter && queue ? { dbAdapter, queue } : {}), + dbAdapter, + queue, onIndexer: async (indexer) => { - // TODO remove this guard after the feature flag is removed - if (queue) { - let worker = new Worker({ - realmURL: new URL(url), - indexer, - queue, - realmAdapter, - runnerOptsManager: manager, - loader: virtualNetwork.createLoader(), - indexRunner: getRunner, - }); - await worker.run(); - } + // Note for future: we are taking advantage of the fact that the realm + // does not need to auth with itself and are passing in the realm's + // loader which includes a url handler for internal requests that + // bypasses auth. when workers are moved outside of the realm server + // they will need to provide realm authentication credentials when + // indexing. + let worker = new Worker({ + realmURL: new URL(url), + indexer, + queue, + realmAdapter, + runnerOptsManager: manager, + loader: realm.loaderTemplate, + indexRunner: getRunner, + }); + await worker.run(); }, }, { diff --git a/packages/realm-server/scripts/remove-test-dbs.sh b/packages/realm-server/scripts/remove-test-dbs.sh index 89058c1765..3fd88c4441 100755 --- a/packages/realm-server/scripts/remove-test-dbs.sh +++ b/packages/realm-server/scripts/remove-test-dbs.sh @@ -2,6 +2,7 @@ databases=$(docker exec boxel-pg psql -U postgres -w -lqt | cut -d \| -f 1 | grep -E 'test_db_' | tr -d ' ') +echo "cleaning up old test databases..." for db in $databases; do docker exec boxel-pg dropdb -U postgres -w $db done diff --git a/packages/realm-server/scripts/start-production.sh b/packages/realm-server/scripts/start-production.sh index ba954b9f3b..2e224bdef0 100755 --- a/packages/realm-server/scripts/start-production.sh +++ b/packages/realm-server/scripts/start-production.sh @@ -3,7 +3,6 @@ pnpm setup:base-in-deployment pnpm setup:drafts-in-deployment pnpm setup:published-in-deployment NODE_NO_WARNINGS=1 \ - PG_INDEXER=true \ LOG_LEVELS='*=info' \ ts-node \ --transpileOnly main \ diff --git a/packages/realm-server/scripts/start-staging.sh b/packages/realm-server/scripts/start-staging.sh index 7a1fc600c6..7fd70aebcc 100755 --- a/packages/realm-server/scripts/start-staging.sh +++ b/packages/realm-server/scripts/start-staging.sh @@ -3,7 +3,6 @@ pnpm setup:base-in-deployment pnpm setup:drafts-in-deployment pnpm setup:published-in-deployment NODE_NO_WARNINGS=1 \ - PG_INDEXER=true \ LOG_LEVELS='*=info' \ ts-node \ --transpileOnly main \ diff --git a/packages/realm-server/scripts/start-test-realms.sh b/packages/realm-server/scripts/start-test-realms.sh index 632ea95dcf..a18ae14568 100755 --- a/packages/realm-server/scripts/start-test-realms.sh +++ b/packages/realm-server/scripts/start-test-realms.sh @@ -2,13 +2,10 @@ check_postgres_ready() { docker exec boxel-pg pg_isready -U postgres >/dev/null 2>&1 } -# remove this check after the feature flag is removed -if [ -n "$PG_INDEXER" ]; then - while ! check_postgres_ready; do - printf '.' - sleep 1 - done -fi +while ! check_postgres_ready; do + printf '.' + sleep 1 +done NODE_ENV=test \ PGPORT=5435 \ diff --git a/packages/realm-server/scripts/start-without-matrix.sh b/packages/realm-server/scripts/start-without-matrix.sh index 1b5fb620b0..a534401c4d 100755 --- a/packages/realm-server/scripts/start-without-matrix.sh +++ b/packages/realm-server/scripts/start-without-matrix.sh @@ -1,6 +1,6 @@ #! /bin/sh NODE_NO_WARNINGS=1 start-server-and-test \ - 'run-p start:development start:base:root' \ + 'run-p start:pg start:development start:base:root' \ 'http-get://localhost:4201/base/fields/boolean-field?acceptHeader=application%2Fvnd.card%2Bjson|http-get://localhost:4203/fields/boolean-field?acceptHeader=application%2Fvnd.card%2Bjson|http-get://localhost:4201/drafts/index?acceptHeader=application%2Fvnd.card%2Bjson' \ 'run-p start:test-realms start:test-container' \ 'http-get://localhost:4202/node-test/person-1?acceptHeader=application%2Fvnd.card%2Bjson|http-get://127.0.0.1:4205' \ diff --git a/packages/realm-server/scripts/wait-for-pg.sh b/packages/realm-server/scripts/wait-for-pg.sh index 56ad0acafe..234d10e568 100755 --- a/packages/realm-server/scripts/wait-for-pg.sh +++ b/packages/realm-server/scripts/wait-for-pg.sh @@ -7,19 +7,16 @@ wait_for_postgres() { check_postgres_ready() { docker exec boxel-pg pg_isready -U postgres >/dev/null 2>&1 } - # remove this check after the feature flag is removed - if [ -n "$PG_INDEXER" ]; then - while ! check_postgres_ready; do - if [ $COUNT -eq 0 ]; then - echo "Waiting for postgres" - fi - if [ $COUNT -eq $MAX_ATTEMPTS ]; then - echo "Failed to detect postgres after $MAX_ATTEMPTS attempts." - exit 1 - fi - COUNT=$((COUNT + 1)) - printf '.' - sleep 5 - done - fi + while ! check_postgres_ready; do + if [ $COUNT -eq 0 ]; then + echo "Waiting for postgres" + fi + if [ $COUNT -eq $MAX_ATTEMPTS ]; then + echo "Failed to detect postgres after $MAX_ATTEMPTS attempts." + exit 1 + fi + COUNT=$((COUNT + 1)) + printf '.' + sleep 5 + done } diff --git a/packages/realm-server/tests/helpers/index.ts b/packages/realm-server/tests/helpers/index.ts index 7dc1e86295..16b4754f57 100644 --- a/packages/realm-server/tests/helpers/index.ts +++ b/packages/realm-server/tests/helpers/index.ts @@ -8,13 +8,13 @@ import { RealmPermissions, VirtualNetwork, Worker, + RunnerOptionsManager, type MatrixConfig, type Queue, + type IndexRunner, } from '@cardstack/runtime-common'; import { makeFastBootIndexRunner } from '../../fastboot'; -import { RunnerOptionsManager } from '@cardstack/runtime-common/search-index'; import type * as CardAPI from 'https://cardstack.com/base/card-api'; -import { type IndexRunner } from '@cardstack/runtime-common/search-index'; import { RealmServer } from '../../server'; import PgAdapter from '../../pg-adapter'; import PgQueue from '../../pg-queue'; @@ -142,18 +142,17 @@ export async function createRealm({ } let adapter = new NodeAdapter(dir); - return new Realm({ + let realm = new Realm({ url: realmURL, adapter, - indexRunner, - runnerOptsMgr: manager, getIndexHTML: async () => readFileSync(join(distPath, 'index.html')).toString(), matrix: matrixConfig, permissions, realmSecretSeed: "shhh! it's a secret", virtualNetwork, - ...((globalThis as any).__enablePgIndexer?.() ? { dbAdapter, queue } : {}), + dbAdapter, + queue, onIndexer: async (indexer) => { let worker = new Worker({ realmURL: new URL(realmURL!), @@ -161,12 +160,13 @@ export async function createRealm({ queue, realmAdapter: adapter, runnerOptsManager: manager, - loader: virtualNetwork.createLoader(), + loader: realm.loaderTemplate, indexRunner, }); await worker.run(); }, }); + return realm; } export function setupBaseRealmServer( diff --git a/packages/realm-server/tests/indexing-test.ts b/packages/realm-server/tests/indexing-test.ts index 68dd64d273..e5e611ad12 100644 --- a/packages/realm-server/tests/indexing-test.ts +++ b/packages/realm-server/tests/indexing-test.ts @@ -533,17 +533,15 @@ module('indexing', function (hooks) { delete actual.error.stack; assert.ok( // assert.deepEqual returns false because despite having the same shape, the constructors are different - isEqual(await realm.searchIndex.card(new URL(`${testRealm}post-1`)), { + isEqual(actual, { type: 'error', error: { isCardError: true, additionalErrors: null, detail: 'http://test-realm/post not found', - source: undefined, status: 404, title: 'Not Found', deps: ['http://test-realm/post'], - responseText: undefined, }, }), 'card instance is an error document', diff --git a/packages/realm-server/tests/loader-test.ts b/packages/realm-server/tests/loader-test.ts index f856a3ce90..839a39e6d8 100644 --- a/packages/realm-server/tests/loader-test.ts +++ b/packages/realm-server/tests/loader-test.ts @@ -82,7 +82,6 @@ module('loader', function (hooks) { let loader = virtualNetwork.createLoader(); await loader.import<{ a(): string }>(`${testRealmHref}a`); assert.deepEqual(await loader.getConsumedModules(`${testRealmHref}a`), [ - `${testRealmHref}a`, `${testRealmHref}b`, `${testRealmHref}c`, ]); @@ -96,7 +95,6 @@ module('loader', function (hooks) { } catch (e: any) { assert.strictEqual(e.message, 'intentional error thrown'); assert.deepEqual(await loader.getConsumedModules(`${testRealmHref}d`), [ - `${testRealmHref}d`, `${testRealmHref}a`, `${testRealmHref}b`, `${testRealmHref}c`, @@ -109,10 +107,7 @@ module('loader', function (hooks) { let loader = virtualNetwork.createLoader(); await loader.import<{ three(): number }>(`${testRealmHref}cycle-two`); let modules = await loader.getConsumedModules(`${testRealmHref}cycle-two`); - assert.deepEqual(modules, [ - `${testRealmHref}cycle-two`, - `${testRealmHref}cycle-one`, - ]); + assert.deepEqual(modules, [`${testRealmHref}cycle-one`]); }); test('supports identify API', async function (assert) { diff --git a/packages/realm-server/tests/queue-test.ts b/packages/realm-server/tests/queue-test.ts index 3192032d9e..c883d10440 100644 --- a/packages/realm-server/tests/queue-test.ts +++ b/packages/realm-server/tests/queue-test.ts @@ -54,12 +54,16 @@ module('queue', function (hooks) { assert.strictEqual( startedCount, expectedStartedCount, - `the expected started count before job run, ${expectedStartedCount}, is correct`, + `For Queue #${ + expectedStartedCount + 1 + }, the expected started count before job run, ${expectedStartedCount}, is correct`, ); assert.strictEqual( completedCount, expectedStartedCount, - `the expected completed count before job run, ${expectedStartedCount}, is correct`, + `For Queue #${ + expectedStartedCount + 1 + }, the expected completed count before job run, ${expectedStartedCount}, is correct`, ); startedCount++; await new Promise((r) => setTimeout(r, 500)); @@ -67,14 +71,18 @@ module('queue', function (hooks) { assert.strictEqual( startedCount, expectedStartedCount + 1, - `the expected started count after job run, ${ + `For Queue #${ + expectedStartedCount + 1 + }, the expected started count after job run, ${ expectedStartedCount + 1 }, is correct`, ); assert.strictEqual( completedCount, expectedStartedCount + 1, - `the expected completed count after job run, ${ + `For Queue #${ + expectedStartedCount + 1 + }, the expected completed count after job run, ${ expectedStartedCount + 1 }, is correct`, ); @@ -83,16 +91,17 @@ module('queue', function (hooks) { queue.register('count', count); queue2.register('count', count); - let [job1, job2] = await Promise.all([ - queue.publish('count', 0, { - queueName: 'serial-queue', - }), - queue2.publish('count', 1, { - queueName: 'serial-queue', - }), - ]); - - await Promise.all([job2.done, job1.done]); + let promiseForJob1 = queue.publish('count', 0, { + queueName: 'serial-queue', + }); + // start the 2nd job before the first job finishes + await new Promise((r) => setTimeout(r, 100)); + let promiseForJob2 = queue2.publish('count', 1, { + queueName: 'serial-queue', + }); + let [job1, job2] = await Promise.all([promiseForJob1, promiseForJob2]); + + await Promise.all([job1.done, job2.done]); }); test('different queues are processed concurrently across different queue clients', async function (assert) { diff --git a/packages/runtime-common/indexer.ts b/packages/runtime-common/indexer.ts index b92d4c3c7f..230a496b99 100644 --- a/packages/runtime-common/indexer.ts +++ b/packages/runtime-common/indexer.ts @@ -48,7 +48,6 @@ import { } from './query'; import { type SerializedError } from './error'; import { type DBAdapter } from './db'; -import { type SearchEntryWithErrors } from './search-index'; import type { BaseDef, Field } from 'https://cardstack.com/base/card-api'; import type * as CardAPI from 'https://cardstack.com/base/card-api'; @@ -829,6 +828,18 @@ export class Indexer { } } +export interface SearchEntry { + resource: CardResource; + searchData: Record; + isolatedHtml?: string; + types: string[]; + deps: Set; +} + +export type SearchEntryWithErrors = + | { type: 'entry'; entry: SearchEntry } + | { type: 'error'; error: SerializedError }; + export class Batch { readonly ready: Promise; private touched = new Set(); @@ -870,7 +881,7 @@ export class Batch { type: 'instance', pristine_doc: entry.entry.resource, search_doc: entry.entry.searchData, - isolated_html: entry.entry.html, + isolated_html: entry.entry.isolatedHtml, deps: [...entry.entry.deps], types: entry.entry.types, } @@ -907,6 +918,7 @@ export class Batch { } async makeNewGeneration() { + await this.setNextGenerationRealmVersion(); this.isNewGeneration = true; let cols = [ 'url', @@ -953,9 +965,12 @@ export class Batch { if (this.isNewGeneration) { await this.client.query([ `DELETE FROM boxel_index`, - 'WHERE realm_version <', - param(this.realmVersion), - ]); + 'WHERE', + ...every([ + ['realm_version <', param(this.realmVersion)], + ['realm_url =', param(this.realmURL.href)], + ]), + ] as Expression); } } @@ -984,6 +999,18 @@ export class Batch { } } + // this will use a version higher than any in-progress indexing in case there + // are artifacts left over from a failed index + private async setNextGenerationRealmVersion() { + let [maxVersionRow] = (await this.client.query([ + 'SELECT MAX(realm_version) as max_version FROM boxel_index WHERE realm_url =', + param(this.realmURL.href), + ])) as { max_version: number }[]; + let maxVersion = (maxVersionRow?.max_version ?? 0) + 1; + let nextVersion = Math.max(this.realmVersion, maxVersion); + this.realmVersion = nextVersion; + } + async invalidate(url: URL): Promise { await this.ready; let alias = trimExecutableExtension(url).href; @@ -1052,13 +1079,26 @@ export class Batch { ) { let message = `Invalidation conflict error in realm ${this.realmURL.href} version ${this.realmVersion}`; if (opts?.url && opts?.invalidations) { - message = `${message}: the invalidation ${ - opts.url.href - } resulted in invalidation graph: ${JSON.stringify( - opts.invalidations, - )} that collides with unfinished indexing`; + message = + `${message}: the invalidation ${ + opts.url.href + } resulted in invalidation graph: ${JSON.stringify( + opts.invalidations, + )} that collides with unfinished indexing. The most likely reason this happens is that there ` + + `was an error encountered during incremental indexing that prevented the indexing from completing ` + + `(and realm version increasing), then there was another incremental update to the same document ` + + `that collided with the WIP artifacts from the indexing that never completed. Removing the WIP ` + + `indexing artifacts (the rows(s) that triggered the unique constraint will solve the immediate ` + + `problem, but likely the issue that triggered the unfinished indexing will need to be fixed to ` + + `prevent this from happening in the future.`; } else if (opts?.isMakingNewGeneration) { - message = `${message}. created a new generation while there was still unfinished indexing`; + message = + `${message}. created a new generation while there was still unfinished indexing. ` + + `The most likely reason this happens is that there was an error encountered during incremental ` + + `indexing that prevented the indexing from completing (and realm version increasing), ` + + `then the realm was restarted and the left over WIP indexing artifact(s) collided with the ` + + `from-scratch indexing. To resolve this issue delete the WIP indexing artifacts (the row(s) ` + + `that triggered the unique constraint) and restart the realm.`; } throw new Error(message); } @@ -1066,7 +1106,13 @@ export class Batch { } } - private async calculateInvalidations(alias: string): Promise { + private async calculateInvalidations( + alias: string, + visited: string[] = [], + ): Promise { + if (visited.includes(alias)) { + return []; + } let childInvalidations = await this.client.itemsThatReference( alias, this.realmVersion, @@ -1077,7 +1123,9 @@ export class Batch { ...invalidations, ...flatten( await Promise.all( - aliases.map((alias) => this.calculateInvalidations(alias)), + aliases.map((a) => + this.calculateInvalidations(a, [...visited, alias]), + ), ), ), ]; diff --git a/packages/runtime-common/loader.ts b/packages/runtime-common/loader.ts index 1cf0609262..6e3f259f98 100644 --- a/packages/runtime-common/loader.ts +++ b/packages/runtime-common/loader.ts @@ -159,11 +159,15 @@ export class Loader { async getConsumedModules( moduleIdentifier: string, consumed = new Set(), + initialIdentifier = moduleIdentifier, ): Promise { if (consumed.has(moduleIdentifier)) { return []; } - consumed.add(moduleIdentifier); + // you can't consume yourself + if (moduleIdentifier !== initialIdentifier) { + consumed.add(moduleIdentifier); + } let resolvedModuleIdentifier = new URL(moduleIdentifier); let module = this.getModule(resolvedModuleIdentifier.href); @@ -189,7 +193,11 @@ export class Loader { return cached; } for (let consumedModule of module?.consumedModules ?? []) { - await this.getConsumedModules(consumedModule, consumed); + await this.getConsumedModules( + consumedModule, + consumed, + initialIdentifier, + ); } cached = [...consumed]; this.consumptionCache.set(module, cached); diff --git a/packages/runtime-common/package.json b/packages/runtime-common/package.json index 7250dbc15e..5ffb057af5 100644 --- a/packages/runtime-common/package.json +++ b/packages/runtime-common/package.json @@ -51,6 +51,7 @@ "safe-stable-stringify": "^2.4.3", "super-fast-md5": "^1.0.1", "transform-modules-amd-plugin": "workspace:*", + "typescript-memoize": "^1.1.1", "uuid": "^9.0.1" }, "devDependencies": { diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index baff517bf7..8711b4319b 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -1,9 +1,5 @@ import { Deferred } from './deferred'; -import { - SearchIndex, - type IndexRunner, - type RunnerOptionsManager, -} from './search-index'; +import { SearchIndex } from './search-index'; import { type SingleCardDocument } from './card-document'; import { Loader } from './loader'; import { RealmPaths, LocalPath, join } from './paths'; @@ -225,26 +221,10 @@ interface MessageEvent { id?: string; } -type Operation = WriteOperation | DeleteOperation; - interface WriteResult { lastModified: number; } -interface WriteOperation { - type: 'write'; - path: LocalPath; - contents: string; - clientRequestId?: string | null; // Used for client to be able to see if the SSE event is a result of the client's own write - deferred: Deferred; -} - -interface DeleteOperation { - type: 'delete'; - path: LocalPath; - deferred: Deferred; -} - export class Realm { #startedUp = new Deferred(); #matrixClient: MatrixClient; @@ -259,8 +239,6 @@ export class Realm { #updateItems: UpdateItem[] = []; #flushUpdateEvents: Promise | undefined; #recentWrites: Map = new Map(); - #flushOperations: Promise | undefined; - #operationQueue: Operation[] = []; #realmSecretSeed: string; #permissions: RealmPermissions; #realmAuthHandler: RealmAuthHandler; @@ -281,10 +259,6 @@ export class Realm { readonly loaderTemplate: Loader; readonly paths: RealmPaths; - private get isDbIndexerEnabled() { - return Boolean((globalThis as any).__enablePgIndexer?.()); - } - get url(): string { return this.paths.url; } @@ -301,10 +275,6 @@ export class Realm { { url, adapter, - // TODO remove this after feature flag is removed - indexRunner, - // TODO remove this after feature flag is removed - runnerOptsMgr, getIndexHTML, matrix, realmSecretSeed, @@ -316,14 +286,12 @@ export class Realm { }: { url: string; adapter: RealmAdapter; - indexRunner: IndexRunner; - runnerOptsMgr: RunnerOptionsManager; getIndexHTML: () => Promise; matrix: MatrixConfig; permissions: RealmPermissions; realmSecretSeed: string; - dbAdapter?: DBAdapter; - queue?: Queue; + dbAdapter: DBAdapter; + queue: Queue; virtualNetwork: VirtualNetwork; onIndexer?: (indexer: Indexer) => Promise; }, @@ -355,10 +323,6 @@ export class Realm { this.#onIndexer = onIndexer; this.#searchIndex = new SearchIndex({ realm: this, - readdir: this.#adapter.readdir.bind(this.#adapter), - readFileAsText: this.readFileAsText.bind(this), - runner: indexRunner, - runnerOptsManager: runnerOptsMgr, dbAdapter, queue, }); @@ -445,39 +409,6 @@ export class Realm { return this.#flushUpdateEvents; } - async flushOperations() { - return this.#flushOperations; - } - - // in order to prevent issues with concurrent index manipulation clobbering - // each other we use a queue of operations to mutate realm state. We should - // remove this queue when we move to a pg backed index - private async drainOperations() { - await this.#flushOperations; - - let operationsDrained: () => void; - this.#flushOperations = new Promise( - (res) => (operationsDrained = res), - ); - let operations = [...this.#operationQueue]; - this.#operationQueue = []; - for (let operation of operations) { - if (operation.type === 'write') { - let result = await this.#write( - operation.path, - operation.contents, - operation.clientRequestId, - ); - operation.deferred.fulfill(result); - } else { - await this.#delete(operation.path); - operation.deferred.fulfill(); - } - } - - operationsDrained!(); - } - createJWT(claims: TokenClaims, expiration: string): string { return this.#adapter.createJWT(claims, expiration, this.#realmSecretSeed); } @@ -486,27 +417,6 @@ export class Realm { path: LocalPath, contents: string, clientRequestId?: string | null, - ): Promise { - if (this.isDbIndexerEnabled) { - return await this.#write(path, contents, clientRequestId); - } else { - let deferred = new Deferred(); - this.#operationQueue.push({ - type: 'write', - path, - contents, - clientRequestId, - deferred, - }); - this.drainOperations(); - return deferred.promise; - } - } - - async #write( - path: LocalPath, - contents: string, - clientRequestId?: string | null, ): Promise { await this.trackOwnWrite(path); let results = await this.#adapter.write(path, contents); @@ -572,21 +482,6 @@ export class Realm { } async delete(path: LocalPath): Promise { - let deferred = new Deferred(); - this.#operationQueue.push({ - type: 'delete', - path, - deferred, - }); - if (this.isDbIndexerEnabled) { - return await this.#delete(path); - } else { - this.drainOperations(); - return deferred.promise; - } - } - - async #delete(path: LocalPath): Promise { await this.trackOwnWrite(path, { isDelete: true }); await this.#adapter.remove(path); await this.#searchIndex.update(this.paths.fileURL(path), { @@ -888,10 +783,7 @@ export class Realm { try { // local requests are allowed to query the realm as the index is being built up if (!isLocal) { - // allow any WIP index requests to query the index while it's building up - if (!request.headers.get('X-Boxel-Use-WIP-Index')) { - await this.ready; - } + await this.ready; let isWrite = ['PUT', 'PATCH', 'POST', 'DELETE'].includes( request.method, @@ -942,6 +834,9 @@ export class Realm { return undefined; } + // TODO we could really improve performance if this utilized the index instead + // of directly hitting the filesystem--especially the TS transpilation + // involved in making JS. async fallbackHandle(request: Request) { let url = new URL(request.url); let localPath = this.paths.local(url); diff --git a/packages/runtime-common/search-index.ts b/packages/runtime-common/search-index.ts index 4fd1b36937..9bb0626a8c 100644 --- a/packages/runtime-common/search-index.ts +++ b/packages/runtime-common/search-index.ts @@ -1,11 +1,10 @@ -import * as JSONTypes from 'json-typescript'; +import { Memoize } from 'typescript-memoize'; import { - baseRealm, SupportedMimeType, - internalKeyFor, maxLinkDepth, maybeURL, Indexer, + type Stats, type LooseCardResource, type DBAdapter, type Queue, @@ -16,24 +15,12 @@ import { type IncrementalArgs, type IncrementalResult, } from '.'; -import { Kind, Realm } from './realm'; -import { LocalPath, RealmPaths } from './paths'; +import { Realm } from './realm'; +import { RealmPaths } from './paths'; import { Loader } from './loader'; -import type { - Query, - Filter, - Sort, - EqFilter, - ContainsFilter, - RangeFilter, -} from './query'; +import type { Query } from './query'; import { CardError, type SerializedError } from './error'; -import { URLMap } from './url-map'; -import flatMap from 'lodash/flatMap'; import ignore, { type Ignore } from 'ignore'; -import type { BaseDef, Field } from 'https://cardstack.com/base/card-api'; -import type * as CardAPI from 'https://cardstack.com/base/card-api'; -import { type CodeRef, getField, identifyCard, loadCard } from './code-ref'; import { isSingleCardDocument, type SingleCardDocument, @@ -42,74 +29,6 @@ import { type Saved, } from './card-document'; -export interface Reader { - readFileAsText: ( - path: LocalPath, - opts?: { withFallbacks?: true }, - ) => Promise<{ content: string; lastModified: number } | undefined>; - readdir: ( - path: string, - ) => AsyncGenerator<{ name: string; path: string; kind: Kind }, void>; -} - -export interface Stats extends JSONTypes.Object { - instancesIndexed: number; - instanceErrors: number; - moduleErrors: number; -} - -export interface RunState { - realmURL: URL; - instances: URLMap; - ignoreMap: URLMap; - ignoreData: Record; - modules: Map; - stats: Stats; - invalidations: string[]; -} - -export type RunnerRegistration = ( - fromScratch: (realmURL: URL) => Promise, - incremental: ( - prev: RunState, - url: URL, - operation: 'update' | 'delete', - onInvalidation?: (invalidatedURLs: URL[]) => void, - ) => Promise, -) => Promise; - -export type EntrySetter = (url: URL, entry: SearchEntryWithErrors) => void; - -export interface RunnerOpts { - _fetch: typeof fetch; - reader: Reader; - entrySetter: EntrySetter; - registerRunner: RunnerRegistration; - // TODO make this required after feature flag is removed - indexer?: Indexer; -} -export type IndexRunner = (optsId: number) => Promise; - -export interface SearchEntry { - resource: CardResource; - searchData: Record; - html?: string; // we don't have this until after the indexer route is rendered... - types: string[]; - deps: Set; -} - -export type SearchEntryWithErrors = - | { type: 'entry'; entry: SearchEntry } - | { type: 'error'; error: SerializedError }; - -export interface Module { - url: string; - consumes: string[]; -} -export type ModuleWithErrors = - | { type: 'module'; module: Module } - | { type: 'error'; moduleURL: string; error: SerializedError }; - type Options = { loadLinks?: true; } & QueryOptions; @@ -124,370 +43,141 @@ interface SearchResultError { error: SerializedError; } -type CurrentIndex = RunState & { - loader: Loader; -}; - -// This class is used to support concurrent index runs against the same fastboot -// instance. While each index run calls visit on the fastboot instance and has -// its own memory space, the globals that are passed into fastboot are shared. -// This global is what holds loader context (specifically the loader fetch) and -// index mutators for the fastboot instance. each index run will have a -// different loader fetch and its own index mutator. in order to keep these from -// colliding during concurrent indexing we hold each set of fastboot globals in -// a map that is unique for the index run. When the server visits fastboot it -// will provide the indexer route with the id for the fastboot global that is -// specific to the index run. -let optsId = 0; -export class RunnerOptionsManager { - #opts = new Map(); - setOptions(opts: RunnerOpts): number { - let id = optsId++; - this.#opts.set(id, opts); - return id; - } - getOptions(id: number): RunnerOpts { - let opts = this.#opts.get(id); - if (!opts) { - throw new Error(`No runner opts for id ${id}`); - } - return opts; - } - removeOptions(id: number) { - this.#opts.delete(id); - } -} - export class SearchIndex { #realm: Realm; - #runner: IndexRunner; - runnerOptsMgr: RunnerOptionsManager; - #reader: Reader; - #index: CurrentIndex; - // TODO make this required after we remove the feature flag - #indexer: Indexer | undefined; - // TODO make this required after we remove the feature flag - #queue: Queue | undefined; - #fromScratch: ((realmURL: URL) => Promise) | undefined; - #incremental: - | (( - prev: RunState, - url: URL, - operation: 'update' | 'delete', - onInvalidation?: (invalidatedURLs: URL[]) => void, - ) => Promise) - | undefined; + #loader: Loader; + #ignoreData: Record = {}; + #stats: Stats = { + instancesIndexed: 0, + instanceErrors: 0, + moduleErrors: 0, + }; + #indexer: Indexer; + #queue: Queue; constructor({ realm, - readdir, - readFileAsText, - runner, - runnerOptsManager, dbAdapter, queue, }: { realm: Realm; - readdir: ( - path: string, - ) => AsyncGenerator<{ name: string; path: string; kind: Kind }, void>; - readFileAsText: ( - path: LocalPath, - opts?: { withFallbacks?: true }, - ) => Promise<{ content: string; lastModified: number } | undefined>; - runner: IndexRunner; - runnerOptsManager: RunnerOptionsManager; - dbAdapter?: DBAdapter; - queue?: Queue; + dbAdapter: DBAdapter; + queue: Queue; }) { - if (this.isDbIndexerEnabled) { - if (!dbAdapter) { - throw new Error( - `DB Adapter was not provided to SearchIndex constructor--this is required when using a db based index`, - ); - } - this.#indexer = new Indexer(dbAdapter); + if (!dbAdapter) { + throw new Error( + `DB Adapter was not provided to SearchIndex constructor--this is required when using a db based index`, + ); } + this.#indexer = new Indexer(dbAdapter); this.#queue = queue; this.#realm = realm; - this.#reader = { readdir, readFileAsText }; - this.runnerOptsMgr = runnerOptsManager; - this.#runner = runner; - this.#index = { - realmURL: new URL(realm.url), - loader: Loader.cloneLoader(realm.loaderTemplate), - ignoreMap: new URLMap(), - ignoreData: new Object(null) as Record, - instances: new URLMap(), - modules: new Map(), - invalidations: [], - stats: { - instancesIndexed: 0, - instanceErrors: 0, - moduleErrors: 0, - }, - }; - } - - private get isDbIndexerEnabled() { - return Boolean((globalThis as any).__enablePgIndexer?.()); - } - - // TODO we can get rid of this after the feature flag is removed. this is just - // some type sugar so we don't have to check to see if the indexer exists - // since ultimately it will be required. - private get indexer() { - if (!this.#indexer) { - throw new Error(`Indexer is missing`); - } - return this.#indexer; - } - - // TODO remove after feature flag, same reason as above - private get queue() { - if (!this.#queue) { - throw new Error(`Queue is missing`); - } - return this.#queue; + this.#loader = Loader.cloneLoader(this.#realm.loaderTemplate); } get stats() { - return this.#index.stats; + return this.#stats; } get loader() { - return this.#index.loader; + return this.#loader; } - get runState() { - return this.#index; + @Memoize() + private get realmURL() { + return new URL(this.#realm.url); } - async run(onIndexer?: (indexer: Indexer) => Promise) { - if (this.isDbIndexerEnabled) { - await this.queue.start(); - await this.indexer.ready(); - if (onIndexer) { - await onIndexer(this.indexer); - } + @Memoize() + private get ignoreMap() { + let ignoreMap = new Map(); + for (let [url, contents] of Object.entries(this.#ignoreData)) { + ignoreMap.set(url, ignore().add(contents)); + } + return ignoreMap; + } - let args: FromScratchArgs = { - realmURL: this.#realm.url, - }; - let job = await this.queue.publish( - `from-scratch-index:${this.#realm.url}`, - args, - ); - let { ignoreData, stats } = await job.done; - let ignoreMap = new URLMap(); - for (let [url, contents] of Object.entries(ignoreData)) { - ignoreMap.set(new URL(url), ignore().add(contents)); - } - // TODO clean this up after we remove feature flag. For now I'm just - // including the bare minimum to keep this from blowing up using the old APIs - this.#index = { - stats, - ignoreMap, - realmURL: new URL(this.#realm.url), - ignoreData, - instances: new URLMap(), - modules: new Map(), - invalidations: [], - loader: Loader.cloneLoader(this.#realm.loaderTemplate), - }; - } else { - await this.setupRunner(async () => { - if (!this.#fromScratch) { - throw new Error(`Index runner has not been registered`); - } - let current = await this.#fromScratch(this.#index.realmURL); - this.#index = { - ...this.#index, // don't clobber the instances that the entrySetter has already made - modules: current.modules, - ignoreMap: current.ignoreMap, - realmURL: current.realmURL, - stats: current.stats, - loader: Loader.cloneLoader(this.#realm.loaderTemplate), - }; - }); + async run(onIndexer?: (indexer: Indexer) => Promise) { + await this.#queue.start(); + await this.#indexer.ready(); + if (onIndexer) { + await onIndexer(this.#indexer); } + + let args: FromScratchArgs = { + realmURL: this.#realm.url, + }; + let job = await this.#queue.publish( + `from-scratch-index:${this.#realm.url}`, + args, + ); + let { ignoreData, stats } = await job.done; + this.#stats = stats; + this.#ignoreData = ignoreData; + this.#loader = Loader.cloneLoader(this.#realm.loaderTemplate); } async update( url: URL, opts?: { delete?: true; onInvalidation?: (invalidatedURLs: URL[]) => void }, ): Promise { - if (this.isDbIndexerEnabled) { - let args: IncrementalArgs = { - url: url.href, - realmURL: this.#realm.url, - operation: opts?.delete ? 'delete' : 'update', - ignoreData: { ...this.#index.ignoreData }, - }; - let job = await this.queue.publish( - `incremental-index:${this.#realm.url}`, - args, + let args: IncrementalArgs = { + url: url.href, + realmURL: this.#realm.url, + operation: opts?.delete ? 'delete' : 'update', + ignoreData: { ...this.#ignoreData }, + }; + let job = await this.#queue.publish( + `incremental-index:${this.#realm.url}`, + args, + ); + let { invalidations, ignoreData, stats } = await job.done; + this.#stats = stats; + this.#ignoreData = ignoreData; + this.#loader = Loader.cloneLoader(this.#realm.loaderTemplate); + if (opts?.onInvalidation) { + opts.onInvalidation( + invalidations.map((href) => new URL(href.replace(/\.json$/, ''))), ); - let { invalidations, ignoreData, stats } = await job.done; - let ignoreMap = new URLMap(); - for (let [url, contents] of Object.entries(ignoreData)) { - ignoreMap.set(new URL(url), ignore().add(contents)); - } - // TODO clean this up after we remove feature flag. For now I'm just - // including the bare minimum to keep this from blowing up using the old APIs - this.#index = { - stats, - ignoreMap, - ignoreData, - invalidations, - realmURL: new URL(this.#realm.url), - instances: new URLMap(), - modules: new Map(), - loader: Loader.cloneLoader(this.#realm.loaderTemplate), - }; - if (opts?.onInvalidation) { - opts.onInvalidation( - invalidations.map((href) => new URL(href.replace(/\.json$/, ''))), - ); - } - } else { - await this.setupRunner(async () => { - if (!this.#incremental) { - throw new Error(`Index runner has not been registered`); - } - // TODO this should be published into the queue - let current = await this.#incremental( - this.#index, - url, - opts?.delete ? 'delete' : 'update', - opts?.onInvalidation, - ); - // TODO we should handle onInvalidation here in the case where we are doing db based index - - this.#index = { - // we overwrite the instances in the incremental update, as there may - // have been instance removals due to invalidation that the entrySetter - // cannot accommodate in its current form - instances: current.instances, - modules: current.modules, - ignoreMap: current.ignoreMap, - ignoreData: current.ignoreData, - realmURL: current.realmURL, - stats: current.stats, - invalidations: current.invalidations, - loader: Loader.cloneLoader(this.#realm.loaderTemplate), - }; - }); } } - // TODO I think we can break this out into a different module specifically a - // queue handler for incremental and fromScratch indexing - private async setupRunner(start: () => Promise) { - let optsId = this.runnerOptsMgr.setOptions({ - _fetch: this.loader.fetch.bind(this.loader), - reader: this.#reader, - entrySetter: (url, entry) => { - this.#index.instances.set(url, entry); - }, - registerRunner: async (fromScratch, incremental) => { - this.#fromScratch = fromScratch; - this.#incremental = incremental; - await start(); - }, - ...(this.isDbIndexerEnabled - ? { - indexer: this.indexer, - } - : {}), - }); - await this.#runner(optsId); - this.runnerOptsMgr.removeOptions(optsId); - } - async search(query: Query, opts?: Options): Promise { let doc: CardCollectionDocument; - if (this.isDbIndexerEnabled) { - let { cards: data, meta: _meta } = await this.indexer.search( - new URL(this.#realm.url), - query, - this.loader, - opts, - ); - doc = { - data: data.map((resource) => ({ - ...resource, - ...{ links: { self: resource.id } }, - })), - }; - - let omit = doc.data.map((r) => r.id); - // TODO eventually the links will be cached in the index, and this will only - // fill in the included resources for links that were not cached (e.g. - // volatile fields) - if (opts?.loadLinks) { - let included: CardResource[] = []; - for (let resource of doc.data) { - included = await this.loadLinks( - { - realmURL: this.#index.realmURL, - resource, - omit, - included, - }, - opts, - ); - } - if (included.length > 0) { - doc.included = included; - } - } - } else { - let matcher = await this.buildMatcher(query.filter, { - module: `${baseRealm.url}card-api`, - name: 'CardDef', - }); - - // fallback to always sorting by id - query.sort = query.sort ?? []; - query.sort.push({ - by: 'id', - on: { module: `${baseRealm.url}card-api`, name: 'CardDef' }, - }); - doc = { - data: flatMap([...this.#index.instances.values()], (maybeError) => - maybeError.type !== 'error' ? [maybeError.entry] : [], - ) - .filter(matcher) - .sort(this.buildSorter(query.sort)) - .map((entry) => ({ - ...entry.resource, - ...{ links: { self: entry.resource.id } }, - })), - }; + let { cards: data, meta: _meta } = await this.#indexer.search( + new URL(this.#realm.url), + query, + this.loader, + opts, + ); + doc = { + data: data.map((resource) => ({ + ...resource, + ...{ links: { self: resource.id } }, + })), + }; - let omit = doc.data.map((r) => r.id); - // TODO eventually the links will be cached in the index, and this will only - // fill in the included resources for links that were not cached (e.g. - // volatile fields) - if (opts?.loadLinks) { - let included: CardResource[] = []; - for (let resource of doc.data) { - included = await loadLinksForInMemoryIndex({ - realmURL: this.#index.realmURL, - instances: this.#index.instances, - loader: this.loader, + let omit = doc.data.map((r) => r.id); + // TODO eventually the links will be cached in the index, and this will only + // fill in the included resources for links that were not cached (e.g. + // volatile fields) + if (opts?.loadLinks) { + let included: CardResource[] = []; + for (let resource of doc.data) { + included = await this.loadLinks( + { + realmURL: this.realmURL, resource, omit, included, - }); - } - if (included.length > 0) { - doc.included = included; - } + }, + opts, + ); + } + if (included.length > 0) { + doc.included = included; } } - return doc; } @@ -501,68 +191,37 @@ export class SearchIndex { ) { return true; } - return isIgnored(this.#index.realmURL, this.#index.ignoreMap, url); + return isIgnored(this.realmURL, this.ignoreMap, url); } async card(url: URL, opts?: Options): Promise { let doc: SingleCardDocument | undefined; - if (this.isDbIndexerEnabled) { - let maybeCard = await this.indexer.getCard(url, opts); - if (!maybeCard) { - return undefined; - } - if (maybeCard.type === 'error') { - return maybeCard; - } - doc = { - data: { ...maybeCard.card, ...{ links: { self: url.href } } }, - }; - if (!doc) { - throw new Error( - `bug: should never get here--search index doc is undefined`, - ); - } - if (opts?.loadLinks) { - let included = await this.loadLinks( - { - realmURL: this.#index.realmURL, - resource: doc.data, - omit: [doc.data.id], - }, - opts, - ); - if (included.length > 0) { - doc.included = included; - } - } - } else { - let card = this.#index.instances.get(url); - if (!card) { - return undefined; - } - if (card.type === 'error') { - return card; - } - doc = { - data: { ...card.entry.resource, ...{ links: { self: url.href } } }, - }; - - if (!doc) { - throw new Error( - `bug: should never get here--search index doc is undefined`, - ); - } - if (opts?.loadLinks) { - let included = await loadLinksForInMemoryIndex({ - realmURL: this.#index.realmURL, - instances: this.#index.instances, - loader: this.loader, + let maybeCard = await this.#indexer.getCard(url, opts); + if (!maybeCard) { + return undefined; + } + if (maybeCard.type === 'error') { + return maybeCard; + } + doc = { + data: { ...maybeCard.card, ...{ links: { self: url.href } } }, + }; + if (!doc) { + throw new Error( + `bug: should never get here--search index doc is undefined`, + ); + } + if (opts?.loadLinks) { + let included = await this.loadLinks( + { + realmURL: this.realmURL, resource: doc.data, omit: [doc.data.id], - }); - if (included.length > 0) { - doc.included = included; - } + }, + opts, + ); + if (included.length > 0) { + doc.included = included; } } return { type: 'doc', doc }; @@ -570,44 +229,11 @@ export class SearchIndex { // this is meant for tests only async searchEntry(url: URL): Promise { - if (this.isDbIndexerEnabled) { - let result = await this.indexer.getCard(url); - if (result?.type !== 'error') { - return result; - } - } else { - let result = this.#index.instances.get(url); - if (!result) { - return undefined; - } - if (result?.type !== 'error') { - return { - type: 'card', - card: result.entry.resource, - // search docs will now be persisted in JSONB objects--this means that - // `undefined` values will no longer be represented since `undefined` - // does not exist in JSON and it is not the same as `null` - searchDoc: JSON.parse(JSON.stringify(result.entry.searchData)), - isolatedHtml: result.entry.html ?? null, - realmVersion: -1, - realmURL: this.#realm.url, - types: result.entry.types, - indexedAt: 0, - deps: [...result.entry.deps], - }; - } + let result = await this.#indexer.getCard(url); + if (result?.type === 'error') { + return undefined; } - return undefined; - } - - private loadAPI(): Promise { - return this.loader.import(`${baseRealm.url}card-api`); - } - - private cardHasType(entry: SearchEntry, ref: CodeRef): boolean { - return Boolean( - entry.types?.find((t) => t === internalKeyFor(ref, undefined)), // assumes ref refers to absolute module URL - ); + return result; } // TODO The caller should provide a list of fields to be included via JSONAPI @@ -650,7 +276,7 @@ export class SearchIndex { ); let linkResource: CardResource | undefined; if (realmPath.inRealm(linkURL)) { - let maybeResult = await this.indexer.getCard(linkURL, opts); + let maybeResult = await this.#indexer.getCard(linkURL, opts); linkResource = maybeResult?.type === 'card' ? maybeResult.card : undefined; } else { @@ -721,436 +347,11 @@ export class SearchIndex { } return included; } - - private async loadField(ref: CodeRef, fieldPath: string): Promise { - let composite: typeof BaseDef | undefined; - try { - composite = await loadCard(ref, { loader: this.loader }); - } catch (err: any) { - if (!('type' in ref)) { - throw new Error( - `Your filter refers to nonexistent type: import ${ - ref.name === 'default' ? 'default' : `{ ${ref.name} }` - } from "${ref.module}"`, - ); - } else { - throw new Error( - `Your filter refers to nonexistent type: ${JSON.stringify( - ref, - null, - 2, - )}`, - ); - } - } - let segments = fieldPath.split('.'); - let field: Field | undefined; - while (segments.length) { - let fieldName = segments.shift()!; - let prevField = field; - field = getField(composite, fieldName); - if (!field) { - throw new Error( - `Your filter refers to nonexistent field "${fieldName}" on type ${JSON.stringify( - identifyCard(prevField ? prevField.card : composite), - )}`, - ); - } - } - return field!; - } - - private getFieldData(searchData: Record, fieldPath: string) { - let data = searchData; - let segments = fieldPath.split('.'); - while (segments.length && data != null) { - let fieldName = segments.shift()!; - data = data[fieldName]; - } - return data; - } - - private buildSorter( - expressions: Sort | undefined, - ): (e1: SearchEntry, e2: SearchEntry) => number { - if (!expressions || expressions.length === 0) { - return () => 0; - } - let sorters = expressions.map(({ by, on, direction }) => { - return (e1: SearchEntry, e2: SearchEntry) => { - if (!this.cardHasType(e1, on)) { - return direction === 'desc' ? -1 : 1; - } - if (!this.cardHasType(e2, on)) { - return direction === 'desc' ? 1 : -1; - } - - let a = this.getFieldData(e1.searchData, by); - let b = this.getFieldData(e2.searchData, by); - if (a === undefined) { - return direction === 'desc' ? -1 : 1; // if descending, null position is before the rest - } - if (b === undefined) { - return direction === 'desc' ? 1 : -1; // `a` is not null - } - if (a < b) { - return direction === 'desc' ? 1 : -1; - } else if (a > b) { - return direction === 'desc' ? -1 : 1; - } else { - return 0; - } - }; - }); - - return (e1: SearchEntry, e2: SearchEntry) => { - for (let sorter of sorters) { - let result = sorter(e1, e2); - if (result !== 0) { - return result; - } - } - return 0; - }; - } - - // Matchers are three-valued (true, false, null) because a query that talks - // about a field that is not even present on a given card results in `null` to - // distinguish it from a field that is present but not matching the filter - // (`false`) - private async buildMatcher( - filter: Filter | undefined, - onRef: CodeRef, - ): Promise<(entry: SearchEntry) => boolean | null> { - if (!filter) { - return (_entry) => true; - } - - if ('type' in filter) { - return (entry) => this.cardHasType(entry, filter.type); - } - - let on = filter?.on ?? onRef; - - if ('any' in filter) { - let matchers = await Promise.all( - filter.any.map((f) => this.buildMatcher(f, on)), - ); - return (entry) => some(matchers, (m) => m(entry)); - } - - if ('every' in filter) { - let matchers = await Promise.all( - filter.every.map((f) => this.buildMatcher(f, on)), - ); - return (entry) => every(matchers, (m) => m(entry)); - } - - if ('not' in filter) { - let matcher = await this.buildMatcher(filter.not, on); - return (entry) => { - let inner = matcher(entry); - if (inner == null) { - // irrelevant cards stay irrelevant, even when the query is inverted - return null; - } else { - return !inner; - } - }; - } - - if ('eq' in filter || 'contains' in filter) { - return await this.buildEqOrContainsMatchers(filter, on); - } - - if ('range' in filter) { - return await this.buildRangeMatchers(filter.range, on); - } - - throw new Error('Unknown filter'); - } - - private async buildRangeMatchers( - range: RangeFilter['range'], - ref: CodeRef, - ): Promise<(entry: SearchEntry) => boolean | null> { - // TODO when we are ready to execute queries within computeds, we'll need to - // use the loader instance from current-run and not the global loader, as - // the card definitions may have changed in the current-run loader - let api = await this.loadAPI(); - - let matchers: ((instanceData: Record) => boolean | null)[] = - []; - - for (let [name, value] of Object.entries(range)) { - // Load the stack of fields we're accessing - let fields: Field[] = []; - let nextRef: CodeRef | undefined = ref; - let segments = name.split('.'); - while (segments.length > 0) { - let fieldName = segments.shift()!; - let field = await this.loadField(nextRef, fieldName); - fields.push(field); - nextRef = identifyCard(field.card); - if (!nextRef) { - throw new Error(`could not identify card for field ${fieldName}`); - } - } - - let qValueGT = api.formatQueryValue(fields[fields.length - 1], value.gt); - let qValueLT = api.formatQueryValue(fields[fields.length - 1], value.lt); - let qValueGTE = api.formatQueryValue( - fields[fields.length - 1], - value.gte, - ); - let qValueLTE = api.formatQueryValue( - fields[fields.length - 1], - value.lte, - ); - let queryValue = qValueGT ?? qValueLT ?? qValueGTE ?? qValueLTE; - - let matcher = (instanceValue: any) => { - if (instanceValue == null || queryValue == null) { - return null; - } - // checking for not null below is necessary because queryValue can be 0 - if ( - (qValueGT != null && !(instanceValue > qValueGT)) || - (qValueLT != null && !(instanceValue < qValueLT)) || - (qValueGTE != null && !(instanceValue >= qValueGTE)) || - (qValueLTE != null && !(instanceValue <= qValueLTE)) - ) { - return false; - } - return true; - }; - - while (fields.length > 0) { - let nextField = fields.pop()!; - let nextMatcher = nextField.queryMatcher(matcher); - matcher = (instanceValue: any) => { - if (instanceValue == null || queryValue == null) { - return null; - } - return nextMatcher(instanceValue[nextField.name]); - }; - } - matchers.push(matcher); - } - - return (entry) => - every(matchers, (m) => { - if (this.cardHasType(entry, ref)) { - return m(entry.searchData); - } - return null; - }); - } - - private async buildEqOrContainsMatchers( - filter: EqFilter | ContainsFilter, - ref: CodeRef, - ): Promise<(entry: SearchEntry) => boolean | null> { - let filterType: 'eq' | 'contains'; - let filterValue: EqFilter['eq'] | ContainsFilter['contains']; - if ('eq' in filter) { - filterType = 'eq'; - filterValue = filter.eq; - } else if ('contains' in filter) { - filterType = 'contains'; - filterValue = filter.contains; - } else { - throw new Error('Invalid filter type'); - } - // TODO when we are ready to execute queries within computeds, we'll need to - // use the loader instance from current-run and not the global loader, as - // the card definitions may have changed in the current-run loader - let api = await this.loadAPI(); - - let matchers: ((instanceData: Record) => boolean | null)[] = - []; - - for (let [name, value] of Object.entries(filterValue)) { - // Load the stack of fields we're accessing - let fields: Field[] = []; - let nextRef: CodeRef | undefined = ref; - let segments = name.split('.'); - while (segments.length > 0) { - let fieldName = segments.shift()!; - let field = await this.loadField(nextRef, fieldName); - fields.push(field); - nextRef = identifyCard(field.card); - if (!nextRef) { - throw new Error(`could not identify card for field ${fieldName}`); - } - } - - let queryValue = api.formatQueryValue(fields[fields.length - 1], value); - let matcher: (instanceValue: any) => boolean | null; - if (filterType === 'eq') { - matcher = (instanceValue: any) => { - if (instanceValue === undefined && queryValue != null) { - return null; - } - // allows queries for null to work - if (queryValue == null && instanceValue == null) { - return true; - } - return instanceValue === queryValue; - }; - } else { - matcher = (instanceValue: any) => { - if ( - (instanceValue == null && queryValue != null) || - (instanceValue != null && queryValue == null) - ) { - return null; - } - if (instanceValue == null && queryValue == null) { - return true; - } - return (instanceValue as string) - .toLowerCase() - .includes((queryValue as string).toLowerCase()); - }; - } - while (fields.length > 0) { - let nextField = fields.pop()!; - let nextMatcher = nextField.queryMatcher(matcher); - matcher = (instanceValue: any) => { - if (instanceValue == null && queryValue != null) { - return null; - } - if (instanceValue == null && queryValue == null) { - return true; - } - return nextMatcher(instanceValue[nextField.name]); - }; - } - matchers.push(matcher); - } - - return (entry) => - every(matchers, (m) => { - if (this.cardHasType(entry, ref)) { - return m(entry.searchData); - } - return null; - }); - } -} - -// TODO The caller should provide a list of fields to be included via JSONAPI -// request. currently we just use the maxLinkDepth to control how deep to load -// links -export async function loadLinksForInMemoryIndex({ - realmURL, - instances, - loader, - resource, - omit = [], - included = [], - visited = [], - stack = [], -}: { - realmURL: URL; - instances: URLMap; - loader: Loader; - resource: LooseCardResource; - omit?: string[]; - included?: CardResource[]; - visited?: string[]; - stack?: string[]; -}): Promise[]> { - if (resource.id != null) { - if (visited.includes(resource.id)) { - return []; - } - visited.push(resource.id); - } - let realmPath = new RealmPaths(realmURL); - for (let [fieldName, relationship] of Object.entries( - resource.relationships ?? {}, - )) { - if (!relationship.links.self) { - continue; - } - let linkURL = new URL( - relationship.links.self, - resource.id ? new URL(resource.id) : realmURL, - ); - let linkResource: CardResource | undefined; - if (realmPath.inRealm(linkURL)) { - let maybeEntry = instances.get(linkURL); - linkResource = - maybeEntry?.type === 'entry' ? maybeEntry.entry.resource : undefined; - } else { - let response = await loader.fetch(linkURL, { - headers: { Accept: SupportedMimeType.CardJson }, - }); - if (!response.ok) { - let cardError = await CardError.fromFetchResponse( - linkURL.href, - response, - ); - throw cardError; - } - let json = await response.json(); - if (!isSingleCardDocument(json)) { - throw new Error( - `instance ${ - linkURL.href - } is not a card document. it is: ${JSON.stringify(json, null, 2)}`, - ); - } - linkResource = { ...json.data, ...{ links: { self: json.data.id } } }; - } - let foundLinks = false; - // TODO stop using maxLinkDepth. we should save the JSON-API doc in the - // index based on keeping track of the rendered fields and invalidate the - // index as consumed cards change - if (linkResource && stack.length <= maxLinkDepth) { - for (let includedResource of await loadLinksForInMemoryIndex({ - realmURL, - instances, - loader, - resource: linkResource, - omit, - included: [...included, linkResource], - visited, - stack: [...(resource.id != null ? [resource.id] : []), ...stack], - })) { - foundLinks = true; - if ( - !omit.includes(includedResource.id) && - !included.find((r) => r.id === includedResource.id) - ) { - included.push({ - ...includedResource, - ...{ links: { self: includedResource.id } }, - }); - } - } - } - let relationshipId = maybeURL(relationship.links.self, resource.id); - if (!relationshipId) { - throw new Error( - `bug: unable to turn relative URL '${relationship.links.self}' into an absolute URL relative to ${resource.id}`, - ); - } - if (foundLinks || omit.includes(relationshipId.href)) { - resource.relationships![fieldName].data = { - type: 'card', - id: relationshipId.href, - }; - } - } - return included; } export function isIgnored( realmURL: URL, - ignoreMap: URLMap, + ignoreMap: Map, url: URL, ): boolean { if (url.href === realmURL.href) { @@ -1164,7 +365,7 @@ export function isIgnored( } // Test URL against closest ignore. (Should the ignores cascade? so that the // child ignore extends the parent ignore?) - let ignoreURLs = [...ignoreMap.keys()].map((u) => u.href); + let ignoreURLs = [...ignoreMap.keys()]; let matchingIgnores = ignoreURLs.filter((u) => url.href.includes(u)); let ignoreURL = matchingIgnores.sort((a, b) => b.length - a.length)[0] as | string @@ -1172,44 +373,8 @@ export function isIgnored( if (!ignoreURL) { return false; } - let ignore = ignoreMap.get(new URL(ignoreURL))!; + let ignore = ignoreMap.get(ignoreURL)!; let realmPath = new RealmPaths(realmURL); let pathname = realmPath.local(url); return ignore.test(pathname).ignored; } - -// three-valued version of Array.every that propagates nulls. Here, the presence -// of any nulls causes the whole thing to be null. -function every( - list: T[], - predicate: (t: T) => boolean | null, -): boolean | null { - let result = true; - for (let element of list) { - let status = predicate(element); - if (status == null) { - return null; - } - result = result && status; - } - return result; -} - -// three-valued version of Array.some that propagates nulls. Here, the whole -// expression becomes null only if the whole input is null. -function some( - list: T[], - predicate: (t: T) => boolean | null, -): boolean | null { - let result: boolean | null = null; - for (let element of list) { - let status = predicate(element); - if (status === true) { - return true; - } - if (status === false) { - result = false; - } - } - return result; -} diff --git a/packages/runtime-common/tests/indexer-test.ts b/packages/runtime-common/tests/indexer-test.ts index 5b39fb699d..65ca5dd2a0 100644 --- a/packages/runtime-common/tests/indexer-test.ts +++ b/packages/runtime-common/tests/indexer-test.ts @@ -323,7 +323,7 @@ const tests = Object.freeze({ }, 'can prevent concurrent batch invalidations from colliding when making new generation': - async (assert, { indexer }) => { + async (assert, { indexer, adapter }) => { await setupIndex( indexer, [{ realm_url: testRealmURL, current_version: 1 }], @@ -334,6 +334,12 @@ const tests = Object.freeze({ realm_url: testRealmURL, deps: [], }, + { + url: `${testRealmURL}2.json`, + realm_version: 1, + realm_url: testRealmURL, + deps: [], + }, ], ); @@ -341,16 +347,79 @@ const tests = Object.freeze({ let batch1 = await indexer.createBatch(new URL(testRealmURL)); let batch2 = await indexer.createBatch(new URL(testRealmURL)); await batch1.invalidate(new URL(`${testRealmURL}1.json`)); + { + let index = await adapter.execute( + 'SELECT url, realm_url, realm_version, is_deleted FROM boxel_index ORDER BY url COLLATE "POSIX", realm_version', + { coerceTypes: { is_deleted: 'BOOLEAN' } }, + ); + assert.deepEqual( + index, + [ + { + url: `${testRealmURL}1.json`, + realm_url: testRealmURL, + realm_version: 1, + is_deleted: null, + }, + { + url: `${testRealmURL}1.json`, + realm_url: testRealmURL, + realm_version: 2, + is_deleted: true, + }, + { + url: `${testRealmURL}2.json`, + realm_version: 1, + realm_url: testRealmURL, + is_deleted: null, + }, + ], + 'the index entries are correct', + ); + } - try { - await batch2.makeNewGeneration(); - throw new Error(`expected invalidation conflict error`); - } catch (e: any) { - assert.ok( - e.message.includes( - 'Invalidation conflict error in realm http://test-realm/test/ version 2', - ), - 'received invalidation conflict error', + // this will force batch2 to have a higher version number than batch 1 + await batch2.makeNewGeneration(); + { + let index = await adapter.execute( + 'SELECT url, realm_url, realm_version, is_deleted FROM boxel_index ORDER BY url COLLATE "POSIX", realm_version', + { coerceTypes: { is_deleted: 'BOOLEAN' } }, + ); + assert.deepEqual( + index, + [ + { + url: `${testRealmURL}1.json`, + realm_url: testRealmURL, + realm_version: 1, + is_deleted: null, + }, + { + url: `${testRealmURL}1.json`, + realm_url: testRealmURL, + realm_version: 2, + is_deleted: true, + }, + { + url: `${testRealmURL}1.json`, + realm_url: testRealmURL, + realm_version: 3, + is_deleted: true, + }, + { + url: `${testRealmURL}2.json`, + realm_version: 1, + realm_url: testRealmURL, + is_deleted: null, + }, + { + url: `${testRealmURL}2.json`, + realm_version: 3, + realm_url: testRealmURL, + is_deleted: true, + }, + ], + 'the index entries are correct', ); } }, diff --git a/packages/runtime-common/url-map.ts b/packages/runtime-common/url-map.ts deleted file mode 100644 index 85931e58b6..0000000000 --- a/packages/runtime-common/url-map.ts +++ /dev/null @@ -1,52 +0,0 @@ -export class URLMap { - #map: Map; - constructor(); - constructor(mapTuple: [key: URL, value: T][]); - constructor(map: URLMap); - constructor(mapInit: URLMap | [key: URL, value: T][] = []) { - if (!Array.isArray(mapInit)) { - mapInit = [...mapInit]; - } - this.#map = new Map(mapInit.map(([key, value]) => [key.href, value])); - } - has(url: URL): boolean { - return this.#map.has(url.href); - } - get(url: URL): T | undefined { - return this.#map.get(url.href); - } - set(url: URL, value: T) { - return this.#map.set(url.href, value); - } - get [Symbol.iterator]() { - // eslint-disable-next-line @typescript-eslint/no-this-alias - let self = this; - return function* () { - for (let [key, value] of self.#map) { - yield [new URL(key), value] as [URL, T]; - } - }; - } - values() { - return this.#map.values(); - } - keys() { - // eslint-disable-next-line @typescript-eslint/no-this-alias - let self = this; - return { - get [Symbol.iterator]() { - return function* () { - for (let key of self.#map.keys()) { - yield new URL(key); - } - }; - }, - }; - } - get size() { - return this.#map.size; - } - remove(url: URL) { - return this.#map.delete(url.href); - } -} diff --git a/packages/runtime-common/virtual-network.ts b/packages/runtime-common/virtual-network.ts index 5417709fe3..c96f12cc2c 100644 --- a/packages/runtime-common/virtual-network.ts +++ b/packages/runtime-common/virtual-network.ts @@ -1,6 +1,6 @@ import { RealmPaths } from './paths'; import { Loader, followRedirections } from './loader'; -import type { RunnerOpts } from './search-index'; +import type { RunnerOpts } from './worker'; import { PackageShimHandler, PACKAGES_FAKE_ORIGIN, diff --git a/packages/runtime-common/worker.ts b/packages/runtime-common/worker.ts index ce3f9890fe..09c3d32c50 100644 --- a/packages/runtime-common/worker.ts +++ b/packages/runtime-common/worker.ts @@ -1,5 +1,4 @@ import * as JSONTypes from 'json-typescript'; -import ignore, { type Ignore } from 'ignore'; import { Indexer, Loader, @@ -9,14 +8,46 @@ import { type LocalPath, type RealmAdapter, } from '.'; -import { - type IndexRunner, - type RunnerOptionsManager, - type RunState, - type Stats, - type Reader, // TODO move this type here -} from './search-index'; -import { URLMap } from './url-map'; +import { Kind } from './realm'; + +export interface Stats extends JSONTypes.Object { + instancesIndexed: number; + instanceErrors: number; + moduleErrors: number; +} + +export interface IndexResults { + ignoreData: Record; + stats: Stats; + invalidations: string[]; +} + +export interface Reader { + readFileAsText: ( + path: LocalPath, + opts?: { withFallbacks?: true }, + ) => Promise<{ content: string; lastModified: number } | undefined>; + readdir: ( + path: string, + ) => AsyncGenerator<{ name: string; path: string; kind: Kind }, void>; +} + +export type RunnerRegistration = ( + fromScratch: (realmURL: URL) => Promise, + incremental: ( + url: URL, + realmURL: URL, + operation: 'update' | 'delete', + ignoreData: Record, + ) => Promise, +) => Promise; + +export interface RunnerOpts { + _fetch: typeof fetch; + reader: Reader; + registerRunner: RunnerRegistration; + indexer: Indexer; +} export interface FromScratchArgs extends JSONTypes.Object { realmURL: string; @@ -39,6 +70,37 @@ export interface IncrementalResult { ignoreData: Record; stats: Stats; } +export type IndexRunner = (optsId: number) => Promise; + +// This class is used to support concurrent index runs against the same fastboot +// instance. While each index run calls visit on the fastboot instance and has +// its own memory space, the globals that are passed into fastboot are shared. +// This global is what holds loader context (specifically the loader fetch) and +// index mutators for the fastboot instance. each index run will have a +// different loader fetch and its own index mutator. in order to keep these from +// colliding during concurrent indexing we hold each set of fastboot globals in +// a map that is unique for the index run. When the server visits fastboot it +// will provide the indexer route with the id for the fastboot global that is +// specific to the index run. +let optsId = 0; +export class RunnerOptionsManager { + #opts = new Map(); + setOptions(opts: RunnerOpts): number { + let id = optsId++; + this.#opts.set(id, opts); + return id; + } + getOptions(id: number): RunnerOpts { + let opts = this.#opts.get(id); + if (!opts) { + throw new Error(`No runner opts for id ${id}`); + } + return opts; + } + removeOptions(id: number) { + this.#opts.delete(id); + } +} export class Worker { #realmURL: URL; @@ -48,15 +110,15 @@ export class Worker { #indexer: Indexer; #queue: Queue; #loader: Loader; - #fromScratch: ((realmURL: URL) => Promise) | undefined; + #fromScratch: ((realmURL: URL) => Promise) | undefined; #realmAdapter: RealmAdapter; #incremental: | (( - prev: RunState, url: URL, + realmURL: URL, operation: 'update' | 'delete', - onInvalidation?: (invalidatedURLs: URL[]) => void, - ) => Promise) + ignoreData: Record, + ) => Promise) | undefined; constructor({ @@ -120,11 +182,6 @@ export class Worker { _fetch: this.#loader.fetch.bind(this.#loader), reader: this.#reader, indexer: this.#indexer, - entrySetter: () => { - throw new Error( - `entrySetter is deprecated. remove this after feature flag removed`, - ); - }, registerRunner: async (fromScratch, incremental) => { this.#fromScratch = fromScratch; this.#incremental = incremental; @@ -158,22 +215,11 @@ export class Worker { if (!this.#incremental) { throw new Error(`Index runner has not been registered`); } - let ignoreMap = new URLMap(); - for (let [url, contents] of Object.entries(args.ignoreData)) { - ignoreMap.set(new URL(url), ignore().add(contents)); - } let { ignoreData, stats, invalidations } = await this.#incremental( - // TODO clean this up after we remove feature flag. For now I'm just - // including the bare minimum to keep this from blowing up using the old APIs - { - realmURL: new URL(args.realmURL), - ignoreMap, - ignoreData: { ...args.ignoreData }, - instances: new URLMap(), - modules: new Map(), - } as unknown as RunState, new URL(args.url), + new URL(args.realmURL), args.operation, + { ...args.ignoreData }, ); return { ignoreData: { ...ignoreData }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d6c6ca2a77..170d4dd821 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1785,6 +1785,9 @@ importers: transform-modules-amd-plugin: specifier: workspace:* version: link:../../vendor/transform-modules-amd-plugin + typescript-memoize: + specifier: ^1.1.1 + version: 1.1.1 uuid: specifier: ^9.0.1 version: 9.0.1