Merge pull request #1244 from cardstack/cs-6719-remove-feature-flag-p…

…g-indexer-and-cleanup-obsolete-code

Remove PG_INDEXER feature flag and drop obsolete code

.github/workflows/ci.yaml

@@ -142,11 +142,11 @@ jobs:
       - name: Fail if generated icons have been changed without underlying raw icon changing
         run: git diff --exit-code
 
-  host-test-in-memory-index:
-    name: Host Tests - in-memory index
+  host-test:
+    name: Host Tests
     runs-on: ubuntu-latest
     concurrency:
-      group: boxel-host-test-in-memory-index${{ github.head_ref || github.run_id }}
+      group: boxel-host-test${{ github.head_ref || github.run_id }}
       cancel-in-progress: true
     steps:
       - uses: actions/checkout@v4
@@ -175,43 +175,7 @@ jobs:
         if: always()
         with:
           junit_files: junit/host.xml
-          check_name: Host Tests (In-Memory Index) Test Results
-
-  host-test-db-index:
-    name: Host Tests - db index
-    runs-on: ubuntu-latest
-    concurrency:
-      group: boxel-host-test-db-index${{ github.head_ref || github.run_id }}
-      cancel-in-progress: true
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/init
-      - name: Build boxel-ui
-        run: PG_INDEXER=true pnpm build
-        working-directory: packages/boxel-ui/addon
-      - name: Build host dist/ for fastboot
-        run: PG_INDEXER=true pnpm build
-        env:
-          NODE_OPTIONS: --max_old_space_size=4096
-        working-directory: packages/host
-      - name: Start realm servers
-        run: pnpm start:all &
-        working-directory: packages/realm-server
-      - name: create realm users
-        run: pnpm register-realm-users
-        working-directory: packages/matrix
-      - name: host test suite
-        # don't run the db index host tests in percy until we remove the feature flag
-        run: pnpm test:wait-for-servers
-        env:
-          PERCY_TOKEN: ${{ secrets.PERCY_TOKEN_HOST }}
-        working-directory: packages/host
-      - name: Publish test results
-        uses: EnricoMi/[email protected]
-        if: always()
-        with:
-          junit_files: junit/host.xml
-          check_name: Host Tests (DB Index) Test Results
+          check_name: Host Tests Test Results
 
   matrix-client-test:
     name: Matrix Client Tests
@@ -327,11 +291,11 @@ jobs:
             https://api.github.com/repos/$REPOSITORY/statuses/$HEAD_SHA \
             -d '{"context":"Matrix Playwright tests report","description":"","target_url":"'"$PLAYWRIGHT_REPORT_URL"'","state":"success"}'
 
-  realm-server-in-memory-index-test:
-    name: Realm Server Tests - in-memory index
+  realm-server-test:
+    name: Realm Server Tests
     runs-on: ubuntu-latest
     concurrency:
-      group: realm-server-in-memory-index-test-${{ github.head_ref || github.run_id }}
+      group: realm-server-test-${{ github.head_ref || github.run_id }}
       cancel-in-progress: true
     steps:
       - uses: actions/checkout@v4
@@ -357,36 +321,6 @@ jobs:
         run: pnpm test:dom
         working-directory: packages/realm-server
 
-  realm-server-db-index-test:
-    name: Realm Server Tests - db index
-    runs-on: ubuntu-latest
-    concurrency:
-      group: realm-server-db-index-test-${{ github.head_ref || github.run_id }}
-      cancel-in-progress: true
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/init
-      - name: Build boxel-ui
-        run: pnpm build
-        working-directory: packages/boxel-ui/addon
-      - name: Build host dist/ for fastboot
-        run: pnpm build
-        env:
-          NODE_OPTIONS: --max_old_space_size=4096
-        working-directory: packages/host
-      - name: Start realm servers
-        run: PG_INDEXER=true pnpm start:all &
-        working-directory: packages/realm-server
-      - name: create realm users
-        run: pnpm register-realm-users
-        working-directory: packages/matrix
-      - name: realm server test suite
-        run: PG_INDEXER=true pnpm test:wait-for-servers
-        working-directory: packages/realm-server
-      - name: realm server DOM tests
-        run: PG_INDEXER=true pnpm test:dom
-        working-directory: packages/realm-server
-
   change-check:
     name: Check which packages changed
     if: github.ref == 'refs/heads/main'
@@ -424,9 +358,8 @@ jobs:
     needs:
       - change-check
       - boxel-ui-test
-      # don't forget to change this after we remove the feature flag
-      - host-test-in-memory-index
-      - realm-server-in-memory-index-test
+      - host-test
+      - realm-server-test
     uses: ./.github/workflows/manual-deploy.yml
     secrets: inherit
     with:

README.md

@@ -120,7 +120,7 @@ If you wish to drop the development databases you can execute:
 pnpm drop-all-dbs
 ```
 
-You can then run `PGDATABASE=boxel_dev pnpm migrate up` (with `PGDATABASE` set accordingly) or just start the realm server (`PG_INDEXER=true pnpm start:all`) to create the database again.
+You can then run `PGDATABASE=boxel_dev pnpm migrate up` (with `PGDATABASE` set accordingly) or just start the realm server (`pnpm start:all`) to create the database again.
 
 To interact with your local database directly you can use psql:
 ```

packages/base/card-api.gts

@@ -323,10 +323,6 @@ export interface Field<
   component(model: Box<BaseDef>): BoxComponent;
   getter(instance: BaseDef): BaseInstanceType<CardT>;
   queryableValue(value: any, stack: BaseDef[]): SearchT;
-  // TODO remove this after feature flag is removed
-  queryMatcher(
-    innerMatcher: (innerValue: any) => boolean | null,
-  ): (value: SearchT) => boolean | null;
   handleNotLoadedError(
     instance: BaseInstanceType<CardT>,
     e: NotLoaded,
@@ -457,22 +453,6 @@ class ContainsMany<FieldT extends FieldDefConstructor>
     });
   }
 
-  queryMatcher(
-    innerMatcher: (innerValue: any) => boolean | null,
-  ): (value: any[] | null) => boolean | null {
-    return (value) => {
-      if (Array.isArray(value) && value.length === 0) {
-        return innerMatcher(null);
-      }
-      return (
-        Array.isArray(value) &&
-        value.some((innerValue) => {
-          return innerMatcher(innerValue);
-        })
-      );
-    };
-  }
-
   serialize(
     values: BaseInstanceType<FieldT>[],
     doc: JSONAPISingleResourceDocument,
@@ -676,12 +656,6 @@ class Contains<CardT extends FieldDefConstructor> implements Field<CardT, any> {
     return this.card[queryableValue](instance, stack);
   }
 
-  queryMatcher(
-    innerMatcher: (innerValue: any) => boolean | null,
-  ): (value: any) => boolean | null {
-    return (value) => innerMatcher(value);
-  }
-
   serialize(
     value: InstanceType<CardT>,
     doc: JSONAPISingleResourceDocument,
@@ -835,12 +809,6 @@ class LinksTo<CardT extends CardDefConstructor> implements Field<CardT> {
     return this.card[queryableValue](instance, stack);
   }
 
-  queryMatcher(
-    innerMatcher: (innerValue: any) => boolean | null,
-  ): (value: any) => boolean | null {
-    return (value) => innerMatcher(value);
-  }
-
   serialize(
     value: InstanceType<CardT>,
     doc: JSONAPISingleResourceDocument,
@@ -1183,22 +1151,6 @@ class LinksToMany<FieldT extends CardDefConstructor>
     });
   }
 
-  queryMatcher(
-    innerMatcher: (innerValue: any) => boolean | null,
-  ): (value: any[] | null) => boolean | null {
-    return (value) => {
-      if (Array.isArray(value) && value.length === 0) {
-        return innerMatcher(null);
-      }
-      return (
-        Array.isArray(value) &&
-        value.some((innerValue) => {
-          return innerMatcher(innerValue);
-        })
-      );
-    };
-  }
-
   serialize(
     values: BaseInstanceType<FieldT>[] | null | undefined,
     doc: JSONAPISingleResourceDocument,

packages/base/cards-grid.gts

@@ -183,9 +183,8 @@ class Isolated extends Component<typeof CardsGrid> {
       {
         filter: {
           not: {
-            type: {
-              module: `${baseRealm.url}cards-grid`,
-              name: 'CardsGrid',
+            eq: {
+              _cardType: 'Cards Grid',
             },
           },
         },

packages/host/app/components/card-prerender.gts

@@ -6,13 +6,12 @@ import { didCancel, enqueueTask, restartableTask } from 'ember-concurrency';
 
 import { type Indexer } from '@cardstack/runtime-common';
 import type { LocalPath } from '@cardstack/runtime-common/paths';
+import { readFileAsText as _readFileAsText } from '@cardstack/runtime-common/stream';
 import {
-  type EntrySetter,
+  type IndexResults,
   type Reader,
-  type RunState,
   type RunnerOpts,
-} from '@cardstack/runtime-common/search-index';
-import { readFileAsText as _readFileAsText } from '@cardstack/runtime-common/stream';
+} from '@cardstack/runtime-common/worker';
 
 import { CurrentRun } from '../lib/current-run';
 
@@ -50,10 +49,10 @@ export default class CardPrerender extends Component {
     }
   }
 
-  private async fromScratch(realmURL: URL): Promise<RunState> {
+  private async fromScratch(realmURL: URL): Promise<IndexResults> {
     try {
-      let state = await this.doFromScratch.perform(realmURL);
-      return state;
+      let results = await this.doFromScratch.perform(realmURL);
+      return results;
     } catch (e: any) {
       if (!didCancel(e)) {
         throw e;
@@ -65,17 +64,17 @@ export default class CardPrerender extends Component {
   }
 
   private async incremental(
-    prev: RunState,
     url: URL,
+    realmURL: URL,
     operation: 'delete' | 'update',
-    onInvalidation?: (invalidatedURLs: URL[]) => void,
-  ): Promise<RunState> {
+    ignoreData: Record<string, string>,
+  ): Promise<IndexResults> {
     try {
       let state = await this.doIncremental.perform(
-        prev,
         url,
+        realmURL,
         operation,
-        onInvalidation,
+        ignoreData,
       );
       return state;
     } catch (e: any) {
@@ -98,15 +97,14 @@ export default class CardPrerender extends Component {
   });
 
   private doFromScratch = enqueueTask(async (realmURL: URL) => {
-    let { reader, entrySetter, indexer } = this.getRunnerParams();
+    let { reader, indexer } = this.getRunnerParams();
     await this.resetLoaderInFastboot.perform();
     let current = await CurrentRun.fromScratch(
       new CurrentRun({
         realmURL,
         loader: this.loaderService.loader,
         reader,
         indexer,
-        entrySetter,
         renderCard: this.renderService.renderCard.bind(this.renderService),
       }),
     );
@@ -116,23 +114,22 @@ export default class CardPrerender extends Component {
 
   private doIncremental = enqueueTask(
     async (
-      prev: RunState,
       url: URL,
+      realmURL: URL,
       operation: 'delete' | 'update',
-      onInvalidation?: (invalidatedURLs: URL[]) => void,
+      ignoreData: Record<string, string>,
     ) => {
-      let { reader, entrySetter, indexer } = this.getRunnerParams();
+      let { reader, indexer } = this.getRunnerParams();
       await this.resetLoaderInFastboot.perform();
       let current = await CurrentRun.incremental({
         url,
+        realmURL,
         operation,
-        prev,
         reader,
+        ignoreData,
         indexer,
         loader: this.loaderService.loader,
-        entrySetter,
         renderCard: this.renderService.renderCard.bind(this.renderService),
-        onInvalidation,
       });
       this.renderService.indexRunDeferred?.fulfill();
       return current;
@@ -149,9 +146,7 @@ export default class CardPrerender extends Component {
 
   private getRunnerParams(): {
     reader: Reader;
-    entrySetter: EntrySetter;
-    // TODO make this required after feature flag removed
-    indexer?: Indexer;
+    indexer: Indexer;
   } {
     let self = this;
     function readFileAsText(
@@ -172,7 +167,6 @@ export default class CardPrerender extends Component {
       }
       return {
         reader: getRunnerOpts(optsId).reader,
-        entrySetter: getRunnerOpts(optsId).entrySetter,
         indexer: getRunnerOpts(optsId).indexer,
       };
     } else {
@@ -183,7 +177,6 @@ export default class CardPrerender extends Component {
           ),
           readFileAsText,
         },
-        entrySetter: this.localIndexer.setEntry.bind(this.localIndexer),
         indexer: this.localIndexer.indexer,
       };
     }

packages/host/app/config/environment.d.ts

@@ -28,7 +28,5 @@ declare const config: {
   loginMessageTimeoutMs: number;
   minSaveTaskDurationMs: number;
   sqlSchema: string;
-  featureFlags?: {
-    'pg-indexer'?: boolean;
-  };
+  featureFlags?: {};
 };