(EAI-483): ingest:all command deletes pages and embeddings not in data sources

packages/mongodb-rag-core/src/contentStore/EmbeddedContent.ts

@@ -75,6 +75,7 @@ export type EmbeddedContentStore = VectorStore<EmbeddedContent> & {
   deleteEmbeddedContent(args: {
     page?: Page;
     dataSources?: string[];
+    inverse?: boolean;
   }): Promise<void>;
 
   /**

packages/mongodb-rag-core/src/contentStore/MongoDbEmbeddedContentStore.ts

@@ -95,10 +95,10 @@ export function makeMongoDbEmbeddedContentStore({
       return await embeddedContentCollection.find(pageIdentity(page)).toArray();
     },
 
-    async deleteEmbeddedContent({ page, dataSources }) {
+    async deleteEmbeddedContent({ page, dataSources, inverse = false }) {
       const deleteResult = await embeddedContentCollection.deleteMany({
         ...(page ? pageIdentity(page) : undefined),
-        ...(dataSources ? { sourceName: { $in: dataSources } } : undefined),
+        ...(dataSources ? { sourceName: { [inverse ? '$nin' : '$in']: dataSources } } : undefined),
       });
       if (!deleteResult.acknowledged) {
         throw new Error("EmbeddedContent deletion not acknowledged!");

packages/mongodb-rag-core/src/contentStore/MongoDbPageStore.ts

@@ -80,9 +80,9 @@ export function makeMongoDbPageStore({
         })
       );
     },
-    async deletePages({ dataSources, permanent = false }: DeletePagesArgs) {
+    async deletePages({ dataSources, permanent = false, inverse = false }: DeletePagesArgs) {
       const filter = {
-        ...(dataSources ? { sourceName: { $in: dataSources } } : undefined),
+        ...(dataSources ? { sourceName: { [inverse ? '$nin' : '$in']: dataSources } } : undefined),
       };
       if (permanent) {
         const result = await pagesCollection.deleteMany(filter);

packages/mongodb-rag-core/src/contentStore/Page.ts

@@ -93,6 +93,10 @@ export type DeletePagesArgs = {
     rather than marking them as `"deleted"`.
    */
   permanent?: boolean;
+  /**
+   * If true, delete pages that do NOT match the query.
+   */
+  inverse?: boolean;
 };
 
 /**

packages/mongodb-rag-ingest/src/commands/all.ts

@@ -48,6 +48,17 @@ export const doAllCommand = async (
     since: lastSuccessfulRunDate ?? new Date("2023-01-01"),
   });
 
+  // cleanup - delete pages and embedded content that are no longer in the data sources
+  await config.pageStore.deletePages({
+    dataSources: config.dataSources.map(({ name }) => name),
+    permanent: true,
+    inverse: true,
+  });
+  await config.embeddedContentStore.deleteEmbeddedContent({
+    dataSources: config.dataSources.map(({ name }) => name),
+    inverse: true,
+  });
+
   logger.info(`Updating last successful run date`);
   await ingestMetaStore.updateLastSuccessfulRunDate();
 };