fix: update persistence logic

wmgeolab · Feb 23, 2025 · 726a0bb · 726a0bb
1 parent 6924177
commit 726a0bb
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/ml-api/src/ml_api/ingestion/ingestion_service.py b/ml-api/src/ml_api/ingestion/ingestion_service.py
@@ -51,6 +51,8 @@ def ingest_single_file(self, file_path: Path) -> bool:
                 documents=docs,
                 num_workers=settings.PIPELINE_NUM_WORKERS_SINGLE,
             )
+            if settings.PIPELINE_PERSIST:
+                self.pipeline.persist(persist_dir=settings.PIPELINE_PERSIST_PATH)
 
             logger.info(
                 f"Processed & ingested {len(processed_nodes)} nodes from {file_path}"

diff --git a/ml-api/src/ml_api/ingestion/pipeline.py b/ml-api/src/ml_api/ingestion/pipeline.py
@@ -35,6 +35,6 @@ def get_pipeline(
     pipeline = IngestionPipeline(transformations=transformations, vector_store=qdrant)
 
     if persist and persist_path:
-        pipeline.persist(persist_dir=persist_path)
+        pipeline.load(persist_dir=persist_path)
 
     return pipeline