[BUG] schema: build default with config ef & knn_index, remove #document population in defaults

Author: jairad26

rust/cli/src/commands/vacuum.rs

@@ -11,7 +11,7 @@ use chroma_segment::local_segment_manager::LocalSegmentManager;
 use chroma_sqlite::db::SqliteDb;
 use chroma_sysdb::SysDb;
 use chroma_system::System;
-use chroma_types::{CollectionUuid, KnnIndex, ListCollectionsRequest};
+use chroma_types::{CollectionUuid, KnnIndex, ListCollectionsRequest, Schema};
 use clap::Parser;
 use colored::Colorize;
 use dialoguer::Confirm;
@@ -108,10 +108,15 @@ async fn trigger_vector_segments_max_seq_id_migration(
     for collection_id in collection_ids {
         let mut collection = sysdb.get_collection_with_segments(collection_id).await?;
 
-        collection
-            .collection
-            .reconcile_schema_with_config(default_knn_index)
-            .map_err(|e| Box::new(e) as Box<dyn Error>)?;
+        if collection.collection.schema.is_none() {
+            collection.collection.schema = Some(
+                Schema::convert_collection_config_to_schema(
+                    &collection.collection.config,
+                    default_knn_index,
+                )
+                .map_err(|e| Box::new(e) as Box<dyn Error>)?,
+            );
+        }
 
         // If collection is uninitialized, that means nothing has been written yet.
         let dim = match collection.collection.dimension {

rust/frontend/src/get_collection_with_segments_provider.rs

@@ -185,14 +185,16 @@ impl CollectionsWithSegmentsProvider {
                 .await?
         };
 
-        // reconcile schema and config
-        let reconciled_schema = Schema::reconcile_schema_and_config(
-            collection_and_segments_sysdb.collection.schema.as_ref(),
-            Some(&collection_and_segments_sysdb.collection.config),
-            knn_index,
-        )
-        .map_err(CollectionsWithSegmentsProviderError::InvalidSchema)?;
-        collection_and_segments_sysdb.collection.schema = Some(reconciled_schema);
+        if collection_and_segments_sysdb.collection.schema.is_none() {
+            collection_and_segments_sysdb.collection.schema = Some(
+                Schema::convert_collection_config_to_schema(
+                    &collection_and_segments_sysdb.collection.config,
+                    knn_index,
+                )
+                .map_err(CollectionsWithSegmentsProviderError::InvalidSchema)?,
+            );
+        }
+
         self.set_collection_with_segments(collection_and_segments_sysdb.clone())
             .await;
         Ok(collection_and_segments_sysdb)

rust/frontend/src/impls/service_based_frontend.rs

@@ -380,9 +380,15 @@ impl ServiceBasedFrontend {
             .map_err(|err| Box::new(err) as Box<dyn ChromaError>)?;
         if self.enable_schema {
             for collection in collections.iter_mut() {
-                collection
-                    .reconcile_schema_with_config(self.default_knn_index)
-                    .map_err(GetCollectionsError::InvalidSchema)?;
+                if collection.schema.is_none() {
+                    collection.schema = Some(
+                        Schema::convert_collection_config_to_schema(
+                            &collection.config,
+                            self.default_knn_index,
+                        )
+                        .map_err(GetCollectionsError::InvalidSchema)?,
+                    );
+                }
             }
         }
         Ok(collections)
@@ -424,9 +430,15 @@ impl ServiceBasedFrontend {
             .map_err(|err| Box::new(err) as Box<dyn ChromaError>)?;
         if self.enable_schema {
             for collection in &mut collections {
-                collection
-                    .reconcile_schema_with_config(self.default_knn_index)
-                    .map_err(GetCollectionError::InvalidSchema)?;
+                if collection.schema.is_none() {
+                    collection.schema = Some(
+                        Schema::convert_collection_config_to_schema(
+                            &collection.config,
+                            self.default_knn_index,
+                        )
+                        .map_err(GetCollectionError::InvalidSchema)?,
+                    );
+                }
             }
         }
         collections
@@ -448,10 +460,14 @@ impl ServiceBasedFrontend {
             .await
             .map_err(|err| Box::new(err) as Box<dyn ChromaError>)?;
 
-        if self.enable_schema {
-            collection
-                .reconcile_schema_with_config(self.default_knn_index)
-                .map_err(GetCollectionByCrnError::InvalidSchema)?;
+        if self.enable_schema && collection.schema.is_none() {
+            collection.schema = Some(
+                Schema::convert_collection_config_to_schema(
+                    &collection.config,
+                    self.default_knn_index,
+                )
+                .map_err(GetCollectionByCrnError::InvalidSchema)?,
+            );
         }
         Ok(collection)
     }
@@ -628,10 +644,14 @@ impl ServiceBasedFrontend {
             .await;
         // this is done in the case that get_or_create was a get, in which case we should reconcile the schema and config
         // that was retrieved from sysdb, rather than the one that was passed in
-        if self.enable_schema {
-            collection
-                .reconcile_schema_with_config(self.default_knn_index)
-                .map_err(CreateCollectionError::InvalidSchema)?;
+        if self.enable_schema && collection.schema.is_none() {
+            collection.schema = Some(
+                Schema::convert_collection_config_to_schema(
+                    &collection.config,
+                    self.default_knn_index,
+                )
+                .map_err(CreateCollectionError::InvalidSchema)?,
+            );
         }
         Ok(collection)
     }
@@ -733,10 +753,15 @@ impl ServiceBasedFrontend {
                 target_collection_name,
             )
             .await?;
-        collection_and_segments
-            .collection
-            .reconcile_schema_with_config(self.default_knn_index)
-            .map_err(ForkCollectionError::InvalidSchema)?;
+        if collection_and_segments.collection.schema.is_none() {
+            collection_and_segments.collection.schema = Some(
+                Schema::convert_collection_config_to_schema(
+                    &collection_and_segments.collection.config,
+                    self.default_knn_index,
+                )
+                .map_err(ForkCollectionError::InvalidSchema)?,
+            );
+        }
         let collection = collection_and_segments.collection.clone();
         let latest_collection_logical_size_bytes = collection_and_segments
             .collection

rust/log/src/local_compaction_manager.rs

@@ -15,7 +15,7 @@ use chroma_sysdb::SysDb;
 use chroma_system::Handler;
 use chroma_system::{Component, ComponentContext};
 use chroma_types::{
-    Chunk, CollectionUuid, GetCollectionWithSegmentsError, KnnIndex, LogRecord, SchemaError,
+    Chunk, CollectionUuid, GetCollectionWithSegmentsError, KnnIndex, LogRecord, Schema, SchemaError,
 };
 use serde::{Deserialize, Serialize};
 use thiserror::Error;
@@ -141,9 +141,15 @@ impl Handler<BackfillMessage> for LocalCompactionManager {
             .get_collection_with_segments(message.collection_id)
             .await?;
         let schema_previously_persisted = collection_and_segments.collection.schema.is_some();
-        collection_and_segments
-            .collection
-            .reconcile_schema_with_config(KnnIndex::Hnsw)?;
+        if !schema_previously_persisted {
+            collection_and_segments.collection.schema = Some(
+                Schema::convert_collection_config_to_schema(
+                    &collection_and_segments.collection.config,
+                    KnnIndex::Hnsw,
+                )
+                .map_err(CompactionManagerError::SchemaReconcileError)?,
+            );
+        }
         // If collection is uninitialized, that means nothing has been written yet.
         let dim = match collection_and_segments.collection.dimension {
             Some(dim) => dim,
@@ -267,7 +273,12 @@ impl Handler<PurgeLogsMessage> for LocalCompactionManager {
             .get_collection_with_segments(message.collection_id)
             .await?;
         let mut collection = collection_segments.collection.clone();
-        collection.reconcile_schema_with_config(KnnIndex::Hnsw)?;
+        if collection.schema.is_none() {
+            collection.schema = Some(
+                Schema::convert_collection_config_to_schema(&collection.config, KnnIndex::Hnsw)
+                    .map_err(CompactionManagerError::SchemaReconcileError)?,
+            );
+        }
         // If dimension is None, that means nothing has been written yet.
         let dim = match collection.dimension {
             Some(dim) => dim,