feat: set user being able to set chunk size and overlap for indices

libs/ktem/ktem/index/file/base.py

@@ -55,6 +55,8 @@ class BaseFileIndexIndexing(BaseComponent):
     FSPath = Param(help="The file storage path")
     user_id = Param(help="The user id")
     private = Param(False, help="Whether this is private index")
+    user_chunk_size = Param(help="Chunk size set by user")
+    user_chunk_overlap = Param(help="Chunk overlap set by user")
 
     def run(
         self, file_paths: str | Path | list[str | Path], *args, **kwargs

libs/ktem/ktem/index/file/index.py

@@ -404,6 +404,25 @@ def get_admin_settings(cls):
                 "choices": [("Yes", True), ("No", False)],
                 "info": "If private, files will not be accessible across users.",
             },
+            "chunk_size": {
+                "name": "Size of chunk",
+                "value": -1,
+                "component": "number",
+                "info": (
+                    "Number of characters of each text segment. "
+                    "Set -1 to use developer setting."
+                ),
+            },
+            "chunk_overlap": {
+                "name": "Max number of characters that can be overlap between segments",
+                "value": -1,
+                "component": "number",
+                "info": (
+                    "Number of characters that consecutive text segments "
+                    "should overlap with each other. "
+                    "Set -1 to use developer setting."
+                ),
+            },
         }
 
     def get_indexing_pipeline(self, settings, user_id) -> BaseFileIndexIndexing:
@@ -423,6 +442,8 @@ def get_indexing_pipeline(self, settings, user_id) -> BaseFileIndexIndexing:
         obj.FSPath = self._fs_path
         obj.user_id = user_id
         obj.private = self.config.get("private", False)
+        obj.user_chunk_size = self.config.get("chunk_size", -1)
+        obj.user_chunk_overlap = self.config.get("chunk_overlap", -1)
 
         return obj
 

libs/ktem/ktem/index/file/pipelines.py

@@ -729,7 +729,17 @@ def route(self, file_path: str | Path) -> IndexPipeline:
 
         Can subclass this method for a more elaborate pipeline routing strategy.
         """
-        _, chunk_size, chunk_overlap = dev_settings()
+
+        _, dev_chunk_size, dev_chunk_overlap = dev_settings()
+
+        chunk_size = (
+            self.user_chunk_size if self.user_chunk_size > 0 else dev_chunk_size
+        )
+        chunk_overlap = (
+            self.user_chunk_overlap
+            if self.user_chunk_overlap > 0
+            else dev_chunk_overlap
+        )
 
         # check if file_path is a URL
         if self.is_url(file_path):
@@ -744,6 +754,8 @@ def route(self, file_path: str | Path) -> IndexPipeline:
                     "the suitable pipeline for this file type in the settings."
                 )
 
+        print(f"Chunk size: {chunk_size}, chunk overlap: {chunk_overlap}")
+
         print("Using reader", reader)
         pipeline: IndexPipeline = IndexPipeline(
             loader=reader,