Refactor & implement progressive search (#62)

Refactor & implement progressive search --------- Co-authored-by: Philippe Moussalli <[email protected]>
ml6team · Jan 11, 2024 · 5c69f29 · 5c69f29
1 parent f5a4b1c
commit 5c69f29
Show file tree

Hide file tree

Showing 6 changed files with 563 additions and 468 deletions.
diff --git a/src/evaluation.ipynb b/src/evaluation.ipynb
diff --git a/src/parameter_search.ipynb b/src/parameter_search.ipynb
diff --git a/src/pipeline.ipynb b/src/pipeline.ipynb
@@ -65,8 +65,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!docker compose version >/dev/null\n",
-    "!docker info >/dev/null"
+    "!docker compose version\n",
+    "!docker ps && echo \"Docker running\""
    ]
   },
   {
@@ -581,7 +581,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "version": "3.9.13"
   }
  },
  "nbformat": 4,

diff --git a/src/pipeline_eval.py b/src/pipeline_eval.py
@@ -6,11 +6,12 @@
 
 def create_pipeline(
     *,
-    weaviate_url: str,
-    base_path: str = "./data-dir",
+    base_path: str = "./data",
+    weaviate_url="http://host.docker.internal:8080",
     weaviate_class: str = "Pipeline1",
-    csv_dataset_uri: str = "/data/wikitext_1000_q.csv",
-    csv_separator: str = ";",
+    evaluation_set_path="./evaluation_datasets",
+    evaluation_set_filename="wikitext_1000_q.csv",
+    evaluation_set_separator: str = ";",
     embed_model_provider: str = "huggingface",
     embed_model: str = "all-MiniLM-L6-v2",
     embed_api_key: dict = {},
@@ -32,8 +33,9 @@ def create_pipeline(
     load_from_csv = evaluation_pipeline.read(
         "load_from_csv",
         arguments={
-            "dataset_uri": csv_dataset_uri,
-            "column_separator": csv_separator,
+            "dataset_uri": "/evaldata/" + evaluation_set_filename,
+            # mounted dir from within docker as extra_volumes
+            "column_separator": evaluation_set_separator,
         },
         produces={
             "question": pa.string(),

diff --git a/src/pipeline_index.py b/src/pipeline_index.py
@@ -6,7 +6,7 @@
 def create_pipeline(
     *,
     weaviate_url: str,
-    base_path: str = "./data-dir",
+    base_path: str = "./data",
     n_rows_to_load: int = 1000,
     weaviate_class: str = "Pipeline1",
     weaviate_overwrite: bool = True,