Merge pull request #360 from astronomy-commons/sandro/missing-schema-arg

Add schema to source data in SOAP pipeline
astronomy-commons · Jul 30, 2024 · 8a34d62 · 8a34d62
2 parents aac8b9b + 766c767
commit 8a34d62
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -9,3 +9,4 @@ sphinx
 sphinx-autoapi
 sphinx-copybutton
 sphinx-book-theme
+git+https://github.com/astronomy-commons/hipscat.git@main
diff --git a/src/hipscat_import/soap/map_reduce.py b/src/hipscat_import/soap/map_reduce.py
@@ -108,7 +108,10 @@ def count_joins(soap_args: SoapArguments, source_pixel: HealpixPixel, object_pix
         else:
             read_columns = [soap_args.source_object_id_column]
         source_data = file_io.read_parquet_file_to_pandas(
-            source_path, columns=read_columns, storage_options=soap_args.source_storage_options
+            source_path,
+            columns=read_columns,
+            schema=soap_args.source_catalog.schema,
+            storage_options=soap_args.source_storage_options,
         ).set_index(soap_args.source_object_id_column)
 
         remaining_sources = len(source_data)