Suppress pandas metadata on import.

src/hats_import/catalog/map_reduce.py

@@ -187,7 +187,9 @@ def split_pixels(
                     filtered_data = data.iloc[unique_inverse == unique_index]
                     if _has_named_index(filtered_data):
                         filtered_data = filtered_data.reset_index()
-                    filtered_data = pa.Table.from_pandas(filtered_data, preserve_index=False)
+                    filtered_data = pa.Table.from_pandas(
+                        filtered_data, preserve_index=False
+                    ).replace_schema_metadata()
                 else:
                     filtered_data = data.filter(unique_inverse == unique_index)
 

tests/data/small_sky_object_catalog/properties

@@ -7,8 +7,8 @@ hats_col_dec=dec
 hats_max_rows=1000000
 hats_order=0
 moc_sky_fraction=0.08333
-hats_builder=hats-import v0.4.1.dev2+gaeb92ae
-hats_creation_date=2024-10-21T13\:22UTC
-hats_estsize=70
+hats_builder=hats-import v0.4.2.dev1+g57aaa9d
+hats_creation_date=2024-11-07T15\:20UTC
+hats_estsize=113
 hats_release_date=2024-09-18
 hats_version=v0.1

tests/data/small_sky_source_catalog/properties

@@ -7,8 +7,8 @@ hats_col_dec=source_dec
 hats_max_rows=3000
 hats_order=2
 moc_sky_fraction=0.16667
-hats_builder=hats-import v0.4.1.dev2+gaeb92ae
-hats_creation_date=2024-10-21T13\:22UTC
-hats_estsize=1083
+hats_builder=hats-import v0.4.2.dev1+g57aaa9d
+hats_creation_date=2024-11-07T15\:20UTC
+hats_estsize=1086
 hats_release_date=2024-09-18
 hats_version=v0.1

tests/hats_import/catalog/test_run_import.py

@@ -290,9 +290,9 @@ def test_dask_runner(
         ]
     )
     schema = pq.read_metadata(output_file).schema.to_arrow_schema()
-    assert schema.equals(expected_parquet_schema, check_metadata=False)
+    assert schema.equals(expected_parquet_schema)
     schema = pq.read_metadata(args.catalog_path / "dataset" / "_metadata").schema.to_arrow_schema()
-    assert schema.equals(expected_parquet_schema, check_metadata=False)
+    assert schema.equals(expected_parquet_schema)
 
     # Check that, when re-loaded as a pandas dataframe, the appropriate numeric types are used.
     data_frame = pd.read_parquet(output_file, engine="pyarrow")

tests/hats_import/catalog/test_run_round_trip.py

@@ -116,9 +116,9 @@ def test_import_mixed_schema_csv(
         ]
     )
     schema = pq.read_metadata(output_file).schema.to_arrow_schema()
-    assert schema.equals(expected_parquet_schema, check_metadata=False)
+    assert schema.equals(expected_parquet_schema)
     schema = pq.read_metadata(args.catalog_path / "dataset" / "_metadata").schema.to_arrow_schema()
-    assert schema.equals(expected_parquet_schema, check_metadata=False)
+    assert schema.equals(expected_parquet_schema)
 
 
 @pytest.mark.dask
@@ -549,9 +549,9 @@ def test_import_pyarrow_types(
         ]
     )
     schema = pq.read_metadata(output_file).schema.to_arrow_schema()
-    assert schema.equals(expected_parquet_schema, check_metadata=False)
+    assert schema.equals(expected_parquet_schema)
     schema = pq.read_metadata(args.catalog_path / "dataset" / "_metadata").schema.to_arrow_schema()
-    assert schema.equals(expected_parquet_schema, check_metadata=False)
+    assert schema.equals(expected_parquet_schema)
 
 
 class SimplePyarrowCsvReader(CsvReader):
@@ -896,13 +896,13 @@ def test_gaia_ecsv(
     expected_parquet_schema = pq.read_metadata(schema_path).schema.to_arrow_schema()
 
     schema = pq.read_metadata(output_file).schema.to_arrow_schema()
-    assert schema.equals(expected_parquet_schema, check_metadata=False)
+    assert schema.equals(expected_parquet_schema)
     schema = pq.read_metadata(args.catalog_path / "dataset" / "_metadata").schema.to_arrow_schema()
-    assert schema.equals(expected_parquet_schema, check_metadata=False)
+    assert schema.equals(expected_parquet_schema)
     schema = pq.read_metadata(args.catalog_path / "dataset" / "_common_metadata").schema.to_arrow_schema()
-    assert schema.equals(expected_parquet_schema, check_metadata=False)
+    assert schema.equals(expected_parquet_schema)
     schema = pds.dataset(args.catalog_path, format="parquet").schema
-    assert schema.equals(expected_parquet_schema, check_metadata=False)
+    assert schema.equals(expected_parquet_schema)
 
 
 @pytest.mark.dask

tests/hats_import/hipscat_conversion/test_run_conversion.py

@@ -70,13 +70,13 @@ def test_run_conversion_object(
         ]
     )
     schema = pq.read_metadata(output_file).schema.to_arrow_schema()
-    assert schema.equals(expected_parquet_schema, check_metadata=False)
+    assert schema.equals(expected_parquet_schema)
     assert schema.metadata is None
     schema = pq.read_metadata(args.catalog_path / "dataset" / "_metadata").schema.to_arrow_schema()
-    assert schema.equals(expected_parquet_schema, check_metadata=False)
+    assert schema.equals(expected_parquet_schema)
     assert schema.metadata is None
     schema = pq.read_metadata(args.catalog_path / "dataset" / "_common_metadata").schema.to_arrow_schema()
-    assert schema.equals(expected_parquet_schema, check_metadata=False)
+    assert schema.equals(expected_parquet_schema)
     assert schema.metadata is None
 
     data = file_io.read_parquet_file_to_pandas(

tests/hats_import/index/test_run_index.py

@@ -57,13 +57,13 @@ def test_run_index(
 
     outfile = args.catalog_path / "dataset" / "index" / "part.0.parquet"
     schema = pq.read_metadata(outfile).schema.to_arrow_schema()
-    assert schema.equals(basic_index_parquet_schema, check_metadata=False)
+    assert schema.equals(basic_index_parquet_schema)
 
     schema = pq.read_metadata(args.catalog_path / "dataset" / "_metadata").schema.to_arrow_schema()
-    assert schema.equals(basic_index_parquet_schema, check_metadata=False)
+    assert schema.equals(basic_index_parquet_schema)
 
     schema = pq.read_metadata(args.catalog_path / "dataset" / "_common_metadata").schema.to_arrow_schema()
-    assert schema.equals(basic_index_parquet_schema, check_metadata=False)
+    assert schema.equals(basic_index_parquet_schema)
 
 
 @pytest.mark.dask
@@ -104,13 +104,13 @@ def test_run_index_on_source(
 
     outfile = args.catalog_path / "dataset" / "index" / "part.0.parquet"
     schema = pq.read_metadata(outfile).schema.to_arrow_schema()
-    assert schema.equals(basic_index_parquet_schema, check_metadata=False)
+    assert schema.equals(basic_index_parquet_schema)
 
     schema = pq.read_metadata(args.catalog_path / "dataset" / "_metadata").schema.to_arrow_schema()
-    assert schema.equals(basic_index_parquet_schema, check_metadata=False)
+    assert schema.equals(basic_index_parquet_schema)
 
     schema = pq.read_metadata(args.catalog_path / "dataset" / "_common_metadata").schema.to_arrow_schema()
-    assert schema.equals(basic_index_parquet_schema, check_metadata=False)
+    assert schema.equals(basic_index_parquet_schema)
 
 
 @pytest.mark.dask
@@ -148,7 +148,7 @@ def test_run_index_on_source_object_id(
 
     outfile = args.catalog_path / "dataset" / "index" / "part.0.parquet"
     schema = pq.read_metadata(outfile).schema.to_arrow_schema()
-    assert schema.equals(basic_index_parquet_schema, check_metadata=False)
+    assert schema.equals(basic_index_parquet_schema)
 
     id_range = np.arange(700, 831)
     ## Some of the objects have sources that span two source partitions.
@@ -158,7 +158,7 @@ def test_run_index_on_source_object_id(
     assert_parquet_file_index(outfile, doubled_up)
 
     schema = pq.read_metadata(args.catalog_path / "dataset" / "_metadata").schema.to_arrow_schema()
-    assert schema.equals(basic_index_parquet_schema, check_metadata=False)
+    assert schema.equals(basic_index_parquet_schema)
 
     schema = pq.read_metadata(args.catalog_path / "dataset" / "_common_metadata").schema.to_arrow_schema()
-    assert schema.equals(basic_index_parquet_schema, check_metadata=False)
+    assert schema.equals(basic_index_parquet_schema)

tests/hats_import/soap/test_run_soap.py

@@ -110,7 +110,7 @@ def test_object_to_source_with_leaves(
             pa.field("join_Npix", pa.uint64()),
         ]
     )
-    assert parquet_file.metadata.schema.to_arrow_schema().equals(exepcted_schema, check_metadata=False)
+    assert parquet_file.metadata.schema.to_arrow_schema().equals(exepcted_schema)
 
 
 @pytest.mark.dask
@@ -161,4 +161,4 @@ def test_object_to_source_with_leaves_drop_duplicates(
             pa.field("join_Npix", pa.uint64()),
         ]
     )
-    assert parquet_file.metadata.schema.to_arrow_schema().equals(exepcted_schema, check_metadata=False)
+    assert parquet_file.metadata.schema.to_arrow_schema().equals(exepcted_schema)