Better margin testing and auto-formatting.

tests/data/xmatch/xmatch_with_margin.csv

@@ -0,0 +1,191 @@
+small_sky_order1_id,xmatch_id,dist_arcsec
+700,700,14.01745384678654
+701,701,15.901521432517091
+703,816,2520.471259900567
+703,703,26.874535043770734
+703,815,5267.111745176849
+704,704,29.094256521939307
+705,705,8.6622234512455
+705,786,4662.241812069228
+706,800,6789.566156252407
+706,706,5.071285713708736
+707,707,24.162094360888325
+708,777,7198.893021502899
+708,763,4580.446768729388
+708,708,31.650439060212975
+709,709,18.30591242949293
+709,785,6219.912548299288
+710,726,7169.882780634017
+710,710,12.364487775272975
+711,830,4267.446053402628
+711,711,36.648467340005546
+711,802,2319.3754458192384
+712,759,5941.067826054423
+712,712,27.934517878956378
+713,713,35.59452939878912
+714,714,29.73436104637042
+714,741,3589.038633431455
+715,715,33.558030883500656
+716,807,3528.9960233854745
+716,716,36.58457756377089
+717,727,6305.761453592823
+717,717,25.66070269220809
+718,718,5.998839488816871
+719,719,21.96927980363252
+720,720,33.889501865934136
+722,722,19.071678724622437
+723,723,15.231398879334632
+724,724,28.138679249131883
+725,827,6519.017750379818
+725,725,14.68155100221724
+725,769,4454.650027629775
+726,726,30.84163314966896
+727,727,24.805979905449146
+727,717,6310.9643569538
+728,781,6110.85192209587
+728,728,15.154451870952906
+729,810,3651.511676964429
+729,768,5095.689551958877
+729,729,30.444941783883085
+730,730,30.97470923623241
+731,731,16.338727746196852
+732,732,16.42789467211279
+732,784,4504.8674015470815
+732,745,3634.942098689493
+733,733,15.678747299843298
+734,771,3597.404165561576
+734,734,24.60014645050172
+735,735,35.792978697264246
+735,794,3889.0799107779208
+736,736,24.063003451778744
+738,738,23.082207298748006
+738,776,3960.251724920152
+738,772,4652.414343597835
+739,739,11.68963379186935
+740,740,28.927459278517706
+741,714,3570.371734213002
+741,741,25.64932704541115
+742,742,23.14133566690715
+744,744,29.892501026933147
+744,813,7196.327387342898
+745,745,39.91429126103487
+745,732,3590.8704580140097
+746,746,29.434744741937838
+747,747,30.45478935158673
+748,748,11.024647160876295
+749,749,15.385050075374272
+750,750,19.14396954011775
+751,751,34.65270515053803
+751,781,7183.94107540341
+752,752,36.080966643595126
+753,753,25.649068907860094
+755,714,3570.371734213002
+755,741,25.64932704541115
+757,757,14.840652116071446
+758,780,4207.92666948583
+758,758,20.35614635204724
+759,759,33.364643207396675
+759,712,5910.991949698661
+760,760,30.139180285378863
+760,775,4153.88704010666
+761,761,30.682225898662946
+762,762,25.071125444111317
+762,780,4221.490798743071
+763,708,4612.630435666095
+763,777,4556.488336812311
+763,763,11.450759686607206
+764,785,4408.823341745297
+764,764,41.726430963240965
+765,765,25.93093039977741
+765,740,7171.730052183078
+766,766,25.21372349084662
+766,797,4867.0103279043215
+768,729,5120.00203790734
+768,768,17.427993699093665
+769,769,34.04875483289709
+769,725,4496.280920190264
+770,770,31.160399831348983
+771,734,3577.4544711264784
+771,771,12.094588874233487
+772,772,10.900641381805384
+772,738,4648.246332224635
+773,773,27.386054980349897
+774,774,8.51637850647863
+775,760,4201.769935266226
+775,775,22.30363655513103
+776,776,35.61863385351351
+776,738,3951.817090037485
+777,763,4549.183427650851
+777,777,3.2276976928289938
+779,779,9.641881617592936
+780,762,4248.850734375759
+780,780,15.5897022319681
+780,758,4182.9728686445815
+781,751,7173.06779116899
+781,781,22.73439676233277
+781,728,6082.728524705174
+782,782,17.750195649531882
+783,783,22.34043481627896
+784,732,4533.910879407279
+784,784,35.66339448550627
+785,785,15.153984099699237
+785,709,6251.849124347137
+785,764,4450.71196771372
+786,705,4696.624429529764
+786,786,37.87816186404299
+787,787,11.297395190590814
+787,817,6024.497281870205
+788,788,21.551902074086428
+789,820,4355.614651769525
+789,789,25.462447874613897
+790,790,26.661977453821084
+792,792,36.12126771713934
+793,793,35.07781842306072
+794,794,16.309007002544412
+794,735,3860.6747202666106
+795,795,36.82079020085475
+797,766,4847.043278450503
+797,797,6.788000367040977
+800,706,6780.11098191255
+800,800,29.76571976363768
+801,801,27.801979819735635
+802,830,5862.920789023276
+802,802,22.140036779358745
+802,711,2354.989679664966
+803,803,28.59099197592348
+804,804,28.211126059148928
+805,805,19.869505827988288
+807,807,38.00559090137708
+807,716,3554.6368446087795
+807,810,5110.047737496072
+808,808,20.603286819175075
+809,809,21.020563782070106
+810,810,24.084476991594386
+810,729,3640.3492802754736
+810,807,5126.239532616572
+811,723,15.231398879334632
+812,812,18.577115691273363
+813,813,6.66670499628605
+813,744,7176.83285984452
+815,703,5298.98320836614
+815,815,27.68937113998099
+816,816,34.2795558970545
+816,703,2525.60998556929
+817,817,23.424565536236827
+817,787,6021.175135227668
+818,822,4179.094327758693
+818,818,38.780319488837584
+820,820,32.02566362309998
+820,789,4370.912727248568
+821,821,17.543362564670293
+822,818,4111.626318157001
+822,822,32.76819746475526
+823,823,13.53296329875112
+826,826,11.912542310393269
+827,725,6510.988976904276
+827,827,3.5888459239237336
+828,828,16.682792457674925
+830,830,35.40881557157518
+830,711,4242.959870782587
+830,802,5840.97956528265
+830,801,6880.274479331272

tests/hipscat/catalog/test_catalog_cloud.py

@@ -6,8 +6,9 @@
 from hipscat.catalog import Catalog, PartitionInfo
 from hipscat.io.file_io import file_io
 from hipscat.io.validation import is_valid_catalog
-from hipscat.pixel_math import HealpixPixel
 from hipscat.loaders import read_from_hipscat
+from hipscat.pixel_math import HealpixPixel
+
 from hipscat_cloudtests import TempCloudDirectory
 
 

tests/hipscat/catalog/test_index_catalog.py

@@ -1,5 +1,4 @@
 import numpy.testing as npt
-
 from hipscat.catalog.index.index_catalog import IndexCatalog
 from hipscat.loaders import read_from_hipscat
 from hipscat.pixel_math import HealpixPixel

tests/hipscat_import/test_run_soap.py

@@ -1,15 +1,12 @@
 import os
 
-import pyarrow as pa
-import pyarrow.parquet as pq
+import hipscat_import.soap.run_soap as runner
 import pytest
 from hipscat.catalog.association_catalog.association_catalog import AssociationCatalog
-
-import hipscat_import.soap.run_soap as runner
+from hipscat.io.file_io import read_parquet_metadata
 from hipscat_import.soap.arguments import SoapArguments
 
-from hipscat_cloudtests import TempCloudDirectory, assert_text_file_matches
-from hipscat.io.file_io import read_parquet_metadata
+from hipscat_cloudtests import TempCloudDirectory
 
 
 @pytest.mark.dask

tests/lsdb/catalog/test_crossmatch.py

@@ -1,5 +1,5 @@
-import pytest
 import lsdb
+import pytest
 from lsdb.core.crossmatch.kdtree_match import KdTreeCrossmatch
 
 
@@ -20,7 +20,7 @@ def test_crossmatch_with_margin(
     small_sky_order1_dir_cloud,
     small_sky_xmatch_dir_cloud,
     small_sky_margin_dir_cloud,
-    xmatch_correct_cloud,
+    xmatch_with_margin,
     example_cloud_storage_options,
 ):
     small_sky_margin_catalog = lsdb.read_hipscat(
@@ -37,12 +37,12 @@ def test_crossmatch_with_margin(
     xmatched = small_sky_xmatch_catalog.crossmatch(
         small_sky_order1_catalog, n_neighbors=3, radius_arcsec=2 * 3600, algo=KdTreeCrossmatch
     ).compute()
-    assert len(xmatched) == len(xmatch_correct_cloud)
-    for _, correct_row in xmatch_correct_cloud.iterrows():
-        assert correct_row["ss_id"] in xmatched["id_small_sky"].values
+    assert len(xmatched) == len(xmatch_with_margin)
+    for _, correct_row in xmatch_with_margin.iterrows():
+        assert correct_row["small_sky_order1_id"] in xmatched["id_small_sky_order1"].values
         xmatch_row = xmatched[
-            (xmatched["id_small_sky"] == correct_row["ss_id"])
+            (xmatched["id_small_sky_order1"] == correct_row["small_sky_order1_id"])
             & (xmatched["id_small_sky_xmatch"] == correct_row["xmatch_id"])
         ]
         assert len(xmatch_row) == 1
-        assert xmatch_row["_dist_arcsec"].values == pytest.approx(correct_row["dist"] * 3600)
+        assert xmatch_row["_dist_arcsec"].values == pytest.approx(correct_row["dist_arcsec"])

tests/lsdb/conftest.py

@@ -40,3 +40,9 @@ def small_sky_order1_catalog_cloud(small_sky_order1_dir_cloud, example_cloud_sto
 def xmatch_correct_cloud(local_data_dir):
     pathway = os.path.join(local_data_dir, "xmatch", XMATCH_CORRECT_FILE)
     return file_io.load_csv_to_pandas(pathway)
+
+
+@pytest.fixture
+def xmatch_with_margin(local_data_dir):
+    pathway = os.path.join(local_data_dir, "xmatch", "xmatch_with_margin.csv")
+    return file_io.load_csv_to_pandas(pathway)

tests/lsdb/loaders/hipscat/test_read_hipscat.py

@@ -1,5 +1,6 @@
 import lsdb
 from hipscat.pixel_math import HealpixPixel
+from lsdb.catalog.margin_catalog import MarginCatalog
 
 
 def test_read_hipscat(
@@ -16,16 +17,17 @@ def test_read_hipscat(
         catalog.get_partition(healpix_pixel.order, healpix_pixel.pixel)
 
 
-# def test_read_hipscat_margin(
-#     small_sky_order1_dir_cloud,
-#     small_sky_order1_hipscat_catalog_cloud,
-#     example_cloud_storage_options,
-# ):
-#     catalog = lsdb.read_hipscat(small_sky_order1_dir_cloud, storage_options=example_cloud_storage_options)
-#     assert isinstance(catalog, lsdb.Catalog)
-#     assert catalog.hc_structure.catalog_base_dir == small_sky_order1_hipscat_catalog_cloud.catalog_base_dir
-#     assert catalog.get_healpix_pixels() == small_sky_order1_hipscat_catalog_cloud.get_healpix_pixels()
-
-#     catalog = lsdb.read_hipscat(small_sky_order1_dir_cloud, storage_options=example_cloud_storage_options)
-#     for healpix_pixel in small_sky_order1_hipscat_catalog_cloud.get_healpix_pixels():
-#         catalog.get_partition(healpix_pixel.order, healpix_pixel.pixel)
+def test_read_hipscat_margin(
+    small_sky_margin_dir_cloud,
+    example_cloud_storage_options,
+):
+    catalog = lsdb.read_hipscat(small_sky_margin_dir_cloud, storage_options=example_cloud_storage_options)
+    assert isinstance(catalog, MarginCatalog)
+    assert catalog.hc_structure.catalog_base_dir == small_sky_margin_dir_cloud
+    assert catalog.get_healpix_pixels() == [
+        HealpixPixel(0, 4),
+        HealpixPixel(1, 44),
+        HealpixPixel(1, 45),
+        HealpixPixel(1, 46),
+        HealpixPixel(1, 47),
+    ]