Add tests for additional import pipelines (#17)

* Clean up test data generation * Almanac data generation. * Linting * Test creation of margin and index. * SOAP test * Basic index and margin testing * Better margin testing and auto-formatting. * Fix merge errors
astronomy-commons · Mar 27, 2024 · 4f81c84 · 4f81c84
1 parent 7f5e6ca
commit 4f81c84
Show file tree

Hide file tree

Showing 15 changed files with 864 additions and 26 deletions.
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -54,6 +54,11 @@ def small_sky_dir_local(local_data_dir):
     return os.path.join(local_data_dir, SMALL_SKY_DIR_NAME)
 
 
+@pytest.fixture
+def small_sky_order1_dir_local(local_data_dir):
+    return os.path.join(local_data_dir, SMALL_SKY_ORDER1_DIR_NAME)
+
+
 @pytest.fixture
 def tmp_dir_cloud(example_cloud_path):
     return os.path.join(example_cloud_path, "tmp")
@@ -65,15 +70,25 @@ def test_data_dir_cloud(example_cloud_path):
 
 
 @pytest.fixture
-def almanac_dir_cloud(test_data_dir_cloud):
-    return os.path.join(test_data_dir_cloud, ALMANAC_DIR_NAME)
+def almanac_dir_cloud(example_cloud_path):
+    return os.path.join(example_cloud_path, "data", ALMANAC_DIR_NAME)
+
+
+@pytest.fixture
+def small_sky_dir_cloud(example_cloud_path):
+    return os.path.join(example_cloud_path, "data", SMALL_SKY_DIR_NAME)
+
+
+@pytest.fixture
+def small_sky_order1_dir_cloud(example_cloud_path):
+    return os.path.join(example_cloud_path, "data", SMALL_SKY_ORDER1_DIR_NAME)
 
 
 @pytest.fixture
-def small_sky_dir_cloud(test_data_dir_cloud):
-    return os.path.join(test_data_dir_cloud, SMALL_SKY_DIR_NAME)
+def small_sky_index_dir_cloud(example_cloud_path):
+    return os.path.join(example_cloud_path, "data", "small_sky_object_index")
 
 
 @pytest.fixture
-def small_sky_order1_dir_cloud(test_data_dir_cloud):
-    return os.path.join(test_data_dir_cloud, SMALL_SKY_ORDER1_DIR_NAME)
+def small_sky_margin_dir_cloud(example_cloud_path):
+    return os.path.join(example_cloud_path, "data", "small_sky_order1_margin")
diff --git a/tests/data/generate_cloud_data.ipynb b/tests/data/generate_cloud_data.ipynb
@@ -103,6 +103,81 @@
     "runner.pipeline(args)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### small_sky_object_index\n",
+    "\n",
+    "An index table mapping the `\"id\"` field in the `small_sky_order` catalog to the pixels they can be found in."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "args = IndexArguments(\n",
+    "    input_catalog_path=\"small_sky_order1\",\n",
+    "    indexing_column=\"id\",\n",
+    "    output_path=\"abfs://hipscat/pytests/data\",\n",
+    "    output_artifact_name=\"small_sky_object_index\",\n",
+    "    output_storage_options=storage_options,\n",
+    "    tmp_dir=tmp_dir,\n",
+    "    dask_tmp=tmp_dir,\n",
+    "    overwrite=True,\n",
+    ")\n",
+    "runner.pipeline(args)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "margin_args = MarginCacheArguments(\n",
+    "    margin_threshold=7200,\n",
+    "    input_catalog_path=\"small_sky_order1\",\n",
+    "    output_path=\"abfs://hipscat/pytests/data\",\n",
+    "    output_artifact_name=\"small_sky_order1_margin\",\n",
+    "    output_storage_options=storage_options,\n",
+    "    tmp_dir=tmp_dir,\n",
+    "    dask_tmp=tmp_dir,\n",
+    "    overwrite=True,\n",
+    ")\n",
+    "runner.pipeline(margin_args)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### small_sky_xmatch\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "args = ImportArguments(\n",
+    "    input_file_list=[\"xmatch/xmatch_catalog_raw.csv\"],\n",
+    "    file_reader=\"csv\",\n",
+    "    constant_healpix_order=1,\n",
+    "    output_path=\"abfs://hipscat/pytests/data\",\n",
+    "    output_storage_options=storage_options,\n",
+    "    output_artifact_name=\"small_sky_xmatch\",\n",
+    "    pixel_threshold=100,\n",
+    "    tmp_dir=tmp_dir,\n",
+    "    dask_tmp=tmp_dir,\n",
+    "    overwrite=True,\n",
+    ")\n",
+    "runner.pipeline(args)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

diff --git a/tests/data/xmatch/xmatch_catalog_raw.csv b/tests/data/xmatch/xmatch_catalog_raw.csv
@@ -0,0 +1,112 @@
+id,ra,dec,ra_error,dec_error,calculated_dist
+700,282.4973369642065,-58.50363666133865,0.0,0.0,0.0038937371796661955
+701,299.49467483293046,-48.49734307091762,0.0,0.0,0.004417089286822645
+703,286.496411087899,-69.50735861948591,0.0,0.0,0.007465148623271675
+704,326.49040324229554,-45.495520527607916,0.0,0.0,0.008081737922778262
+705,335.49823037373636,-32.501887378268684,0.0,0.0,0.0024061731809319658
+706,297.500464791248,-36.501358239740256,0.0,0.0,0.001408690476035035
+707,308.5075969565267,-69.4938383061932,0.0,0.0,0.006711692878024653
+708,307.5029788288972,-37.49153183031141,0.0,0.0,0.008791788627833986
+709,294.49616872024575,-45.504318132438996,0.0,0.0,0.005084975674846571
+710,341.4978471489342,-39.50300614323703,0.0,0.0,0.0034345799375555225
+711,305.50726574449976,-49.50902068709017,0.0,0.0,0.010180129816670302
+712,288.5086919592041,-49.49467628037758,0.0,0.0,0.007759588299716976
+713,298.5065623969436,-41.49142094471404,0.0,0.0,0.009887369277439008
+714,303.50087516287584,-37.50823031355936,0.0,0.0,0.008259544735103866
+715,280.4920029860987,-35.493328898986256,0.0,0.0,0.009321675245426753
+716,305.50485429271373,-60.49012283336824,0.0,0.0,0.010162382656610396
+717,303.49551485898627,-43.493657899870186,0.0,0.0,0.007127972970049637
+718,292.4998857701481,-60.49833460540147,0.0,0.0,0.0016663443024497987
+719,344.5057892953421,-39.504157776676124,0.0,0.0,0.006102577723220374
+720,344.49180401377896,-47.49238721690136,0.0,0.0,0.009413750518319552
+722,350.5066702419223,-58.503990027830326,0.0,0.0,0.0052976885346261715
+723,315.5057480531566,-68.49633092457547,0.0,0.0,0.004230944133156015
+724,323.50025185670245,-41.49218597689782,0.0,0.0,0.007816299791430795
+725,308.49957081155947,-41.495934479946726,0.0,0.0,0.004078208611731745
+726,341.497677897641,-37.50836672303974,0.0,0.0,0.008567120319353118
+727,301.5058853360593,-44.505464475193094,0.0,0.0,0.00689054997375034
+728,328.4985228769857,-47.4959104355073,0.0,0.0,0.004209569964155283
+729,299.5078224357862,-59.492533155301345,0.0,0.0,0.008456928273301101
+730,322.503532621669,-61.4915626897568,0.0,0.0,0.008604085898949954
+731,343.50430005504376,-52.4962925300877,0.0,0.0,0.004538535485067151
+732,337.50491014722763,-39.49745671731206,0.0,0.0,0.00456330407561165
+733,329.5084347497268,-65.49740543323598,0.0,0.0,0.004355207583297679
+734,348.5068364247373,-66.5062662345514,0.0,0.0,0.00683337401402983
+735,299.49082846328395,-65.50918654570688,0.0,0.0,0.009942494082571283
+736,303.4983559057966,-52.49339120107316,0.0,0.0,0.006684167625502773
+738,345.5002276899114,-64.50641097508905,0.0,0.0,0.006411724249653764
+739,332.49949909633966,-57.496764053264656,0.0,0.0,0.003247120497737458
+740,306.502038995067,-33.507853471752895,0.0,0.0,0.008035405355139288
+741,303.5082177891894,-38.49693414493168,0.0,0.0,0.007124813068174697
+742,348.4923647073171,-45.503561288933,0.0,0.0,0.0064281487963744185
+744,349.5067887719698,-39.49355760507958,0.0,0.0,0.008303472507483669
+745,337.50687984813663,-38.49030801833998,0.0,0.0,0.011087303128068821
+746,283.50410682523454,-31.507388615099895,0.0,0.0,0.008176317983887032
+747,327.4941653811482,-61.50798855584927,0.0,0.0,0.008459663708771591
+748,296.5056058933602,-63.49823329956506,0.0,0.0,0.003062401989135037
+749,293.50153068732527,-55.50418476732588,0.0,0.0,0.004273625020931212
+750,338.50522647000923,-67.49507277907341,0.0,0.0,0.005317769316699682
+751,330.49152270935974,-44.50749002371427,0.0,0.0,0.009625751430705213
+752,291.4955894157001,-34.509340206777665,0.0,0.0,0.010022490734344493
+753,307.50382161489284,-45.50660210985809,0.0,0.0,0.007124741363292291
+757,346.4950112209766,-34.499698877690996,0.0,0.0,0.004122403365558441
+758,325.5078334102866,-53.496796766377805,0.0,0.0,0.005654485097803249
+759,290.5093456000377,-48.50689580760101,0.0,0.0,0.009267956446493674
+760,320.5000737589899,-53.49162812045962,0.0,0.0,0.008371994523713507
+761,329.49540674568857,-29.507527142435443,0.0,0.0,0.008522840527410218
+762,327.5099504159638,-51.49681749721089,0.0,0.0,0.006964201512249593
+763,306.503717405159,-38.501285910531244,0.0,0.0,0.003180766579609346
+764,297.5099542731024,-45.509255951474096,0.0,0.0,0.011590675267576811
+765,306.496103225253,-35.50646685874661,0.0,0.0,0.007203036222158015
+766,310.50272138608136,-63.49310227862395,0.0,0.0,0.0070038120807915365
+768,297.4914435257802,-60.49761626032595,0.0,0.0,0.004841109360846844
+769,307.50237859392246,-42.490706041524994,0.0,0.0,0.009457987453580678
+770,285.4927384711785,-29.505914310576795,0.0,0.0,0.008655666619830435
+771,348.49142887507156,-67.49927332524071,0.0,0.0,0.0033596080206363273
+772,348.5019801991146,-64.49709454175894,0.0,0.0,0.0030279559393869
+773,293.50236539635927,-50.507456987826934,0.0,0.0,0.007607237494553983
+774,281.50398704986833,-54.50048563240917,0.0,0.0,0.0023656606961959385
+775,321.4964848355465,-54.49415053054973,0.0,0.0,0.006195454598648734
+776,344.5040629272389,-63.49027349458767,0.0,0.0,0.009894064959312963
+777,307.49890861984954,-39.49969232361634,0.0,0.0,0.0008965826924609473
+779,347.4969228255386,-29.49998150129861,0.0,0.0,0.0026783004493342623
+780,326.49588744940496,-52.49646663506878,0.0,0.0,0.004330472842230649
+781,330.5064861077739,-46.49553401087197,0.0,0.0,0.006315110211752639
+782,290.4939722205269,-39.501636435668566,0.0,0.0,0.004930609902645136
+783,286.49311278633206,-42.49643278377242,0.0,0.0,0.00620567633787424
+784,338.4988614687707,-40.49013140913713,0.0,0.0,0.009906498468190418
+785,296.4952052131704,-44.502454439261875,0.0,0.0,0.00420944002766141
+786,336.49506064610426,-33.49031808212207,0.0,0.0,0.010521711628911383
+787,320.5045402214284,-47.50066309308334,0.0,0.0,0.003138165330728707
+788,283.4922848683516,-61.50472118895328,0.0,0.0,0.005986639465010942
+789,287.50448435842037,-45.506336229262274,0.0,0.0,0.00707290218738277
+790,286.50676981362795,-35.504947384779705,0.0,0.0,0.007406104848292133
+792,320.49034480171366,-69.49055349450275,0.0,0.0,0.010033685476984718
+793,289.505563765141,-58.49070004539731,0.0,0.0,0.009743838450860704
+794,300.49040502540464,-66.50242624855619,0.0,0.0,0.0045302797229310335
+795,306.5098743562623,-58.50883175329203,0.0,0.0,0.010227997278010968
+797,308.4959293826286,-62.499850337011466,0.0,0.0,0.0018855556575143944
+800,299.49634025860985,-37.50774175666009,0.0,0.0,0.008268255489902671
+801,309.50459869420075,-50.49285272539373,0.0,0.0,0.007722772172148005
+802,304.50794381610393,-49.503347861482055,0.0,0.0,0.006150010216472576
+803,336.49605397088095,-25.492901513969866,0.0,0.0,0.007941942215510794
+804,322.5054370607925,-66.50753064652223,0.0,0.0,0.007836423905316952
+805,297.49710125684453,-52.4947704286713,0.0,0.0,0.005519307174432609
+807,303.5090064842533,-60.50958066205481,0.0,0.0,0.010557108583715828
+808,320.5045014375577,-40.50458679591953,0.0,0.0,0.005723135227553679
+809,283.49417113157375,-34.496680696199,0.0,0.0,0.005839045494964553
+810,301.4983229507129,-59.493364244927065,0.0,0.0,0.006690132497665152
+812,346.50457453895694,-60.49535739120018,0.0,0.0,0.005160309914247046
+813,349.49805237860545,-37.50102075111495,0.0,0.0,0.0018518624989871174
+815,283.4983131010114,-68.50766661235697,0.0,0.0,0.007691491983328082
+816,288.49865033568864,-69.49048964470823,0.0,0.0,0.009522098860287752
+817,318.5015571535228,-48.50642450595604,0.0,0.0,0.006506823760067181
+818,300.50725929462396,-55.49004348192754,0.0,0.0,0.010772310969114914
+820,286.50112309222663,-46.49113764300142,0.0,0.0,0.00889601767308156
+821,330.5037920754957,-52.49570836845381,0.0,0.0,0.004873156267967185
+822,301.50048283917886,-54.49090204341423,0.0,0.0,0.00910227707354056
+823,338.5023056673547,-45.50339407619643,0.0,0.0,0.003759156471877518
+826,335.4912329464967,-69.50123435184229,0.0,0.0,0.003309039530666688
+827,310.50102642943847,-40.50062018826431,0.0,0.0,0.0009969016455611704
+828,330.50517018442724,-26.499743067357116,0.0,0.0,0.004634109015995284
+830,306.50925890930665,-50.49212269691352,0.0,0.0,0.009835782103215954