From 170f512baf900b0ec1dae5ad1e4a1bb903346be0 Mon Sep 17 00:00:00 2001 From: delucchi-cmu Date: Mon, 6 Nov 2023 08:40:58 -0500 Subject: [PATCH] Exercise more hipscat_index behavior. --- .../hipscat_import/catalog/test_map_reduce.py | 33 +++++ .../data/test_formats/hipscat_index.csv | 132 ++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 tests/hipscat_import/data/test_formats/hipscat_index.csv diff --git a/tests/hipscat_import/catalog/test_map_reduce.py b/tests/hipscat_import/catalog/test_map_reduce.py index eb91e967..98c9b9f6 100644 --- a/tests/hipscat_import/catalog/test_map_reduce.py +++ b/tests/hipscat_import/catalog/test_map_reduce.py @@ -134,6 +134,39 @@ def test_map_headers(tmp_path, formats_headers_csv): assert (result == expected).all() +def test_map_with_hipscat_index(tmp_path, formats_dir, small_sky_single_file): + os.makedirs(os.path.join(tmp_path, "histograms")) + input_file = os.path.join(formats_dir, "hipscat_index.csv") + mr.map_to_pixels( + input_file=input_file, + file_reader=get_file_reader("csv"), + highest_order=0, + ra_column="NOPE", + dec_column="NOPE", + use_hipscat_index=True, # radec don't matter. just use existing index + resume_path=tmp_path, + mapping_key="map_0", + ) + + expected = hist.empty_histogram(0) + expected[11] = 131 + + result = read_partial_histogram(tmp_path, "map_0") + npt.assert_array_equal(result, expected) + + with pytest.raises(ValueError, match="_hipscat_index not in"): + mr.map_to_pixels( + input_file=small_sky_single_file, + file_reader=get_file_reader("csv"), + highest_order=0, + ra_column="NOPE", + dec_column="NOPE", + use_hipscat_index=True, # no pre-existing index! expect failure. + resume_path=tmp_path, + mapping_key="map_0", + ) + + def test_map_small_sky_order0(tmp_path, small_sky_single_file): """Test loading the small sky catalog and partitioning each object into the same large bucket""" os.makedirs(os.path.join(tmp_path, "histograms")) diff --git a/tests/hipscat_import/data/test_formats/hipscat_index.csv b/tests/hipscat_import/data/test_formats/hipscat_index.csv new file mode 100644 index 00000000..153091e5 --- /dev/null +++ b/tests/hipscat_import/data/test_formats/hipscat_index.csv @@ -0,0 +1,132 @@ +_hipscat_index,id,ra,dec,ra_error,dec_error,Norder,Dir,Npix +12749688880727326720,707,308.5,-69.5,0,0,0,0,11 +12751184493818150912,792,320.5,-69.5,0,0,0,0,11 +12753202806647685120,811,315.5,-68.5,0,0,0,0,11 +12753202806647685121,723,315.5,-68.5,0,0,0,0,11 +12770681119980912640,826,335.5,-69.5,0,0,0,0,11 +12771980657148559360,750,338.5,-67.5,0,0,0,0,11 +12776409575968473088,771,348.5,-67.5,0,0,0,0,11 +12782714789977653248,734,348.5,-66.5,0,0,0,0,11 +12786706826733289472,738,345.5,-64.5,0,0,0,0,11 +12786894563780329472,772,348.5,-64.5,0,0,0,0,11 +12788339839317573632,776,344.5,-63.5,0,0,0,0,11 +12797951905556856832,733,329.5,-65.5,0,0,0,0,11 +12801026705158307840,804,322.5,-66.5,0,0,0,0,11 +12818067795442925568,747,327.5,-61.5,0,0,0,0,11 +12823504327528153088,739,332.5,-57.5,0,0,0,0,11 +12842381331509805056,816,288.5,-69.5,0,0,0,0,11 +12842473731565551616,703,286.5,-69.5,0,0,0,0,11 +12855054043935932416,794,300.5,-66.5,0,0,0,0,11 +12856781556059996160,735,299.5,-65.5,0,0,0,0,11 +12859878138972209152,797,308.5,-62.5,0,0,0,0,11 +12866984851890241536,815,283.5,-68.5,0,0,0,0,11 +12882093266048122880,748,296.5,-63.5,0,0,0,0,11 +12886291525662670848,716,305.5,-60.5,0,0,0,0,11 +12886577464536465408,807,303.5,-60.5,0,0,0,0,11 +12887770713741590528,768,297.5,-60.5,0,0,0,0,11 +12888117478487490560,729,299.5,-59.5,0,0,0,0,11 +12888375204127965184,810,301.5,-59.5,0,0,0,0,11 +12890425758039670784,718,292.5,-60.5,0,0,0,0,11 +12897705201133158400,818,300.5,-55.5,0,0,0,0,11 +12901304742075957248,766,310.5,-63.5,0,0,0,0,11 +12904011555938500608,730,322.5,-61.5,0,0,0,0,11 +12924400840801779712,758,325.5,-53.5,0,0,0,0,11 +12924737222707511296,780,326.5,-52.5,0,0,0,0,11 +12926803467124604928,775,321.5,-54.5,0,0,0,0,11 +12927513300782022656,760,320.5,-53.5,0,0,0,0,11 +12935235931912273920,795,306.5,-58.5,0,0,0,0,11 +12946238438616596480,822,301.5,-54.5,0,0,0,0,11 +12947523513744359424,736,303.5,-52.5,0,0,0,0,11 +12949977409238597632,801,309.5,-50.5,0,0,0,0,11 +12951015418364952576,830,306.5,-50.5,0,0,0,0,11 +12957936896993918976,817,318.5,-48.5,0,0,0,0,11 +12958541318065225728,787,320.5,-47.5,0,0,0,0,11 +12980498864409673728,812,346.5,-60.5,0,0,0,0,11 +12985050869937471488,722,350.5,-58.5,0,0,0,0,11 +13025270726448381952,731,343.5,-52.5,0,0,0,0,11 +13031060802264629248,720,344.5,-47.5,0,0,0,0,11 +13040468461170458624,823,338.5,-45.5,0,0,0,0,11 +13055884976753475584,742,348.5,-45.5,0,0,0,0,11 +13093160001097170944,719,344.5,-39.5,0,0,0,0,11 +13094378277252890624,710,341.5,-39.5,0,0,0,0,11 +13095317624672223232,726,341.5,-37.5,0,0,0,0,11 +13097779065304121344,744,349.5,-39.5,0,0,0,0,11 +13100157308065808384,813,349.5,-37.5,0,0,0,0,11 +13109184215138697216,757,346.5,-34.5,0,0,0,0,11 +13114993892334239744,821,330.5,-52.5,0,0,0,0,11 +13117165557772189696,762,327.5,-51.5,0,0,0,0,11 +13122077940282032128,728,328.5,-47.5,0,0,0,0,11 +13123208770404483072,781,330.5,-46.5,0,0,0,0,11 +13130546552927944704,704,326.5,-45.5,0,0,0,0,11 +13135578070553460736,751,330.5,-44.5,0,0,0,0,11 +13158407025211736064,724,323.5,-41.5,0,0,0,0,11 +13164283224702058496,808,320.5,-40.5,0,0,0,0,11 +13186894729939255296,784,338.5,-40.5,0,0,0,0,11 +13187453677775880192,732,337.5,-39.5,0,0,0,0,11 +13189921792761790464,745,337.5,-38.5,0,0,0,0,11 +13202401744484564992,786,336.5,-33.5,0,0,0,0,11 +13203103043639312384,705,335.5,-32.5,0,0,0,0,11 +13211086588563423232,779,347.5,-29.5,0,0,0,0,11 +13235029212974284800,761,329.5,-29.5,0,0,0,0,11 +13239388759557931008,828,330.5,-26.5,0,0,0,0,11 +13250788433850269696,803,336.5,-25.5,0,0,0,0,11 +13263647230914461696,788,283.5,-61.5,0,0,0,0,11 +13272631885323829248,700,282.5,-58.5,0,0,0,0,11 +13277499429092327424,793,289.5,-58.5,0,0,0,0,11 +13283409463257071616,749,293.5,-55.5,0,0,0,0,11 +13284984179453329408,805,297.5,-52.5,0,0,0,0,11 +13293316792777703424,773,293.5,-50.5,0,0,0,0,11 +13300970211545972736,774,281.5,-54.5,0,0,0,0,11 +13316869903572008960,712,288.5,-49.5,0,0,0,0,11 +13319655515505033216,759,290.5,-48.5,0,0,0,0,11 +13325709382806142976,820,286.5,-46.5,0,0,0,0,11 +13326118614579806208,789,287.5,-45.5,0,0,0,0,11 +13335640766354030592,711,305.5,-49.5,0,0,0,0,11 +13335856080517857280,802,304.5,-49.5,0,0,0,0,11 +13341394068685455360,701,299.5,-48.5,0,0,0,0,11 +13347311673342427136,727,301.5,-44.5,0,0,0,0,11 +13348003826582421504,717,303.5,-43.5,0,0,0,0,11 +13351146793404989440,753,307.5,-45.5,0,0,0,0,11 +13358998609274601472,769,307.5,-42.5,0,0,0,0,11 +13359333484913491968,725,308.5,-41.5,0,0,0,0,11 +13362536511002640384,827,310.5,-40.5,0,0,0,0,11 +13364612928339181568,777,307.5,-39.5,0,0,0,0,11 +13368388511275679744,764,297.5,-45.5,0,0,0,0,11 +13369482380335644672,785,296.5,-44.5,0,0,0,0,11 +13369514156621824000,709,294.5,-45.5,0,0,0,0,11 +13374210622061805568,713,298.5,-41.5,0,0,0,0,11 +13382429402164363264,800,299.5,-37.5,0,0,0,0,11 +13384601479449411584,706,297.5,-36.5,0,0,0,0,11 +13387360701694083072,755,303.5,-38.5,0,0,0,0,11 +13387360701694083073,741,303.5,-38.5,0,0,0,0,11 +13388334615593222144,714,303.5,-37.5,0,0,0,0,11 +13389212170495983616,763,306.5,-38.5,0,0,0,0,11 +13389509163101454336,708,307.5,-37.5,0,0,0,0,11 +13392589952663945216,765,306.5,-35.5,0,0,0,0,11 +13393588426222075904,740,306.5,-33.5,0,0,0,0,11 +13425161974698737664,783,286.5,-42.5,0,0,0,0,11 +13462800185222496256,790,286.5,-35.5,0,0,0,0,11 +13465233373970563072,809,283.5,-34.5,0,0,0,0,11 +13467391906581315584,715,280.5,-35.5,0,0,0,0,11 +13477206946360590336,782,290.5,-39.5,0,0,0,0,11 +13488986123334057984,752,291.5,-34.5,0,0,0,0,11 +13520476867982786560,746,283.5,-31.5,0,0,0,0,11 +13521835979425447936,770,285.5,-29.5,0,0,0,0,11 +13552942781667737600,756,319.5,-35.5,0,0,0,0,11 +13553697461939208192,798,316.5,-36.5,0,0,0,0,11 +13557123557418336256,778,313.5,-36.5,0,0,0,0,11 +13557377060258709504,829,314.5,-35.5,0,0,0,0,11 +13557816572940124160,819,313.5,-35.5,0,0,0,0,11 +13560168899495854080,814,312.5,-33.5,0,0,0,0,11 +13560933976658411520,721,314.5,-34.5,0,0,0,0,11 +13561582046530240512,737,316.5,-33.5,0,0,0,0,11 +13563711661973438464,799,313.5,-31.5,0,0,0,0,11 +13564690156971098112,825,315.5,-30.5,0,0,0,0,11 +13565852277582856192,796,320.5,-33.5,0,0,0,0,11 +13588709332114997248,754,313.5,-30.5,0,0,0,0,11 +13590818251897569280,806,312.5,-29.5,0,0,0,0,11 +13591216801265483776,791,312.5,-28.5,0,0,0,0,11 +13596001812279721984,824,305.5,-28.5,0,0,0,0,11 +13598131468743213056,702,310.5,-27.5,0,0,0,0,11 +13601023174257934336,767,314.5,-29.5,0,0,0,0,11 +13696722494273093632,743,307.5,-25.5,0,0,0,0,11