From 5420930a2ef123913bab5aad2fc4930f30d4f622 Mon Sep 17 00:00:00 2001 From: delucchi-cmu Date: Tue, 14 Nov 2023 10:14:04 -0500 Subject: [PATCH 1/3] Fix test data for removing partition_info --- cloud_tests/conftest.py | 6 ------ .../lsdb/loaders/hipscat/test_read_hipscat.py | 11 ----------- tests/conftest.py | 6 ------ .../Norder=1/Dir=0/Npix=44.parquet | Bin 7086 -> 8798 bytes .../Norder=1/Dir=0/Npix=45.parquet | Bin 6566 -> 8278 bytes .../Norder=1/Dir=0/Npix=46.parquet | Bin 7078 -> 8790 bytes tests/data/small_sky_xmatch/_common_metadata | Bin 3421 -> 4423 bytes tests/data/small_sky_xmatch/_metadata | Bin 6188 -> 8210 bytes tests/data/small_sky_xmatch/catalog_info.json | 12 ++++-------- .../lsdb/loaders/hipscat/test_read_hipscat.py | 7 ------- 10 files changed, 4 insertions(+), 38 deletions(-) diff --git a/cloud_tests/conftest.py b/cloud_tests/conftest.py index c5a5eda1..31ffa5e8 100644 --- a/cloud_tests/conftest.py +++ b/cloud_tests/conftest.py @@ -10,7 +10,6 @@ DATA_DIR_NAME = "data" SMALL_SKY_DIR_NAME = "small_sky" SMALL_SKY_XMATCH_NAME = "small_sky_xmatch" -SMALL_SKY_NO_METADATA_DIR_NAME = "small_sky_no_metadata" SMALL_SKY_ORDER1_DIR_NAME = "small_sky_order1" XMATCH_CORRECT_FILE = "xmatch_correct.csv" XMATCH_CORRECT_005_FILE = "xmatch_correct_0_005.csv" @@ -67,11 +66,6 @@ def small_sky_xmatch_dir_cloud(test_data_dir_cloud): return os.path.join(test_data_dir_cloud, SMALL_SKY_XMATCH_NAME) -@pytest.fixture -def small_sky_no_metadata_dir_cloud(test_data_dir_cloud): - return os.path.join(test_data_dir_cloud, SMALL_SKY_NO_METADATA_DIR_NAME) - - @pytest.fixture def small_sky_order1_dir_cloud(test_data_dir_cloud): return os.path.join(test_data_dir_cloud, SMALL_SKY_ORDER1_DIR_NAME) diff --git a/cloud_tests/lsdb/loaders/hipscat/test_read_hipscat.py b/cloud_tests/lsdb/loaders/hipscat/test_read_hipscat.py index 2603c9c0..d932092b 100644 --- a/cloud_tests/lsdb/loaders/hipscat/test_read_hipscat.py +++ b/cloud_tests/lsdb/loaders/hipscat/test_read_hipscat.py @@ -56,17 +56,6 @@ def test_read_hipscat_specify_catalog_type( assert catalog.hc_structure.catalog_info == small_sky_catalog_cloud.hc_structure.catalog_info -def test_read_hipscat_no_parquet_metadata( - small_sky_catalog_cloud, small_sky_no_metadata_dir_cloud, example_cloud_storage_options -): - catalog = lsdb.read_hipscat( - small_sky_no_metadata_dir_cloud, storage_options=example_cloud_storage_options - ) - pd.testing.assert_frame_equal(catalog.compute(), small_sky_catalog_cloud.compute()) - assert catalog.hc_structure.get_healpix_pixels() == small_sky_catalog_cloud.get_healpix_pixels() - assert catalog.hc_structure.catalog_info == small_sky_catalog_cloud.hc_structure.catalog_info - - def test_read_hipscat_specify_wrong_catalog_type(small_sky_dir_cloud): with pytest.raises(ValueError): lsdb.read_hipscat(small_sky_dir_cloud, catalog_type=int) diff --git a/tests/conftest.py b/tests/conftest.py index acfa3ba9..07c8c270 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,6 @@ DATA_DIR_NAME = "data" SMALL_SKY_DIR_NAME = "small_sky" SMALL_SKY_XMATCH_NAME = "small_sky_xmatch" -SMALL_SKY_NO_METADATA_DIR_NAME = "small_sky_no_metadata" SMALL_SKY_ORDER1_DIR_NAME = "small_sky_order1" SMALL_SKY_ORDER1_CSV = "small_sky_order1.csv" XMATCH_CORRECT_FILE = "xmatch_correct.csv" @@ -34,11 +33,6 @@ def small_sky_xmatch_dir(test_data_dir): return os.path.join(test_data_dir, SMALL_SKY_XMATCH_NAME) -@pytest.fixture -def small_sky_no_metadata_dir(test_data_dir): - return os.path.join(test_data_dir, SMALL_SKY_NO_METADATA_DIR_NAME) - - @pytest.fixture def small_sky_order1_dir(test_data_dir): return os.path.join(test_data_dir, SMALL_SKY_ORDER1_DIR_NAME) diff --git a/tests/data/small_sky_xmatch/Norder=1/Dir=0/Npix=44.parquet b/tests/data/small_sky_xmatch/Norder=1/Dir=0/Npix=44.parquet index c462a5f0410febd18d29afb4200e2a3ecafc1d3b..576537515e812327521186a8f1a98a605549db16 100644 GIT binary patch delta 2801 zcmb_eU2Icj7(V@3H@0F5$}sVQ7(=`;#^{Zg1n-RTLb*0Zec$hNKW!Pf#4qXh_x^m( z`@YY6(oaVpj_;Wy(lD>*S3CIMv=Lt7xf^@#YfLwOn$vLx6Ma$R>*IJW$8)?17LHR@ z-|JwiYwv54nobjM;0-o`-q7{Z>pBY^)SY6st49BmY`@VZH5wB=)SbIvJE7yHqHrcTLw@p>**^ z24yeD>5Upp1jp~?%?`*|BYr*J3T!rWN4A{cdRIT@PSR*3PW*nr#%mcYt6+#-Ud-ke zd97J8uWHOUHR1zNyery-`WE{ZZ&nA?;EC|ysmH*>TS1RAI@~8)nmZ{@84C)43+&>`{-qk``W$o!P?zj=YXAW_e8tzkJxy&ja%L`9Nb&pVou27_?Q`1o5`!`vFrVs55(+h^sRlD$++ z6>|2|{r1g!)!@=nt;7U!amnRG{bHq1sx@EJl&L=eY-)GcxZI(y5O!0$v-R5@dZRDx z{qXGY$cdp7!}eaJUDHDv%C*sm9xQ7t( z^ewYhpPf-EG-9!-8=G)2JeoAdZ=ED%#%vA79N9yp2N(JVumBcLx|T$rUSULlZTZ81W=T#$4;I+g&Qz};YE3)RJbD}@bk>30!lUGjLl{JW2JOK zQgWouqya+m_#&(X-y|e7-|W!S>JYU&6PR99C+E%sVGscEFqTAG?U`4G4V4H1(m(*I zgYgg3bU?3=HsbiWg8RRYOJY0-@m>mRA_2wE8dr54b ze%yDcjw$if5}>@Y3^LTrLrpzXn705S88VLEASoNwP~&Wa33gziFxFwmtM=IWIM!2t t<_stFmr{#YD!HZOnG&t-H+88W9qVnnx}T@F_8W{g_zUqfe30>f?!RrFSaSdX delta 1477 zcmah}OK%cU6dqupO)X8Ec3?0lh>9`x0a$D?sd|RTbdaHi#=^|T@G8&(#kQ2v)Kn7_ z6Jw0cO^k7&iE*n7>%xUA<60AcfD2c~orz0hy!S#-sya_Rsw*`az5I)}f@PC6#RC(YIcA z&%+Tb$2MCH8?E}UXPmE`NH!ih(#c3`x$9+k!)m6GE2p?VblW(Gei$9STh1?RdsZxY zCp+g$V;jVp?g4*tZSXG%oj>?C_?KwTPJQ;2{dA(8iRIV>{*Ugd4!Wj~Crxo@x9R)w zA%?LG^M-1wl1;7hZ@PEi57cz%hUwYV$#QH;&R1t-tKUZdPhBu z>kQuu@%T*Qd{Z|5b^jPy4G>YcQR5<|pjnLP{*vGoEN6 z8}e1t<#;LQM>i~Hbm}ON1}#I|mHBGA67xj;(yHVS*RlzBJvZ-j<&r{4T3pUco%ouX zj2BcfBgzfNp zwn6)N)MR)wmGISiX_Li#P4#d zApXD%Ho#xU8!V75NPGeSjL%MtjS(a!(aX!^k|efPAdiiZZ#k6|HWg3EmGqQqs<=Hr z-ozE~(?WCrdNnUqdA(sF57!l$sLCZY>iqy`q0_+q<&bU*G8^x@vw3lhY}VN(Hh6)LCC{20A&;V zoz!NDc8(1}RV5X$NUa2X(uE68O)7%Q6kh+tiCqJB>^1A+2G?!3Rhl-A6H{WTrDMl-Hnp9^X|Wv+mA!VHO>AeI zKgX3SUpR0<5az;-QdLNx9IBK9LL7?3i3=46)B^&E0|LQ;YYPY78{3KFHff7k#j`W> z-uvD+?|rk%C&NEnv`&%a5U=O&>-e{|F+RX^o7VRvm@fLV- zT*LE$j(O6zG!MHiyn#2k1bR(tqZREN7)9@BBL{_9bPqKaYSOb5^jA~V+nK|{+he?z z=XyD=#|3HG10;HR#{h7&(3@QsOxt!BZ*uxEVfaw&zhHzJ+fNV04`MrMc-X}bNxkL) zTI)W=f?hPw)H_-C@a7h4y;%dC+ZZ>hq{m%>jSlWq+X}XgZY*#>qX}^0&7&?}%Z}z0 z42h-Xd|{c_Is%UOHI5H8;vKj6j@u>F+u}RWoCDCP1L;AkC(y%PL60K>?yg|gPVdKK zNUx5icM7^OrJdZL9`PHGs{!&vsHbx~1L+&;~cxevtM+UXOjve8z# zTPMUv$AR2qSS_e0l7qYPeM9k}$KNy$IPPBmyS<~dujlusGE}2Lj~kDgwg+6iz9Gp; zqpR!oX>t1`RgFiEbo9dcm7fy~EAo7yoaOEGGvka~dqUf4H8{fp>)M#HQ_#PS$IV;f zFS?fK;cg0hV8w4uEgRmz4&&$69V6Vh4&&ad)M5Pc<(){lDjmk{z8&LZSK`@Z{5&ca z8_L^Pc=hL0p7=kD<&e0a0#5E}mE)G%5fFPETd&$V&N(0$)NCc6T@fCc?*3$6({-^c zPqQ!CnwOt$zG!nPmDORNXTa^Ltd-T;)zz9>@C4i*woU(f!L1!rFa&k;O#X-QavtZBASFN5tA`y7PIFJndg4ImQznDqK2^n9C+`Vo6NZ&J=8=hF4 zO-uAUo3Ok1!{+SHru^Rcb&1w(LbsZ$Oka;Hbb;@8C!?jBq*RMic~+fGZ7imJ{`p+$ ziZZDX1#R?x?_on}Ek72o(Tlyp4$h#x->~SfX46H?)io(cOCtyP(yy)d{(aJ<7FlXE~5_!lWq?#ZVxZ^nyx^|+uGTVH)B9}5vBGdxiO|8-fBUX#XGlw7o=$mK|_ zFazt*M75BL)a3H$Fgsdhwj@^)YR)&pjupQ=y%rCRY)BRF3i>w6n3&6#->k{L(s~Xt zLsBiaK-S{5iBYKaB`o?mEPg#kf`o*^WSWpGxDl+vdTbFLuHi6Vgdxb<1O{l>_0xlc zuj2%Q;0d9<*+!g^5%>k?ve8mi_9f=?kx?~QPL>MHMxuliF(JGOrX(>UE$jshb^@YX zkTnE@$fe65CL}Z z*JYq8!#qR?=n$xaYMBELGi)ZKm|+$q8*I(+S!vkPK&J3b@A39J3FewyC32<3Xgbaq zj5BXGWwQ`YqsdZcXcW3Tv5qWAcsIWIw0J}rpO{bkpzGz)8m#KDu?w)$0T97}wh|C5 z4-%+`3e6*iNMfORfbdvD0*o-5vT#Yxd6rI=;CzvE8G%p>4MXagnl@&WCDLBHEH~yH6@i>Vi{4rfr_Bj4X?1@{; zaU6lX>4F6kirx+(l}c6D2v}5A*Ijf~v8bv_ELtIcK!uP%EPC&Ak`hy>)Mw4ieec|R z?zv}d|9a)S^HZxdJx@gPi=BLBOOhob?oZtw68FE!*+s#z|2lR?BsPHvkYW=Ad_d-J zcAm*T8+th+5(ja(CH95w17_Ji#VU5+7Jpyar2XZP$ebc?!Jdq%fx#Y=WI9P~L_94B zqi$%+7l=4bW_+lT-5b8_d@$`MPS+wfyfmKtKO8Zx_TISuKD8?jRI82j)ecXr|6B6o z14o{ncmc^zoMGq9kKIJ0$9$OP-1 zTxZWu%465Pf7%acgjXRQtMdNrJjBrO0r1h(5eNnkfXAI10Dt)Ks1lqR03Vz?0>Oy^ z@LePO-X3}8FZaxncXa0I*=a#=`6NfPV3i6j>BrHBKb>mZ+3xhW;fr&Dxqy7b=aKv1 z=Q|r;wrbm!yyTJR=N9LJ@(n>6qv`ZYcByZj^!=nCSC^{HAYuFRT8!Nx@`R!*lv2EC zv=e*yJNa7llnZLTV+K>zQascwHq!Nth}O~~iAEutX&LblBW~GcDOG@~=6ooykyGm0 z=5|H1GR>mJu1IHx`_Wz3xn6F)Q_&i!u91xgis!Z3N=><-HFleMD^WGAE!fkVxm<@h zu~X{uK0zaw*ehg1ZBDvbY3q@Nj%Ec=LnohI+{))tZTL&RH~``350kYN}dPQ}abN z9x$@$vZ2=ZN|a{tl>r)|@hC(WA&f=2N`-6ZA*Z8T@KCmlY^n_Ib-`)etNz3xf%abn zIF{1QXF_#+6((+}57%Q6ka>Flg6RAjn`cVx3Wyxv}w}Bj!m0NsTlueUE4{V7TdX0_S$hav7N2+ zshcyHE;9k)qa#L7E6 zZ{FWG?|n1!N9Vo|9+@FYH*er?8~NY#F+Rz2caA(TT6hz0qO-bTs@I+G;!GAgsS6Eo zyq@DZ-U1(v(|q1HGN0|cx_O-ez3AP(5jxnVIo;|DALR5^*St<=p~sDjEX6HjWUMoV zg}26dJ&We;FD%G-tktcCt#3^C`3DfVBSFZRCWaPYl$AMVlI9X|+ewVT2H zrbmN&`0!k#6K)URw*%MErvtEUP_d*A!GNpUXyZ<`Nw9p#iG@w-bV*KpIOyQ@43=Fm zC045WLY3FslD6AA+j~0k2T}Y~bO?<$|Bh((VQA2S@F3Mw;NhxZz!3wsB$&0syE%mL z8dx^F&=pkL!M%H0{NohtwB_6p>e<-Ng71^1!&Sk6BL;+Tqbb5lJGjq>TeUMJRAsZR za$lbjKRg5Ao`|&*>Z#!1biAW69f{f#m~SbKkq+fw!mbhyQ-^ZzDd|u?I=Sol#G!aTBEJZa zMU8v=Qm^rX+!Oz2x|qfNC@{FERkpWJ+md3BZEMiVarR-sq-HDm?5gm%cl)Qq>s>~6 z=xO#yfAjd$JC=TjQdt}KxQ3ms%6eI?UtO!K1=j_qi!IYXUv}z)3c4V1p38m!j|awE zPzt5|)k?O^;^FFZ`O~?h_b$4T3Z%D0?J#|fEQ32)!)|G>~Qog0rX z&!;8&Eibf$-+6m?Hsu?OUzccu7rND4W%hbpp-abxoXJS3E-AI5RGwGoQ=7|ak8d%T zx}r=gL_r<>&N^T!t>=UBI(@?`?8X_j4VjjGwQRbGxs20tq~txomVVi99oi>N+Ow$2 zQ;DTaI=KKa0e_JK^EdZdR>}YRdSJ%BA3JU z!W_tfXswV6*X8oWINMrfz9d%?YR=YULCKmGLH|w&ew2{M@ z0jVBaBJ1&bbOLI935$LXi{FS5KOun-nI+^3F8FI82ba;{8V*xM82qeFpn!&5KQlUd z9w*=gPXP7JHsXwU;q@DIt2jZii%k+B)ZI4}yaaK4#y$Uw*`VK|L@3;;?H3avuE z4H=-yFb@y{JOrq~T4sR59Gl5VW|;ZOCX-owRvIRn$P9kzgJaeP@#mUYC30m4(QKR{ zm}1s!%4PwaMv|qBdjh%}-9Qv1)bcMr%O9gmMHkZ^=z4jg4pIXaS^-u&0))|_tpo(i zg8*uQLUW4&l2~ePAQaRnzyPx;b7$4@*E6uZxdEmaqXY}j6dgNbHw{UeRDI4kHEkHc8%*q&gv1p_-0{a`jK_%`)gQA#_Bj4X?1@#& zaU8WWON3Z}W>-~$1!9SivSGmjv0&E)ELcQhSMdvwKy2XN=U|ATMdDdAbKg7X-h1vD z+dsrV+g#j*+!_vpnw;)|g^M5`!2GR5h@HaLF(c=m5S7{8QhZEpsb0)y^m&Tds#RT}YnFIfv z+4KF&o}7Q$^W?=Fg5V8E?oQdNmb=nd(`Vl<_NMF;&!^F?)zE54eh^rZN8mfh`}exd zBTK%sBCoA(tcK+Wf;0=c+)m-n$UEuFg&?f~sJ5{bo!ZVO?1xM8yrL@rKo`XOiL>*i zkHwW*S#9;raHdgBMmm*tuC*W2x>_vNE*J7$BN?&TE!k@+4TKuzS|qezR9f1>QC+k0 zor-0@wS3bxiXPp(H7xG+>smWAFbc^~vHFR1e>8fX5kAYp2zQBgB%6*U<$3b~r0 zwoa=61$t$O#vmETXbZzcoU2s0b{_I_aEK4(&?sbT=-vRGro9?W{aHZ!uL3ldGAQLE zEqWCuZ9xf-1*GHl;Dy^m@{-!zeyD(dml1xtYw%7VLe@`qk?qXo|Bia@E?iyz8#d=RY6=3 I1o~6|1KDL*8UO$Q diff --git a/tests/data/small_sky_xmatch/_common_metadata b/tests/data/small_sky_xmatch/_common_metadata index 1045d34643adb9818d6e6c0e8443e1f78fd9a68b..312a2a79be89cccbe6f1e87d4d99152c10e4b421 100644 GIT binary patch delta 1494 zcmb7EOK;Oy6t>#XCRHneXe75nkja2RFw-`#VZx}4j@{UCh|?x5B+iUfIgXpUaZ=MX zNnHu`2h0jxv1Y>(At5#_*t6g#AhvAy2h3vToa>Z|qAq$R-<*3M-~G-b_tX6Q`0N@~ z64B7yyNn=C^sH^WV@7A6&aTJCZj6Z&rQN-zy(dKJFuvl9q)^Fo4xZWzV_&CUKkJX{ zfmi&OSbWQ|-R5T3?AY;pvG|GJGrLW*x4G|o+~9QgQ%Umg_B)>cy_#Jdi>Y|kk8mU9 zIk7Y%`y<$~_MB*JD)i!hBE7J*kcdCQ1V-Tld#~s0b}8v3+EVc+f;bJ8%4+pqP*d@3 zbdlB&H1;L5JYEEa-MJ9I3L4&8pjH|Awyw!iUJ!LZV4&8L5Jcz5~m_l=3~Np}LOMQG7c{acEZ+-|A#j znlo%^-MVXaOI{~m(vIA`1PsC@lYP`Ck|2$rv}LO9&`OrJn@ZMebSkb5 zN8G9cwCG0s1-cVJ+xZ?)It@r((khjg+~L6n4hs`ARNhIGu z=Ys*-$g_n#mdaE&6~KZOLc^kb>&_WzVjM=edruK!(PH2%z02;$lkuc@wX-b_ntrP}d^$dVHgx*Fc(5&|;_0{~ zUWJ5vP^%ql2RA6fr$~Zi3QBMo&Mua!h2UjaT$5E90BAsdQuy#S{Px!Uc2jXDmQ?Aq zikX4w)!g&o+p-WuR)sfrqf35$VERF7<-uHgedpn-f%uiBmqHF1%-l()&UVUmw z?$~tL6P9BaN8PiL>Xb7?7eL1$Q|=6tRWvMes89tc;0MR3KyHE)v}r1kO#y)2lf6WO zB2~tnhvcNfx#VqBikW`1Az$iJG1-vZk)hOgZ^Am{I1VR(t{Fh-1e?nNRgS`V=3jsm zQz{)(siaJ$m^8Fn+fdxg7Jx?gg3%ll^Qi5kTF7&jGUv`!4hKD4i)bf*WPCAz>@{5!qbk UFJa?(u)Q8z2~}4^A$mYR0h2WIvj6}9 diff --git a/tests/data/small_sky_xmatch/_metadata b/tests/data/small_sky_xmatch/_metadata index 2b26d7b3865011093c0d5e28140555c680e5ceb9..38a344ef927620f9bdc269ed7d40caf1e64de72b 100644 GIT binary patch literal 8210 zcmcgyTWk~A8J>iY&9WC*h>b(tZfcg{7D(*aVI4IkV<*0CY={BhR&6>np2U-kFZdGM zp;Ro*vh<-9LOkrQs;Y`s+trd%Hj4zJsEVq6Sh3aeR0$zOQN;soX^X0T*hPr#f6k0O zV~-t>Ww2m;&VMf7cm8w!@BALvPx+0SwyrA{Lf4*QV_clpbP}31TUvFkd{io=X|b>> z%CTapBTY|m>2xwJR-cCkWqmD63v5oHvs`o{%4f2KrWGb^_a)O&4u+u0su5l)wfRzf zfzY&p@#TpapUSXwc7jhtxxyCx*y4ua^#nm^Z?$y&pleb0YOgrQkg*rW5B2q;u|sC_ z?i8Iqoa3_EyKQ>Snl1~`zOD-`K+~%AYmYytz1>EhUPX53bsaDnLEQhvuvGf`?O`pk zt%jB8>d~}mfGYuyb?`+Hk4Im6Nr9fGOX#PbBX7PyF0=#uCs&UX`p5L$FKd>a%eoHtarddhPsW$g^3~Rc@E_`A{PGFG zBrdgRms`kt8_6phOM8GED*Bxuj+OVI!rBmtdA&?RK9>p>Uu_~k+w`K4qfX4QdxwYj3O~7h7euyIod0;>0kl zU5vU4YYj5>)J{gfx<-MH)efVdTTPyMp8WM{IM(kcDF1ivBDFhlxGS~3Ba7kH)`s9O zyduL(YFEO4w3a;aBH-IU-0{WkpIJfeKAHdK*3@3#GFrac+7SLyr;M*uyFsm}b_*NG z|E+&IIwSAF<&83l3biY-_*hFm&^&DxOW%`O)KI(9wmj66^IgDY=pTPrxPJkuoj!SX zk0bbFtafT^L)-G(@5@pktKFbhRJ-%tH49!tH8SgYH zJ=%Ps2CR0WTRRA9xA^LcYBx|;yF0aWyPIv~uN1l6^|IRCR^)aS)*58$shx~| zu5NC3{RQ$jD7X99OCEDuECXss{rl}7{_Pzxw^Lgif`=e3C$?3&-3OiIKT&R%f9vD- ze*V@FsNKm8<_qsHi)yE~HiZB6V{*H@8_7#gDYujN;Mn7GyZf8S<)@U}$t-R>S#CGK znY@Z}yWd~?wCl!|cR=mVUb}X-llEe@Q(GI_md{tr?XHvL`6lId^77~F=XPhdllPjG z+sSyBSIF(I?j#pbZnqn&9eN_QlWPt6bm{TA-Su{IzMs5N%I${V+cEIFKO(hTj^Doj z%B32xrQB{2=XO21_7t6n(iz=@HAf8i$L<816ms!I#xU6ZrlIn8u;E8N-3GB){9(-D zUPlZGI?h4=L{1Q($0W}Q(TPf%h%%8arcg&iF2S?OC})7)iCjEYEH@;XX^zbzdz{PC zQ94UQ_ag=_0p0n;6dHTzpkvqThNDMiGLxH743Os94)tx zp}wdNA3O_ghdJ{UjDv4H3Br&%g`s52hl7@xaL7wht|W}N?QzG2T*MhLhfKa%#udmi z2`|5ow=r(PnxsRvxv~6#>P6q>-h{Ku-2aN*i2R@p?cnMpKiTh)$(b~s_ zY*Je!!+?)`)8jl^FOvvlS(l((Q=D;HW6_atP0r9i!?=K(tAa~0e4q}OgeUflxy(d@ zUc5HdUfMn7HrDBYbL5y)?o}jSm<@-xFBN6Tku@<1N# ztGpMm{*Wn9WaHMNcm^V=NU$&yGL_CA#j|n8WF+J%GW~AM`9$#BwIwAILzw2ynh<=ujz+)IO}+ z_u0k3KBgbeS4iB$8fUMfz1ALsKJ|I$#PK!aGQ}ju=e*)RJ(2(XPND9ZE4S73kC2Ow zc%`xGy{zE*Brz~!$ zXb|x%X{Op|mF<=HfT*d)DAMC_zefN(6psT~W1Rxhha!jK1nwX%=s;QmN&)vxA_B4N zOED%hzH7V($8%o0x!_LVyF-rCIOvZ_m}StMc6V`Xn#y^L9xGh1aF0{e9(3c5p=;Pa zitgq^sBX(cePjl;A4U%sx}RKKS?`4#YMEt$KMQ3$MZsjiU(AW1 z5k3d-lSQc#lx$Q1*YW(=U${!`M_;wHin1}%dfe2Z62EaT=HtR+aS@m7sOVH^H|8gg zpL*PQG>c2Vm++)6d3>g7JWAZ;)DDWsR|r3E=cM)8Dc`ije{=*p-G=!p{Exh9D~7>b zn1!xrjwx_5^aFo7DT<{=z=RG(c~G78<5bZp;Ij$dl*Xfc5q-qnqo|=FLf0+*3-~TJ gmD+_9;%+J7?G{tvu0=gTtni0u?b3W_wiCzkBPPai z9D!00hahnYBOz{FI6@T>5FDUYB_z~d5J(^>5~rNFAP}c20%m3{*LH-M)$aZ?-~QkK zf8Xrd@4oa=(6$WI)9i@joms${nuU$En#!I(w5=Y#J)wIIJ=A*-zcuR(z;yU?%wk|i z01H?HdSld$Bt3&99mSBWi=P3RzgZW{IA(D!`cWsG#ZS2&o*7v@i@)XS;lGSX!}ulA z6jJr8!vMp)Kj~r(T488(E&O)P_o>fy6TDyn;O@nHr#~EXI0Cj8|M~q0IIgn>d}zAx z4FLQ1SM6z^#s1fX4FCtRfI~+n27P7Jw0>h8G6I9lW2V<@%rxZ1kxA?IYtxVkL9+%# z%|q%*xd&#g+#~al^q_h4+?8H6RnMZLWns_+($T@yJ`-$XbM~hO`(2~&>4fjVgO(Y$ zYf$u`F8dr7ed5$aUmEi=Cx%{?&zJ$te&*g-54$lspf^DG<&)7qtvai=7_tuN^rMfh zn#Rwq0~-CzljnQ%s@{#1v77w@{meoC4m&JAnS4*K`Hp;O;ey*gsH?k~Cv-EfbEbw` zFRD%MOUv<_7g~BWWBbr}I}i*6(XY14&rJtr10nD0fV0AMdP$zsJ_qiLi&JEDpn&Y` zygsoKLmAe4o|kwCAvs~R9Xo!H{lMT^SA}Z35K3(nJvuRN`Uu zozsijPMfiwY2@qZ^s=)XWgS-%+0}MQtfe}N91rGih~=d+e@m?GHgfgkMxkE8b674U zsyI&W6g!kRq-2wOsvK^Kn_DGuv(%CzGi`L=HNL;QxOt}~m6Ku8AA~u4^6i{FTghc5 z92dx3f-4Zl@Bx=_03&UDf$QWFBXa^kIJ10fYKkB$B3*o$ERw|Th?Z~=4{xbielHh_ z2eYASOA+=nGa8vjZ9LA!wOP=;lLrPbH6@2F_ikw=<9(AyvWG?gw#Q!0HL>Ax4Wx`eBcY#bH zAL`{VKFde<-OhbnNrLqqjFV1E(OmaQ9`2G%c!yl*%R&Mo0gplrcP!=cqeO~G49Cxv zAYXzB2&tr%XG#Ad73X1QzM&9C>|B=s-|4`G=H~%7-cYw+X|26FS!gsS)$Q%Awr6K; ZyIJ0 Date: Tue, 14 Nov 2023 10:23:47 -0500 Subject: [PATCH 2/3] And remove untested lines. --- src/lsdb/loaders/hipscat/hipscat_catalog_loader.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/lsdb/loaders/hipscat/hipscat_catalog_loader.py b/src/lsdb/loaders/hipscat/hipscat_catalog_loader.py index 053bb80f..519bd934 100644 --- a/src/lsdb/loaders/hipscat/hipscat_catalog_loader.py +++ b/src/lsdb/loaders/hipscat/hipscat_catalog_loader.py @@ -88,8 +88,5 @@ def _load_parquet_metadata_schema( self, catalog: hc.catalog.Catalog, paths: List[hc.io.FilePointer] ) -> pyarrow.Schema: metadata_pointer = hc.io.paths.get_parquet_metadata_pointer(catalog.catalog_base_dir) - if file_pointer.does_file_or_directory_exist(metadata_pointer, storage_options=self.storage_options): - metadata = file_io.read_parquet_metadata(metadata_pointer, storage_options=self.storage_options) - return metadata.schema.to_arrow_schema() - metadata = file_io.read_parquet_metadata(paths[0], storage_options=self.storage_options) + metadata = file_io.read_parquet_metadata(metadata_pointer, storage_options=self.storage_options) return metadata.schema.to_arrow_schema() From 1ba7fb2a701b6f9d45e09cfffdc3b114fd6ebd2c Mon Sep 17 00:00:00 2001 From: delucchi-cmu Date: Tue, 14 Nov 2023 10:42:20 -0500 Subject: [PATCH 3/3] Address lint warning. --- src/lsdb/loaders/hipscat/hipscat_catalog_loader.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/lsdb/loaders/hipscat/hipscat_catalog_loader.py b/src/lsdb/loaders/hipscat/hipscat_catalog_loader.py index 519bd934..77fbe613 100644 --- a/src/lsdb/loaders/hipscat/hipscat_catalog_loader.py +++ b/src/lsdb/loaders/hipscat/hipscat_catalog_loader.py @@ -3,7 +3,7 @@ import dask.dataframe as dd import hipscat as hc import pyarrow -from hipscat.io.file_io import file_io, file_pointer +from hipscat.io.file_io import file_io from hipscat.pixel_math import HealpixPixel from hipscat.pixel_math.hipscat_id import HIPSCAT_ID_HEALPIX_ORDER @@ -74,7 +74,7 @@ def _get_paths_from_pixels( def _load_df_from_paths( self, catalog: hc.catalog.Catalog, paths: List[hc.io.FilePointer] ) -> dd.DataFrame: - metadata_schema = self._load_parquet_metadata_schema(catalog, paths) + metadata_schema = self._load_parquet_metadata_schema(catalog) dask_meta_schema = metadata_schema.empty_table().to_pandas() ddf = dd.from_map( file_io.read_parquet_file_to_pandas, @@ -84,9 +84,7 @@ def _load_df_from_paths( ) return ddf - def _load_parquet_metadata_schema( - self, catalog: hc.catalog.Catalog, paths: List[hc.io.FilePointer] - ) -> pyarrow.Schema: + def _load_parquet_metadata_schema(self, catalog: hc.catalog.Catalog) -> pyarrow.Schema: metadata_pointer = hc.io.paths.get_parquet_metadata_pointer(catalog.catalog_base_dir) metadata = file_io.read_parquet_metadata(metadata_pointer, storage_options=self.storage_options) return metadata.schema.to_arrow_schema()