From 79f5fbc318834fafef9585503e89e501ec125142 Mon Sep 17 00:00:00 2001 From: Jayachithra Date: Mon, 26 Aug 2024 19:02:28 +0000 Subject: [PATCH] chore: add examples tests for synapse --- .../test_data_factory_copy_blobs_unit.py | 7 - examples/synapse/copy_blobs/README.md | 12 ++ examples/synapse/copy_blobs/copy_blobs.png | Bin 0 -> 23172 bytes .../copy_blobs/pipeline/copy_blobs.json | 172 ++++++++++++++++++ .../test_synapse_copy_blobs_functional.py | 75 ++++++++ .../test_synapse_copy_blobs_unit.py | 131 +++++++++++++ 6 files changed, 390 insertions(+), 7 deletions(-) create mode 100644 examples/synapse/copy_blobs/README.md create mode 100644 examples/synapse/copy_blobs/copy_blobs.png create mode 100644 examples/synapse/copy_blobs/pipeline/copy_blobs.json create mode 100644 examples/synapse/copy_blobs/test_synapse_copy_blobs_functional.py create mode 100644 examples/synapse/copy_blobs/test_synapse_copy_blobs_unit.py diff --git a/examples/data_factory/copy_blobs/test_data_factory_copy_blobs_unit.py b/examples/data_factory/copy_blobs/test_data_factory_copy_blobs_unit.py index 4548aff1..6b8feebf 100644 --- a/examples/data_factory/copy_blobs/test_data_factory_copy_blobs_unit.py +++ b/examples/data_factory/copy_blobs/test_data_factory_copy_blobs_unit.py @@ -4,7 +4,6 @@ from data_factory_testing_framework.models.activities import Activity, ForEachActivity from data_factory_testing_framework.state import ( PipelineRunState, - PipelineRunVariable, RunParameter, RunParameterType, ) @@ -27,9 +26,6 @@ def test_list_blobs(pipeline: Pipeline) -> None: # Arrange activity = pipeline.get_activity_by_name("List Folders") state = PipelineRunState( - variables=[ - PipelineRunVariable(name="SourceContainerName", default_value="source"), - ], parameters=[ RunParameter(RunParameterType.Global, "SourceStorageAccountName", "sourcestorage"), RunParameter( @@ -55,9 +51,6 @@ def test_for_each(pipeline: Pipeline) -> None: # Arrange activity = pipeline.get_activity_by_name("For Each SourceFolder") state = PipelineRunState( - variables=[ - PipelineRunVariable(name="SourceContainerName", default_value="source"), - ], parameters=[ RunParameter(RunParameterType.Global, "SourceStorageAccountName", "sourcestorage"), RunParameter( diff --git a/examples/synapse/copy_blobs/README.md b/examples/synapse/copy_blobs/README.md new file mode 100644 index 00000000..83c4f396 --- /dev/null +++ b/examples/synapse/copy_blobs/README.md @@ -0,0 +1,12 @@ +# Copy Blobs + +This is an example pipeline which intends to list all the blobs in a given container and copies these blobs to another container + +![image](copy_blobs.png) + +The pipeline has two activities: + +1. **List folders**: Web activity to list all blobs in a container that has a given prefix +2. **For each activity**: Iterates over each item in the list returned above and executes the sub-activity on each item. + + 2.1. **Copy files to destination**: Copy activity which copies the blobs to a given destination. diff --git a/examples/synapse/copy_blobs/copy_blobs.png b/examples/synapse/copy_blobs/copy_blobs.png new file mode 100644 index 0000000000000000000000000000000000000000..068725e475168c63843a20f47b154fee6c1447e0 GIT binary patch literal 23172 zcmd4(Wl&rH7xs-J#jO-~2rb3k9g3AAZE=_2PH=bkwz##0;_mJw6ao}?w^H04p3U!n z-_NV_=DavFCo^qkXOb^FTh_YPwLU9RA64XWFv&3y5D;+QE699CKtLKmKtPm7M+IJK z`_-}yJR!P#mX}7T7@^z+HeOgtDN7+BR3~8Fo1g&O7)}a0E(i!@)6f49Nyi2cfEP`y z)wNyK9ITy`&0S5*Ok7QX*ANhVKwvv-r;m=-4z4aBVLcy^CrFr+732l{@5lPzmH-#e z|Ho|}Zo&Waj{m*?dE3D6tqcYX0YRtiy^NH)r{Td8n(?=b2Z>{kX@kq0vMO^tooHId zXyp1PGzUG~DP4JcD*LH`;?{>LgS{exs|Fh)v=4}%BrkJyfQQ}xzdj?`ur80v%Jiuw zr>5rB)e$;0F0{(yHU|`|r&vYb9Xt~|AFpw3rUda2|ApdM|bQCsY6cFdwaRY5FB{0;QR;X!JnGoJ%FboqZ+_2T zJ2KK0f(eU|0}hjnOg@2{pt$Sv?yslT)YKf020Ri3+%2pq1-{&k^(`mHhP?~N{v_y- zA4nee=OzBhhvejBMrLMinkxa4y)SP*C~aByM-wfJNr;^E5jgMuN(G6DEwvVU@jEWZ z?&JngyX=fy9Nx?oeadsX*-8apAFnNo4Z{NO=WQ#00B4fKz!l&7;@_5-eEvqQR$SED zb<19bh?U@_rzkoGh7<%+V>CKe_TNDur%1FM&Ins>_u05V8>PSWeinHh#cQObD9>a1-Y1kV@zJUFx)* zKq(krj^>Choa3*!PAD_D*4PIR&CjzLbx?|U9QLs_E(QbF0@Xx;@&d~&^+1V6DZDFB ziYdQH&!{5 zKV6ZRrBhsVdw@QnFAx#A&4)Ze!{9DZtRopdn)mak(nSW5Nw|BsLC5<+MNXEUK1F_x zmCk?wmkdv%c(F@uO26~5ZP%3!zv1s)Jw4KPD;-Pw1o%%sfzu2^>mx>QV9fHnoMTXc zhyeSm?1O~XQvI2NhQ?9u_nj_NWcODdeJ7gK_VSXF(;T&hS7YrySE!{>yQ$TE(vF37 zJF*1P(*bb}D-m2_OdK3pL&F>l*xcM)fpm6aqOrrC0WQ0-@X zyh~67K>*2(PrK{xxL-pYg+R|1#4F+3_GjRhNY3|JmH~!e(g|UmT_03Rg#Sn=?(HD zC0q`8@|z$TemR_D-r#awbvkrBvO0o!7If@?{CLgD=#txx^DbXZyWN}bu;b3*?CN8{ zT`$d}@gqaN`G036q9@2t%gC?(#uG;6k@j-9JIL;+3cMxE&OV|ZDk~h~u+*}aB%4f9O&ATyg>%UKatv*WCmJ zp|f;#cX!{_xktL1MMXswRIZT33PnynjzrZ_K8H1H?gkF0Ae@GeW%|vKN`6eu+6Mat z3)Dg9!G;v{0qpht+D*4PJ=)pN)nFLzQSm&rrDlRxF2CLE>JrPi5GSb%{>$)|iRcnDn0_BmC4j%Sd6yoVd(4IMhKGh%)Sva({^PM3AJxb4BkaX*>{ z)cGm$k5u~tK{E3nU!m=Wq_n9K_Ypy1G{J7^B>o;Qf_!9k!N8Os^1WA`B0c&aY z%2^uQy?7G5rSEgR{}nHP;EmH9Fa7mj@B%WcaCnGD{EY|P8G*9TqskfwPY$s;MVIwC z@y>JMDsA!m-(RCHFbkpyq4n?!P!5f%OFie9651FAzY^M+&nVB~TBh%RVf+7!=dF9jl&1QSnOTFC|GUs(F(R0Mb(}Xjvk61J|tyl+lx1Bw9YXL|C^zB`EvC^NjJFj zVVMRypfgM5?JEKTd@8EA@84;UgvJS<^B)ICMDLoUl~s|4U-dYq}fr6pN!RFk8f9U7+&@2Nj2WQ%wFoX=qDYM~7^EW8>jg zsC@}_^UIf4XRBQ_Dhdjb9+i=G4C0j64dE|Pr#p6Co2YrcIz_1y=AIMTcB?AnuP!}p zK+7`bz>s2++WN`ioZdS`uMKG2b0$)@MF}|pQgmWU%C=bJ0TV3??Hv!wrtSeht>at` z3LvYt&)mI9WH&iY7%ddyqoTOzbTSIukRZ)Oi66+R5)anWuA69xh{9G^ja_afLG<)v zodHk8mHae{lG|6uA$a1PY-A$t`}|KWJ!ZlwNv7DNNL* zqJ2O$LnUZjI9Q@Tkl(-}zv$&U`RxUy+XB2BV~Q1P(Jc3)WqZ`d_Q%`3ri*F4YLi~n zGTnMiAV`Tlr*%;vFD=uX7)JU6nx?3t0_t1>F-(r&OLJ8as(oq+Z@jeyA>E89*L&(;Slw+551_iIO;pk$V5 z;AQ_i`!mqDnZj660*#K)_6jO8do=|9 zXUId>p=Fl=n^N%4{*IrCq~)81h?(CdHI=5zjgCex0Xo08XPDMipDTmSDq{rj2T41- zG7nYz7!*DHS^lv<&}eHt6UNO+%S)Zt7Vxy7Ff0;~03Z#g%XzMnYYnQBdm_qd?&k#) zC$AhBM=ePY#_}SBmPpG4d0yN(_zWb{6}*CLMuu~)Li_vEJ+uW4s2wvk9Ktux&(A|p z&n73;T{``YuwYNe_U;X>*A=RLM*n3qEaAFA|A$0)pn$`oV3zy5HI4&zw0+paaDv!n zS~2vN|MHmHZSqq=$>6YncAdDt7gm13(Ml@RI zI;&L^twqwubMiPi@niU0o4KX(xN@d7Xi8d4r8QbRX0IMV3NSCJ?y7du*M3HQHzfJ~#S}JYmuvtz9!(8<-eb(`Zqhy zvRmKq=0i$azii}0hcha+KVeeJkK}Wk;^0LE%9qX;Q(BosUz@OtBO~`qi+-XGxofQt zDOY&wHP21&sQ(zx`xy~g=Sam*+j)OFB=~fHraVf-LhJ6k<+9>;wKo2NgYSdgv{AhK z$$Gzq4dtZ`)!{EiSsCH4Hw%L&0cX+2qe2&r(zRkI2GvC0OWWjQBI{Iv#av|RDy&DR zVu3jA;mB6O-A9artGulXFWzwI9*o+jZ0_+mLLzOf8)q#m z)$Zt+%4e^<@sRH*|GzRypQ^=I7_zCr(~DAAae`u|8Pj}tpT-4-mndPwDb<0ROn)g7 zR?{9l^x)VA1jOqhqes*28egC+g4L-Ya8MRqYd9;M(*K*Vm}1FOv_(f1o#++n`Qo=Vk+V0`)db=X?7I^br= zC?jzMG3e4A7a#vAkBnI+e`D_tV!G{UZCvn>1RL0w69Gr@sqj!x4+fDcwzTK zw6(Pjo~j=_@4kE&%}z_Rif@cCWM>=PIA|Zb1n*tF17y=Ras{>LLH$gIhvq@puW-AC z1{@&M;rTz^t$^B(T!zgt78nJL)f z2(yJA8{RjL?H4J#Ock9=kRdh(I{`UG#X)=L58I$*oZ>cU25Tet)HhX=AoDn zafq4?P3{5KV z+zjU6S^1iey4{dP|0enFaWHSL%Q?$ocSJ|cWg5C0u&!qvw@YK#=wM*bSvxvkZ#!)6 zXQD5oOJkXVZ_UP9Q}wqSaRse+hj_%hFKDfUKP-m5WdIxDEwRRPD!;#->4d(`^)GXlW$OaEHzNh-nJf3Gcw*;?#oD!O#~HR2T%&w_(l}Z%TmCBtC&Iov1ToTf zROi{Fih_ka_LvQLryaA73CMFg)(+U06hO>Aa7)eRH0klb`!;ubGRRW7T9g;h4TF5D z%5?0Q(~rRGtlW2d1f+F18nq`vZv-C9Jk`L0`2ocCt5+*AiY`?(#W!)C=Bwi^|N4$y z?`(@(>8{i^t%zolpXdFv=Av#RX4U>EK$}2!4V3@9 zhQ%-KwE*_T#v1l})}Ne_CriJP*F;hqm)tWk%7M~Z#UKU=2CH)WQG7kZ5W*cJPI-S> z8SpTqq&k?MM#_Jab-A7Cn6Z#4;7H95MA#M&&K-DqNWlFD&2zHESZ#xQ?PNzh`$-&a z$31ix#I+eC$0O^109VVwAz?Jxw`0TJDS8rljrJy;$1)>v80>XZnBzU@+xZX8UA$5p z&};umtEY#nOt4ob_OJ&~#~C7~$ujvHX@6UvxdXCS#*T|M8C?C*F!V1WVmi_K`MIlF zx4B!BKT#@TdaaVvG4%>}ZKD*Y<6r*1*?y;p6ErXOW z>a7V?iKYD=d%#73L3h4QsE7<>A8zyJU;{;0i8!&jpn!O;i{QiCg}ca6e85Tgp>vSj zUBl$L;Aq-B_?`YGBbY92Es-rm9D5VH9DC&Q!QGB*Q{;PFnI9)zj2P=H=eF2(gP@=cwep*J znkFvoTIONNkp;6vbDuD(%F#x51d<(_$fMJ;{Zk(^*rmKt*IV2=ON8x*Sp8b3onPA)9_EW$2L4pSha#~O5p*#>`BFOL^N_laX%1$f{$X)I;H zz1CQ;ll;&daNSE&<+LX4BHqhTTFN>;N%MVpVsUhJhA=DoTj~{1(H+>$)%1rN-T8}9 z_bz~Kk?vf`l>C0A@(5QuHv1bpS`lM4I4ETaj)i_|pgO($N0LLTFZ3AU*n1|<7aFp1 z<#BWCNm&^&<*1o>5R_dw=U@tRheGu$71W-KbcM`fs(_{Hlst?2FCp89?N^2b(q!K# zY?`gG?6Axd&Vww*R*hXdN^S7kFOP?$AP4xJ1w12yz1U5a0i8lvQ&b*{Cd_*vd#Pu& zsQYoFhl3?N%X$qcBgFILOm>}2WK+sfDS4rB?UiMOwWq&V?O?#S9f3cZYPZ_3&YQj43f)=J}1t_qkY z(T_42^KZ8UTiH>dR>qW*WXT2}5aXTkNOO%njp%hFb7L*W* zLZ%J9hdjABZsaXeYE_{v!losMHbCz&s1u+Sv|rBKGQ8%IA+QWIMyQLhR_VS3WS_}! z2H*C1ca>3Et!>p+dg@u23aUnfpHm_QS~-;qoNL?vER9OUJq;|lhisX(`mG?YoYm#~ z+GMes^1v3aO~)Mb^~2h~TI{Aa3G6MLwX4XTT4wm}mSA8aF^8dDN9yHWcaDYTUoQi| z^)DbpKkD6Gx~{j%0b74K;HI@jM8FqHgR(Qch7>_#$-ox;rU;v^U`j^l&8g7QA@C7S2?-?3{T-P<88*I7@%tFw^4O*v5S^>PmDBx-B40~PqNa~tKmqwAB6c6lnw zBfLTebM&1_`!?#!1IRRnW;~fip=z!JDgn2rONJnIuar0TK$iW6(#U zT<}EsNegKg8PrSSx}F+6;kk6Kww_=BjL?4loU^xQkPDw5WJQi`Cvs;$Hvk8Y3kKN$ zF_kg85Hlw=yfO|Ob5E7Kxr{j~r}%74j4u+7Z-iHLle8LmEWP{DjX1q6 zMh6;KDGp`@QN=S4u;uYF4{TPTicaB3IG~}S(klESOLG4}w5jWz_9_WCj^q0cdCP9* zMM7PX@m_jA8RehV0w>_3QmqxVE+3IuC_o^0-`RCn9iYb(86Slhlk7_?QYT+a&I5Kd zmJFN|+#X$lK2!(8Z>)r@J`8QepgTheM&^5Tl>A5;smVPO*3MvUVz=EOpKF-iNH5h32#AxQJ1g>i)#lm;k$75p#<>VZ-! zzqc5|Lu-jdXM*l3^J73G9^q{Kw}v^&kFn&{WNcS!hTrVfhnJAJ2xquCSFl9Ac8%;=Yv5hfHjPS$Fv zc7{>!v*Sp8md3ti2>)Kjjin0rBaLZ<_fHWxc3Zz238gP`e=~CB?~SE8+)N{Kd5H zA1ZEsGoz>HKo)^Cf?X~xLL)*q+VVKMuKi(MCSg0hvcUfTQXPuZ6=}NxqEOxT+KA`LR z(Gny$8}y|{i6Fj}0k^mE?Q)H(pg85Mq4UuV-fF%{;cDT*P!*GH-W4SfFgTvCCbiwK zvv{2%hwib5e!@t|NH3{VB0t1;P;dB9?TP*LBVBnS{igYCeE|pS<3980ArYV#BoUht zr}!i_t0^S*8DVd26_0OPKEqemdlE#3s`)B0lXKi>l0oYMHa1O~3L&##-yw_uXyo#Q z6Lg@I)2)j)RHUbU=a8+d<(1*^V?}BGxyoUP!^u*T-j|sB8YIA}$G66t4B4+|E=5|g zW9)CkF`um)3OnT<_1Bulj6v_IGQ=lc;KBZS8=j{cj7%qGf4|SXUU+S(W8$4yo4AE3 z_1Txz`)xSw_*h+McbRFzMbAv!^&fP=mLs>IR~+vD-8uuSB4&jR*fX^4;^(j?QiN zbnfh!PHl6)KtWrJtl%R3Ms6=AboemzakoOQqjb>w6h}Ige)iVq1oHkvKqrx-AXWpn2=te z+v_Kl=46d2TJ9H-xHvKjOT3Qdj$;L3o&mpG1s(pJqzRv!9W6IJdw~~Bv@@qefxb`s z`fV2m)FonF-_Jc+{>XJ(i}j}Hj8-L26BCh#LiRS+?Y(BxHLw$izA%BtqrkUs>G=ku z))fV$%ifNCWDz`T=55V?^`T))PqjI@31TYh&-EJflgiUW|0`={4F^2;!@(DnYv|Zf z(o)3~zliwi>E^uGysIMkE-7ji_v;K_%Io|P||=IF@oO9jJC zi?ya#Jv+1)QN|hYMcSIi%>}x>42+EE05)J@%L*9&{v?S37q{%JhH9~sp^TJEB8W*N zwEK*Vu)$cQZ^^CBi(P~~#yh3;I*jbhD}|I>PN?AZF7?;)LYzLzDHk&2f%KpA#^~AM z=X90B%-so@5U10>jEjOyh`*D%mp?=+d^qtG9hhyW7^y-Jm6ypI^scrwx9}{BsD+CE zOi#44r<;xeTRH_^Tn+u9G&VeLNRFpWbb&zNTQ{VPpaVRbw;>@RNhv7>fr~Hvi4sSB z{L!WznBL_&I1ASJlzD;l+t_IIJcu%wPsT#j#@^B)<(7}gI2e-I2xiM7YroGir5bWv z+D}e*O4b0;iBoRey#a~CgY@^#v+8{~{Uu5L1ALLje-eF>)qO>C&BMb}?nD{%_b(~W z8Fu44FJ5p3xQ-|D0*?j{X@Q$cPuN>BK0aQ~*SGDt_d)`H3qU9X!^0Yq4xrAPK~;ZW z7EMRie*E1NQuDA416g+8DqiGFcfvHMtZ$^&D03gnmehiKenvlpgPGZiySVotBdfIM zig9vI3W*q#PkJ3#du2ng(D2pkAf{`FY%IKhplHY|Q_DiMvu!t1{Ff1g%cLx|^}o|nivuBh}TjY2|V za`-kP6EFY`3h@RiA3sLk*hb^i5X94IqFcPSV7DN;33e9aw6-Egy;9*7>!+ys@W}#Y z2Z|k1SzH0tWl=h7cKdiG7)zcQ>ymI7^%eCOtuXH__VtHoCsS>2%Iylq?){5jjKyVC z`)@(gVYZwEyNwjtz_xmsI6UdMl;s0 zd@%vOo{Ki{v^{aZuj=)dE^*4U^c95W!$3Xwq4^4EsCq#S@-y0f{bBTr!c^<%{BH`r zCrXBfhtV<%6BhseL~3kcbkxVlBY&D$$B3;QBbVj|-(LLeTV+SGY*0tFR~s|N!89Iv z=8;NFr>0?sUmfKt2jUgP4yyot(@62Y;ve-{=?saG@JC>AFAI!Ny8wtEy1mJe^9nC8 z32_BG^jQ42AV}!dO4+EKg*!%xkWR^)E=9O=^;(**%$MR@epv9Z*KA8Gx<;}@gmEEJ zkt$_II4z5DKnFyiAU?A12o5K%XCniR^L9vi0~AYSRMg4gfoJqk%yF*JXkOV*!W}BN zARYe}A#ExD=M-6ES$na*6qI*M6~`l&9d+L4dW7#YK`WLt@;>A|qjJK3tBjgsMK`=? zfRhJhA!YHi*-q+I7ja!RoaT-R{8hvv4CEL@{8r7n5DI`qqv|2Rp4|dB!#!e4( z|8p6fwdczLN?I!y8jW7iRb2nV$#6L3H+9dCx@-KZ^X^2`Zk3A~FXWwQTi(ih(yX=>{lE%n;6Aq0Pr309w?^W~N2LPn6(MQpkEIB@9$krJNKvKj&r7e$fm6tkoNGv<@l@q$G{J|AOghH?;)?0s`*`mA7)j_>+>7 z7X986Ii@@~8ouhJ{)3G2CSGT^NPzEWp9Me$o@c`X!+LaVVUFpBfT!gN5Dtl{sXI#x zRW&u-_khjb543gvR2p@*H$)Ma7gbLVpKy!YIExYK- zDJ$2{&AOlbwS0=`e7Z6zTd1G*sQ_-bd^OZfb26f$vNE~UYxGc5G!yxbQitZhx3551 zjHTz=Fufo37)!HLQ&R&j8X$JozOtUFL~NGtttBN%#1zr>WXblQr8ibVB6Gb^%oAaetv+Uz!vE$5FhAtqQl66R9`mkvL7ETpW9=rvIUFP%Wf8E=Q zit0@ZOS202La_z7B4h!A54f&Td(m3NZLxZmOR=T7?U*E@?zkhjojm0AV#jOkvuVxU zZB7!2+Bl;_2G)l*RyCgp37w$ z-agMYBDjI=q8Hcf09eKVPe#gbm)5*LkVMaAJI%^-+aaDh6b5+kcz|hF+*%IwY>oy- z9=!s`fhK8a3`fVTJ{rqG+p?ZRGm2iL1K*Cs-F|Hvr-}5^me>BjsnTIDPM;q2m@FyCSaDtwww5p0J@8|RG)Qt{GrWGidNwK{Q2r92uU_4r7iVimci)7O76J5K-w z?yB!}9u_yxfSnc@sxWfX;V8NNPXGG_LchJtt-_+;rP89uV+iS#i~0c~2xREnRwNbp zo1YcIyG#|=t@g39)z^i4dTpFZkCjnS@Q=DnGwHyiVRxuDtcoQVvC)fY(}uu5TXS{U z2&K7f9CKjWQL%{!U8kQ~+oM#uT(^GER}`?Gl@$t}(F^QL+j1`@9U+UNt*gyv00lra z%`YP&A~GlZ?k@R|ajyihP4XvqcnBwVf zVQ&2Wz3={Vduv5!3xIn9EGgFt@``8+05S4%yw={`-U1Ze@RbE%U%551jpL(e{}AlD zO*mtHy3D6=9;%ZtmP`#=c9E6whwkXRTRS5KpO%n+)11`SDRF0EuN1%5{A5q1hofa) z??#2VO3|;RJshbMAzO$u&~YrnFNRB^`ZsJSVXt-6eR-DpS=s!huD%m1;998rQ5=bZ z1I_o3YBDoDLQN1dcPi0|ea%>P8C6p5wmaS*+Lz3tmEkn|dDh$M@n#1Mv|`=ZFj5WN z$@m=>BcDCJF0aW?^65@?l`=f}K+o~d3r!V^<&UK(9HW@LrNaP~$R9Kn5tQ7xrKb=+ z1=jR!bw99SkZq(9G3`SqXz()VsLcLaz(WgzLNg}%?)u`42o^rc2*-p*zHp-ZqNjRC z&w!IB@m7jnL`|oX3X44^r8iBJn2oX;e2i80H_20IOPDOSAMFep zW^c7OOXPlCc-1Ed34bPPoqpsO2y+6=+im)?lg`fex$CI>bOLy>Cjb_k`SKdb z?5=5HGUOrUq|f=)K>Ny`vTR`RvODFC_(9zS{OWJRf7v5~DRp;{%iS(kMU6Bs z#Xxp}(OhdShRuQyQrSXXg`EUPLJyT)C3;|KkJ_DPugRCRH1rYe?q$OUgJ*8J;HrV| znz^$>(a8^Jix$WR!{+orzPPSlM*Xu|UYGUqReU|-PL;*8FgVeJ|3#@$x|C0rb6#89 zqECgw-4uN_T7}S>@V);$4Obb=iRZ7XOg>kdaav5%G3SgHPjdkobD6Qe?qSdZrZ`ox zw-Vq;A94KQN97>v3$#NSgEXRznb6&3=cdI(aLS0ZA>U?1BW4PGvR+z#M7@*My$Vqq zY-SewoBh(Xy~O7BpgxpZ%sZ{aWQ4Xm$o`Ug)u7eAUV-#Eb~MKUR8dCyoBhVMJ+i~; z@=WV`E2{!*RKo_bD#r}iLiI0tdiq}PB=7@@o+qnMh8JsP;~UKru%{)~dj$n_fO#cM z{I&EUM!8oAJPN~jry`?n^$jT627a7e(77N5Lz&B2yZ^z7v{N+}G6rHG5G0j(4K=WN z&bB%TFGOX|G>H-%X3FyBh^azGx~W&)lcMMM`Ar@k-xuLS!+mz&&P^#sNFm zkAm#HlEIK1O-WoF>jlRLfXN11cpYC*xoo3{!%sF~^Z+0KLfy#9!q_2-bD$yq8B%Do z^88m^sHDI!;?O;j6_Cz`Q)VU*`XlwB7rSRv>`H&hjwFVJgTM3zq>HMz6z?wL8M3}Z z*JJjbGFQQXE!*DlI^vkHpqB1s>BehES>SP_Jvtj&Y1aLE;M~B*UF%+;Iag4Sx3Md^ z`UZUKjouPJ+cKiolnrPjJ1ox8D2QX`X>`A|J$@ecFaFYVi3u5!L6jI0jGUCt3Wp2j>r4-H;aN5t;5$iB+J|jwvLsHHF z+fdMKk4tJs`tXd46qtzo5glsl6phy~Sy>ff7w%ad9-7zV@uVgZ+MTN{w2m{DO}`UN z;C?5_GKJQw8NEAEm^tK(m;@jj7|%Ff@#|+KBko-!QyHZM-__A<^^QDY%3*OSD0A@N(eGNK za;|axZ&}>)iU!QvCr@(56c9D5#BT;1G$kvb_w_nO#uF-Hy?1X+KH3m==r*EpCGmM$ zd1KzbruNiC#h;Vtu-a&w_Felm=s!{2n9K4RLPyzgH?Hj)G(s{yHa1qypU@J?OO;0j zJE{^(Qp<(dOh8vji?f1X+07b6W>k5)M&UM`4vrp8@|($b^WWPpOPn082Y@E=Lzv#N z_t?K(4*B~>#_qQVedC`v@%MC*ap=MH*T?Y9DZ=cUU|-)(1y@ywuIkUJ0Y;}EK6Zj2 zddlgTG`|2=uQ$m>HSsYquCxPs=I)`W^!)7D#lMsOj`5K{jBosD^E%WCR0PzN*ldJZzkChWk-6i0eXciuiu0^3^)Y}uqr-Bp2x?j z_Uo%)SeuUxQPb(kShR<-V1Xf%{CbVE7CjgD@2tZCVRK1bER3%F$0Vn4RMWE@)22mg z>_g&^m#~iRx}r(Ufhj)HG~&3Zka4F&u6C!jPhwntGFn-pf<&+@RVY_Gfs@^eOqL)g z*+h83TGrT-(^%+onzxxp>~3}>`kwdM5c8+uAGsIjd*mUEmYkYL%+juAv;{cv(ry0@ z-nb&P#&92x|Fk^bpYd2vmQ`K-)jyi?FvUmBI@j@%!%vSQenL%dV}`&H`iGq!z8hbm zhH91}OpD~qMaRhMYU1N>PbBn)?q>ywkmz`jzu1QJ{VHKCrlHL@0t4{7?q9_-hfLi+ z%Wtl|{cy$gvcvFFEmu2P+`*SEVriX1BtzVo)ujVrMSp0|Bk_1@wPTf4yZ%1DFX-8C zjcaOhI)5L_+YZyYF(+LT!FFEDzSLs}(5`H33TN9?L*$yXnKcxamz~w;M663U$ke%K z2$&xl=(!qbMrizqRLDIugerK@TQW=K>v*yZf@|5FKB3*QqC|k`LctqCe>+M-k~kswXf9gizQCiBt3P>t3;^ zQka`-FxCV+82)hcYc`+F(1~T6hOuj+Z-eNFa%@~BRZK?;lz9$5V+4B3_a1Ngy5UKVr8p%*I$5Uiy>#1YZi>?M21~?PSOzE}WDo1dJ-D z6SNQ#mgzFQIRm?VFL^b~Y4@9@UtZ#%2W`c%_ug%;0J6A%+0acF=NO=%8YR<26d15* z7&z*`806tU8x*E@Rz~BG1HFiSXBvu()wucNdlBVMe3VqDOvKIB%jd}~#+LreuICmF zkDP&{$`RZ!yi%*fPW&iU7&SGa6?KhnZQIpyYG-_N?;Iw?`)LZ{o@@!RJSDm^O zWS^#d)MK9sx`{s~2v#8c`EJS(n3Drhse1cz=<}S&WpSU{98rJwnE%w|K3@L!>+EQ2 z;o0BwD0>;Lw+<3~7@xSh2L@PsJTw)y?TFRT3hCJ1)KnOG9#nwT)iuB$XGjGp@7R6x zjS?oUCd8{3F}sPDcQ>+r(2+_oPTk5zb$wST=Zf()u35qJ$7=XOOGNUu}L> zT?c5+ZrjTC>7xGFCWSu$>lL)}c-d&de(|u@w*Op$Is@$ITI*5A888pvQg09Q*-id~ zHbd=$r9$eC@?`^c^?t0gFTX>rZHGT_=93y2qUEB=3XnxV^S%CkXI%7!V1mjM2NUmy zZ^bMOj*F4ip3TXh;GU1#>hWj zd=}1csWGGv(HFjCK3ewU zdUi;~9&~&{XgSWfWx3vrm*_rqJLm|-*6!*LJ z9JQtvZ)DZ3>SEQ90HDiS(9RFvB@mUY3o+(zHdg31T>6FlYg!@UzQOm(ETib{2Z*%y0iLA#| zc=GzAkky>bI$zfsM;;NW>p}=s$TPy($9`BaL=%*K_tk%T?Hc85x|H+E_xK$ucWn%p zgA?JQ7!MKHn~^dm@@e(!DFEdG)@acG5m}*24{QOQ5?)?j1qXB@A`AMK9{@Vd*Q6ep zisWelkO3F18Dn`VBL8`f_%G3ZBp_RyvU3P`P*tKN&@{%l_ky2Fl#q#>tRE;?1>oS(G9tFL(#gRZ&$--SjjIUnZpELgLs|d7ijBW{2~fb$u`03O*P<41$nv6I z_uKKme1nF9Vlw!RY1<#0T6A_%b(v;)88AbwIv8>SfMP!VOg4i)?@t^_Cr8&^3>Lg~ znTbi|-a+vv+3^bLy`P;TN_tRLRnZF_ujvhGb5%7lx{gj6mcLE53H9 z=KWfzags{f?$Q@fFg^jb*=2Rpz{_QP*Lkl*yw{8Pk>EQS;}#;bx7g8rt2t|=8akDpi%x=~PT@`E!{8;fJ=d!GlR7uUqFPM+AIS=_C2}j@Xk(9P;<4hRc{R%c;%zC1f|MMz^r>~=`;Y?6y&%8>4qvbvbu4C4a~ex7edP_=0n8aI zq;0?l>kwvns$WW%fBG{4EmIe0(Joum~uOQu0*Sb4flZgxr`P0WjqIva(!M`+}rIP zlR0-1YEcoKPH}PkRz=Zkh&=|X2Zk)fY(tkigBy=~0+?`lQ&QG>()5>6M%#;tpBETS z(EJmJDXr-G@=G4lFf5!uCzYJa&JVkWoxlzWVpc>HU;c-ZaI)GvX~9i$JH*TuH!ac? z^{0;=dbm!f6MgHHgM7wt)Tbq?dObWextRqyH;-wm(YVwjG8$9=B;j#9qx?Kmw*(C( zR9=+L0^8q#`WhNx&*r(!9zc#p0giKq%`4eMla9WSQR*SM|a)2X9CxxduiE`l41f)hx%Al!`5r!>e&@p@`PS&Cy#4PnI ze~26V7N~>`!~V@RhVKOUvvB-C^gZ|VbWnMMUAo|(Z4ve!GGk40_ZKs$U?}~gy&K^z z3qTQCfRD{IlGhSVk`J)3E6V0?rI~W8bncDsLMwF_CyCRWngp>a_~Gim$TB?kXHA#~ zvG3Xp!)FTy2M4*QtTdB^6cUZ_LO59l_hYtW5D&pvXlMjAgTrUnZq-EZ@|k?sJP!^b z@U|+JTLzJn!i3r?2h}$XTN)_Q$HFV^2lExd(wZ_|9t9paKT)pT zTyXdI;o}EjwlOWipv9HzfMmFM{nS9op-lShv6lOAVOHQF#^@f2UOv7Lp3Za9*!h6- z{{8!crS;1NM_5l77T#OTmcgQyQq?OPX4a)CFL~+H4+I+rZ*iDiOGW?^yeA z!#p#3rq_lGS)iANWy&tk*jCPk^Y~Q!Xs#AkXLNi^+SdH5pdh1!FGO78)OM@RopKQd zzLEYeWc2kq_jb+G^=iJM`mnm&;-g{e^~={XFV;%(HsJo&6#haAsxrd-mqn1o9Ph-; z!e8ErI&}XHNvt*+J>AYU0Z1U1+duUzrydN!e?@ul(dcAzz>_!~_%o37ndk80dAjoqmbgU|Izk2^=}X zAW<;stkGr5=Ms!tAUHd{!hoGGX_8PsFp#u=$!fZ3lF65&nm%3Y_tv+)$M=f``j#}v&YGxUXt{!iHt7$8lh*&n7{Q1^w?4 z)~>6Cn*V;??_z_!q1}4fTg_|K3JmEY(?0*az~NGBCK^3HzKEuPU}ChC>=~^9Rc8WQ z`^|K=^<&E^EGT~ zd0kD1Zxoh0iMH7?R^*aeYHlU>s!E^$yO5-*{`pg!@?7%Y^#7-uGyjL`{r`WKD7&m> zvW&7#4f7HqWQ}ZD5<+CnE>o7ql8~)zWy#XuC9-DUWs4b_(5Mhu#=ee;vg>mlec#_d z;B)(Ye)-HV=ZBdyGv{2_xvuB(c-&XoxNslrYq?(9*xYvLphz?R^mf>iLcnms7c?qL zS_W>YL4S&Cwm}gu{!37)J#>&>%kE~pl%n)yp8ntr=OF&LWswiBAgG59f)I>Od_(oY4peu|LN672rXgsb7OV~6E=kBvj#I?#~GRYVAev^H>(gR$|F zC65Dd_Dk(gXI`?BR%QteZoX}e$p?xT;IP(_P?FJ(X6VKm=(iVK-t1_VTzK*k*PR<; zJe?YnQVS{hpijY6sd!(KKFqwIN*}GKF#^xO3KQBBbdFg3r7dg3bNofrEnV#8t7Itb%Tn!nyPB(NdncG zS#i?Vws#i}a?RxB=ff{tAYkGyX*xQdkBNzye4xP2#Wni_<9+`8d3;X~D&pDpwm%S8 zxkzrC_w2vS$cU}3zO<1xV!pqJL?VHgC8o#bu;>S!gTP_76>{iKvO|Pp;_o*X1+J;7 ztA7d$^7bzGi@%giU8y~v_joLDRT)xcFB__GFJ^KkWb}u$`aqYMNcZ=C{_k3;w=2@q z&ybw~z~ji12#4Dp9fNZk8_xjGD`?t|Iis6?6y?WiXB}InQjI?WhX+U9ttc;NGbrFN zGBN_4STkeeCz+Wr4J|EUW##dvx*(E?m8Rx#-&9juXRZWGjwm%+OG~z*(hcF#5c|n% zdMYxFw2ub5*u(MVhH|jD?+CWV$J-biL&L{oV^(=@-heUe!1T3K$W8V2yyD{GCzv)T z6pEl)3wCLRD!|$EBxJkx2ns^G!V_JNPt)8z$4|`@$6WXrH;bA`-OR&(m)2BZJ3cdK z5*q=)9XcZsvrdea%B*O!4im%hF^^Zn4R#!|Nb|TFo8Wlnjg-54#9zZzcSU>soQ3fW z^s}?cZy#XT*bFgBKzj$IeHwWt5@qX?Rb2AJl_DI@^qkXmiH>RP%iYnGP~K$Dr>GzH zs8TOj@K?!GAxj)jMw^bQ7b7UWW|r&S9a#f=#qdK&C#j(zikL)>CPj{7EuWY;^>wL} zOV#gOLp?Gzn@%ei)MH}Qu&Luy-9ZK(*oz>80#W&Yez_5L18IJ9reNLy>1^U7@K^S4 zI+ImjH;FK^zTe}xRr2ZQnoIKw(ar9&&desPl@GL+R6UiCU?3(8<%e#1!uP$s*^){H z6@9fC@&z*DY_h%y6fgs^pQ%#<_T%h7k|H;6oT3N-PC(Rlp@EI+ovaA@iP4bb@<3k3 z5vSJ`pyj7-ZpS2RndpC6Eg|0aFkZ{%$gtj6Ex4swnf8ebIu7KJO0gNMiwC@ zS$4A1QU>MU!x1ZBEdz{9j2XT>{_n19C%oSv@ZDu6 zPNYq&{zi&jR|t=74IFRm08~J21Pm28GJP%}h<7DdX0Cicbpqj`rW?)?iVYCuacQCt(0RtbfR!x}RKN5M_Or6ND-!@#u@nF=|q z9UW)KTRxd}604t*qaL^(o#Dc)8{a8x_PkSTEAv@oS=Qs?0#=-)$X4gVU28Tzo;9VV zfw%M)c^gNr3WHDVu&Cg|&vN92*Yz7KIsRW52!H&#dT0-r0YD%z5994;f&GxU>HcJ>pxcAflMM>=bLS&e(8G z6LqW^*R&1RaoP%T0eiS9yYKh#WYI6R!a)eI!Y)r+Bb(p?$n#|dfu0T}P z{zEWb_Fjx6V@(rZK$lUMG8H!@?LEbK`rE^o0r81_8qhxkMQo<>ysd*>Lt zG}pd+2S)KG1yw0JIGIk8vCh+7WJbU!fKiv2q4l@FvH_=WSy2%~L_~zQLRM5v%+EEY zdO&%YnVAt(P&fl<9pf!$Ew4B5jiPlDESt&4O0*HME$tW;wYYbzT0DDU(W!@wSmFu<~e3t%SutB<3fC6ZuEEfk~EWZ+wou55= z$(7XyU71*6Pv2{eB(q=iGE3yMDM7TyF+H_kbJ^LQOL|) z8$RV|oNxVY@pF=OPbisROZn~~J{DKpL%(vWU7&>?IDjcAkDNJ!I5$5#ds`?VgDc5W z)^;vApq#*f-w7}sSEA=W!yjHHDq`D*-M!nF1v+Hg8sK`n(>K>Lbpo5-{F0a=%KpD^ z8bO>J`eH65%X(!PYjtZ$DR;-0yRiaUoa?(Aed|N9WL<}xSFB0TbsSidz^8m-yF0AWqJ77ZE^W0RSFGzjfwK@3dt0rx(r)q(0zY`I zi}dvRH*#Ha(&2%%@|^<-9A>Tkmkw*Ehv@mKwAYc!{l2;bf-Q5`?b}KFY9t{M4vdYgZJzFk$G5>ld-i2gK{!fLoS;#l|xt>yLr%?XNqzt6V%h zujC^}X^W%jqnY-K<1H8mezDS4Ih@I`B0*v0n zy5D^(9=6yM`-?BnF_DS&JF#ZYrjr6TvvyN4aPsRG&93mt%4kZxQ(&gwjZx#{4Li_y z?cc;O;X&(99IBTAb6o|JOzk!lP3+?_ zi(wv?oD>F}vSg4=6xcXdm1^Flu;8Pw8|**wp+>?^ul^XU1wq}7i`-xZM_#b1XPXGz zU)Fg2C?r@~P}7N$52IltjfKouR26bc^H0PG`IX3w zF1+IN9x52YU)&G`6~sNK`7c?x)n={QmCLXyowG|PRP^4$Z)xRYUxU@zmy3&QxY~sS zz-LgvYA*+4ArYUxKy<;2KC{=RN9CclUFyrjtO$?E@fKnv52G-KNf}XlLN!=R|tco`;Je(Hz6SZ8{#8Nl@ zsQBbJV5zR$8Vf?fHhl99vSL6zg9JoJvQuew{jYc-DV ztGw=mwiu0gekB5`8AiWX@y-X7PUf}&q$jt!`U?y#rZnvPzU`AN)2)(`{-6)+TG?jr z?%;b#%}vMgA3sXBr(*TLJgl`6u$CnXmLk{O}!9Pw~&yKh?sQRuD3wH+gq714<4rS_v?gSq`J%~ z+&mEwhome`D2>L(#)ND&7Ac2O`Z_Metx4eNM)b+g!yqmWR8%5pg|PP2m^5(Jy2$;W zV&Zy1zdJ52=dC^G^O;Qe*p>H+PioPN?@Wufwz&qI$R;ChVeNSav-)?*BQzKf9RL>s zCZDuioJq-!np8o6eYpePF*#toIl%mVV1qn>6FG{kRCa}`rl#%yZh)So6h+Rn1XfT` zknvop8^S@H#6w7zspdl7grFy#2h%fpBj<$|$L2C{3gdNzup%#qa;9=WF zU}%b|4aLF1EJ z_Cv~P#b{yhT}VcRJ(i->>HVpK5*6#0{U(1~#%Od`v<1T(jCQA{s~oaWP`tV=z1F})VgTt!Kw)1V=`X&_@>pRX;rJ*F~{0`w<*P~%8 zBb(lY)2K$jhonA}7o+-eDJeNn9UzMz^TNVz<$pOMH9K;Gnl!$fw(%N8SSS-G>_wYq=Q{^Ti6^c8`5>^!HTd zmvz4=9k$Aw=2ldCqhx?E_^gl6uZZ}-9(6YezW$MNKW#XJ=J30~|Bv=TtT|$NEr+ja zD3yde>p1KY^xABtqT@bulD*o1aC@k5Q>2~RJMTPAdW_^}Wf0B%O%lLqM_K(1CGeDh zm+ar+cW`i+1XHdXs7<{j%mMrJFXuBV>NymzzT~E`S1f4k>E^=rYkFI!^a; zf4U4K0F=ohi1*h>f42S{Vl6~&Yvk431MW2dr1T2g9JZqXI_9foMTn7AIRNkpEOUKX zZgUrvr@}e(5`qC_bD+k}%qo!575Si5b3cM^QRZ*HhD3!|87um(m_OsY+_$28;|4BU zHH`4W|JJRP5m7ji2(8O`))E&l%9yU<@!BS{BEb$3uS$6!Y790?TlXl7@QwCjj0_J? z0|nIzYb&eF5m3PjgRZGs7a{_N_yER@dG)=-5gh-_&c?j3loT6(JdSAv0}yWI4n3zf zb`G{Cyp-ovt_Ht%8$zPxL4I*G16HZ&$C>#RdA(W>YF~BPfrnTQ=!owF_N3h$;gu72 z>}hGIFU$k>%Mnr9Za{bl1ys8Fu@4N=yK;9R7#>tL`cL)f|Jw10M#lq^TVv2d0bl~T zECUTq&44X_D}WLOt31IGVBcCWnt4j%Vqz+uo=75*2x96fP}BNbvvPHnpr)p7#vH^$ zV}*4!Q0eRKRRbL`V3?`a0*x;QynY^d5J?5;XA-QDJiQ-3wiIZ9E~p#~yp#Q(cQo@h zjDcel3TOW_*Z-%E!7(54yBo%paAJ8|n@VelJE)g|a2B9?K(2EFXSTFK4#69H718Dz1PEXxJej z9<8`fy;u?~J%yYb!U`=!7k P9`toh1C25@+b918W=sIR literal 0 HcmV?d00001 diff --git a/examples/synapse/copy_blobs/pipeline/copy_blobs.json b/examples/synapse/copy_blobs/pipeline/copy_blobs.json new file mode 100644 index 00000000..3fcaaf00 --- /dev/null +++ b/examples/synapse/copy_blobs/pipeline/copy_blobs.json @@ -0,0 +1,172 @@ +{ + "name": "copy_blobs", + "properties": { + "activities": [ + { + "name": "List Folders", + "type": "WebActivity", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "method": "GET", + "headers": { + "x-ms-version": "2023-01-03" + }, + "url": { + "value": "@concat('https://',pipeline().parameters.SourceStorageAccountName,'.blob.core.windows.net/',pipeline().parameters.SourceContainerName,'?restype=container&comp=list&prefix=',pipeline().parameters.SourceFolderPrefix,'&delimiter=$SourceBlobDelimiter')", + "type": "Expression" + }, + "connectVia": { + "referenceName": "AutoResolveIntegrationRuntime", + "type": "IntegrationRuntimeReference" + }, + "authentication": { + "type": "MSI", + "resource": "https://storage.azure.com" + } + } + }, + { + "name": "For Each SourceFolder", + "type": "ForEach", + "dependsOn": [ + { + "activity": "List Folders", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@xpath(xml(activity('List Folders').output.Response),'/EnumerationResults/Blobs/BlobPrefix/Name/text()')", + "type": "Expression" + }, + "activities": [ + { + "name": "Copy files to Destination", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "ParquetSource", + "storeSettings": { + "type": "AzureBlobStorageReadSettings", + "recursive": true, + "wildcardFolderPath": { + "value": "@item()", + "type": "Expression" + }, + "wildcardFileName": "*.parquet" + }, + "formatSettings": { + "type": "ParquetReadSettings" + } + }, + "sink": { + "type": "ParquetSink", + "storeSettings": { + "type": "AzureBlobStorageWriteSettings", + "copyBehavior": "FlattenHierarchy" + }, + "formatSettings": { + "type": "ParquetWriteSettings" + } + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "Binary", + "type": "DatasetReference", + "parameters": { + "ServiceURI": { + "value": "@concat('https://',pipeline().parameters.SourceStorageAccountName,'.blob.core.windows.net')", + "type": "Expression" + }, + "ContainerName": { + "value": "@pipeline().parameters.SourceContainerName", + "type": "Expression" + }, + "FolderName": { + "value": "@pipeline().parameters.SourceFolderPrefix", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "Binary", + "type": "DatasetReference", + "parameters": { + "ServiceURI": { + "value": "@concat('https://',pipeline().parameters.SinkStorageAccountName,'.blob.core.windows.net')", + "type": "Expression" + }, + "ContainerName": { + "value": "@pipeline().parameters.SinkContainerName", + "type": "Expression" + }, + "FolderName": { + "value": "@pipeline().parameters.SinkFolderName", + "type": "Expression" + } + } + } + ] + } + ] + } + } + ], + "parameters": { + "SourceContainerName": { + "type": "string" + }, + "SourceFolderPrefix": { + "type": "string" + }, + "SinkStorageAccountName": { + "type": "string" + }, + "SinkContainerName": { + "type": "string" + }, + "SinkFolderName": { + "type": "string" + }, + "SourceStorageAccountName": { + "type": "string" + } + }, + "folder": { + "name": "batch" + }, + "annotations": [] + } +} diff --git a/examples/synapse/copy_blobs/test_synapse_copy_blobs_functional.py b/examples/synapse/copy_blobs/test_synapse_copy_blobs_functional.py new file mode 100644 index 00000000..50911328 --- /dev/null +++ b/examples/synapse/copy_blobs/test_synapse_copy_blobs_functional.py @@ -0,0 +1,75 @@ +import pytest +from data_factory_testing_framework import TestFramework, TestFrameworkType +from data_factory_testing_framework.state import ( + DependencyCondition, + RunParameter, + RunParameterType, +) + + +def test_copy_blobs_pipeline(request: pytest.FixtureRequest) -> None: + # Arrange + test_framework = TestFramework( + framework_type=TestFrameworkType.DataFactory, root_folder_path=request.fspath.dirname + ) + pipeline = test_framework.get_pipeline_by_name("copy_blobs") + + # Act + activities = test_framework.evaluate_pipeline( + pipeline=pipeline, + parameters=[ + RunParameter(RunParameterType.Pipeline, "SourceStorageAccountName", "sourcestorageaccount"), + RunParameter(RunParameterType.Pipeline, "SourceContainerName", "sourcecontainer"), + RunParameter(RunParameterType.Pipeline, "SourceFolderPrefix", "sourcefolder"), + RunParameter(RunParameterType.Pipeline, "SinkStorageAccountName", "sinkstorageaccount"), + RunParameter(RunParameterType.Pipeline, "SinkContainerName", "sinkcontainer"), + RunParameter(RunParameterType.Pipeline, "SinkFolderName", "sinkfolder"), + ], + ) + + # Assert + list_folder_activity = next(activities) + assert list_folder_activity.name == "List Folders" + assert ( + list_folder_activity.type_properties["url"].result + == "https://sourcestorageaccount.blob.core.windows.net/sourcecontainer?restype=container&comp=list&prefix=sourcefolder&delimiter=$SourceBlobDelimiter" + ) + assert list_folder_activity.type_properties["method"] == "GET" + list_folder_activity.set_result( + result=DependencyCondition.SUCCEEDED, + output={ + "Response": """ + + testfolder + $SourceBlobDelimiter + + + testfolder_1/$SourceBlobDelimiter + + + testfolder_2/$SourceBlobDelimiter + + + + """ + }, + ) + + copy_activity = next(activities) + + assert copy_activity.name == "Copy files to Destination" + assert copy_activity.type == "Copy" + assert ( + copy_activity.type_properties["source"]["storeSettings"]["wildcardFolderPath"].result + == "testfolder_1/$SourceBlobDelimiter" + ) + + copy_activity = next(activities) + assert copy_activity.name == "Copy files to Destination" + assert copy_activity.type == "Copy" + assert ( + copy_activity.type_properties["source"]["storeSettings"]["wildcardFolderPath"].result + == "testfolder_2/$SourceBlobDelimiter" + ) + + pytest.raises(StopIteration, lambda: next(activities)) diff --git a/examples/synapse/copy_blobs/test_synapse_copy_blobs_unit.py b/examples/synapse/copy_blobs/test_synapse_copy_blobs_unit.py new file mode 100644 index 00000000..4870fd8b --- /dev/null +++ b/examples/synapse/copy_blobs/test_synapse_copy_blobs_unit.py @@ -0,0 +1,131 @@ +import pytest +from data_factory_testing_framework import TestFramework, TestFrameworkType +from data_factory_testing_framework.models import Pipeline +from data_factory_testing_framework.models.activities import Activity, ForEachActivity +from data_factory_testing_framework.state import ( + PipelineRunState, + RunParameter, + RunParameterType, +) + + +@pytest.fixture +def test_framework(request: pytest.FixtureRequest) -> TestFramework: + return TestFramework( + framework_type=TestFrameworkType.Synapse, + root_folder_path=request.fspath.dirname, + ) + + +@pytest.fixture +def pipeline(test_framework: TestFramework) -> Pipeline: + return test_framework.get_pipeline_by_name("copy_blobs") + + +def test_list_blobs(pipeline: Pipeline) -> None: + # Arrange + activity = pipeline.get_activity_by_name("List Folders") + state = PipelineRunState( + parameters=[ + RunParameter(RunParameterType.Pipeline, "SourceStorageAccountName", "sourcestorage"), + RunParameter( + RunParameterType.Pipeline, "SourceContainerName", "container-8b6b545b-c583-4a06-adf7-19ff41370aba" + ), + RunParameter(RunParameterType.Pipeline, "SourceFolderPrefix", "testfolder"), + ], + ) + + # Act + activity.evaluate(state) + + # Assert + assert activity.name == "List Folders" + assert ( + activity.type_properties["url"].result + == "https://sourcestorage.blob.core.windows.net/container-8b6b545b-c583-4a06-adf7-19ff41370aba?restype=container&comp=list&prefix=testfolder&delimiter=$SourceBlobDelimiter" + ) + assert activity.type_properties["method"] == "GET" + + +def test_for_each(pipeline: Pipeline) -> None: + # Arrange + activity = pipeline.get_activity_by_name("For Each SourceFolder") + state = PipelineRunState( + parameters=[ + RunParameter(RunParameterType.Pipeline, "SourceStorageAccountName", "sourcestorage"), + RunParameter( + RunParameterType.Pipeline, "SourceContainerName", "container-8b6b545b-c583-4a06-adf7-19ff41370aba" + ), + RunParameter(RunParameterType.Pipeline, "SourceFolderPrefix", "testfolder"), + ] + ) + state.add_activity_result( + activity_name="List Folders", + status="Succeeded", + output={ + "Response": """ + + testfolder + $SourceBlobDelimiter + + + testfolder_1/$SourceBlobDelimiter + + + testfolder_2/$SourceBlobDelimiter + + + + """ + }, + ) + + # Act + activity.evaluate(state) + + # Assert + assert activity.name == "For Each SourceFolder" + assert activity.type_properties["items"].result == [ + "testfolder_1/$SourceBlobDelimiter", + "testfolder_2/$SourceBlobDelimiter", + ] + assert len(activity.type_properties["activities"]) == 1 + assert activity.type_properties["activities"][0]["name"] == "Copy files to Destination" + assert activity.type_properties["activities"][0]["type"] == "Copy" + + +def _get_child_activity_by_name(foreach_activity: ForEachActivity, name: str) -> Activity: + return next(activity for activity in foreach_activity.activities if activity.name == name) + + +@pytest.mark.parametrize( + "wildcardfolderpath", + [ + ("testfolder_1/$SourceBlobDelimiter"), + ("testfolder_2/$SourceBlobDelimiter"), + ], +) +def test_copy_blobs_activity(pipeline: Pipeline, wildcardfolderpath: str) -> None: + # Arrange + foreach_activity = pipeline.get_activity_by_name("For Each SourceFolder") + activity = _get_child_activity_by_name(foreach_activity, "Copy files to Destination") + state = PipelineRunState( + parameters=[ + RunParameter(RunParameterType.Pipeline, "SourceStorageAccountName", "sourcestorage"), + RunParameter( + RunParameterType.Pipeline, "SourceContainerName", "container-8b6b545b-c583-4a06-adf7-19ff41370aba" + ), + RunParameter(RunParameterType.Pipeline, "SourceFolderPrefix", "testfolder"), + RunParameter(RunParameterType.Pipeline, "SinkStorageAccountName", "sinkstorage"), + RunParameter(RunParameterType.Pipeline, "SinkContainerName", "sinkcontainer"), + RunParameter(RunParameterType.Pipeline, "SinkFolderName", "sinkfolder"), + ], + iteration_item=wildcardfolderpath, + ) + + # Act + activity.evaluate(state) + + # Assert + assert activity.name == "Copy files to Destination" + assert activity.type_properties["source"]["storeSettings"]["wildcardFolderPath"].result == wildcardfolderpath