From 93886e500eac976283d4e06e80cc8f6cf15b3d41 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Fri, 14 Jun 2024 10:47:33 +0800 Subject: [PATCH 1/6] Add Highway & Attention Component And AITM model (#471) * modify highway layer & add attention layer and AITM model --- README.md | 2 +- docs/images/models/aitm.jpg | Bin 0 -> 105268 bytes docs/source/component/backbone.md | 15 +- docs/source/component/component.md | 27 ++ docs/source/models/aitm.md | 118 +++++++++ docs/source/models/loss.md | 6 +- docs/source/models/multi_target.rst | 1 + easy_rec/python/layers/keras/__init__.py | 1 + easy_rec/python/layers/keras/attention.py | 268 +++++++++++++++++++ easy_rec/python/layers/keras/blocks.py | 72 +++-- easy_rec/python/model/easy_rec_model.py | 2 + easy_rec/python/model/multi_task_model.py | 42 +++ easy_rec/python/protos/keras_layer.proto | 1 + easy_rec/python/protos/layer.proto | 4 +- easy_rec/python/protos/loss.proto | 1 + easy_rec/python/protos/tower.proto | 6 +- easy_rec/python/test/train_eval_test.py | 5 + easy_rec/version.py | 2 +- samples/model_config/aitm_on_taobao.config | 295 +++++++++++++++++++++ 19 files changed, 834 insertions(+), 34 deletions(-) create mode 100644 docs/images/models/aitm.jpg create mode 100644 docs/source/models/aitm.md create mode 100644 easy_rec/python/layers/keras/attention.py create mode 100644 samples/model_config/aitm_on_taobao.config diff --git a/README.md b/README.md index 70285409a..79b707bd3 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Running Platform: - [DSSM](docs/source/models/dssm.md) / [MIND](docs/source/models/mind.md) / [DropoutNet](docs/source/models/dropoutnet.md) / [CoMetricLearningI2I](docs/source/models/co_metric_learning_i2i.md) / [PDN](docs/source/models/pdn.md) - [W&D](docs/source/models/wide_and_deep.md) / [DeepFM](docs/source/models/deepfm.md) / [MultiTower](docs/source/models/multi_tower.md) / [DCN](docs/source/models/dcn.md) / [FiBiNet](docs/source/models/fibinet.md) / [MaskNet](docs/source/models/masknet.md) / [PPNet](docs/source/models/ppnet.md) / [CDN](docs/source/models/cdn.md) - [DIN](docs/source/models/din.md) / [BST](docs/source/models/bst.md) / [CL4SRec](docs/source/models/cl4srec.md) -- [MMoE](docs/source/models/mmoe.md) / [ESMM](docs/source/models/esmm.md) / [DBMTL](docs/source/models/dbmtl.md) / [PLE](docs/source/models/ple.md) +- [MMoE](docs/source/models/mmoe.md) / [ESMM](docs/source/models/esmm.md) / [DBMTL](docs/source/models/dbmtl.md) / [AITM](docs/source/models/aitm.md) / [PLE](docs/source/models/ple.md) - [HighwayNetwork](docs/source/models/highway.md) / [CMBF](docs/source/models/cmbf.md) / [UNITER](docs/source/models/uniter.md) - More models in development diff --git a/docs/images/models/aitm.jpg b/docs/images/models/aitm.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4eab9af17ba5e34c67b43bf5639dad8e4079bab3 GIT binary patch literal 105268 zcmbTd2Uru|_5e8aDjn$^q=QKBBGN^QLg*-nH0d2eLKW#vKv4wgJtDpLB1mu2Nk9-V zkRT=WjlcJQ|Mzyk-S6Ao6DB#_xpQaEx#ymC&)odDSqG?eHFY!r92@}9#(sbsIB;Jh z(D@|*=<5SQ000mJI5ExuJ{H4v0hqA|0O01~{<{U=IuGyPI8OQBlQ#%lIvsU&TSH?5 zO`S&?{|t?b|J>8_)!#wf+>;V(;ngizP<@ ziyH>`dj7=&u{fPK_MxzN!e89+UwGbM-0oj^-9LSv8mnXb3}JC9d#C3wu=o!wE@Jmz z?T-J2yS?)Jd*8qCmw)6A?j}#Lzf{>>4+$JlGJ@E_j&r_E~?0OUBZ zCx8B@?fC=%v{nNE?b(0Yc!q+;jod#5m>n)_6GF04_BS9yQKQAAr3*4gtZt%+96UT+d^`dIe60N7gksMF_|yb6ToP)8v_{W~xV`A^ zg{OWd=6O)lO>aDj;+3@Xjvyhu&A`aS%y)-hKu}0ZT1Hk*UP1k#hNhObj_#AECZ-@P zdG;?HUOGBCyZHF}`3D3B1xH3jzm9nm8<&>;HX}3Z-TUnPg2JNWlG3vBueEjc4UJ9B zEj_(`{R4wT--f5AXJ+T-e=ID*5gVI7x3+)n?4l3<937vWV$RP0(uD)y{fjK@_rECn zztBaEr3)7y9}l1CFI_me0oVhN8lQkmf{;edi0HW&E%&`}V!8*ZpKH2FcqENc^mg8p zq_=sc;C$%6r2RwL|2x7W{(n*Szl8mVt|dSn!2JjCaB=Yn@bK^m2nn!2NKE(_h)Ie6 z0n+~g@_&HhFHrt3++YX6!H$8Ck57dCqaq_Aqx%00H!E0`qj|FkkmKQCg$a)uPzA2e zb~ME2;(*5sMfq`nKtZ<3TcBO=79Sl@XKXxyqq8Q}W^|vZh)>*Q>k_mfPK=?|O7ByV znX9b3CB{v^58LOd4>vm^)G-I~e;LyUeBXjHE-f(W<=}f0qJE()-3r@D`!F$rujaoD zdqrw$dF0aUm1;!1z)j-muZ1K=!EpY8@g#$a@zNHjV@@X zE!PFOh|6rNfS;70lZ`N;VxQ5%>Qi;eOm^%24$lMlMzesck4nGe)?`^wZT_R+l^cM> z_L5HPI*J2;uJ7IeH$dtyynXkX9O13bsNe&neOX|>nTl+nWzV8)On+TM`me>r(b;d! zN)zaQvOJta z2BOr+xiFQ&9SyL;Cy^T%m+v#9xY{IY=Iy29; z%{$eqSFPwdziNo)728D>)TGGW$6@TY{}_Isy=+OU9yNZ+my0suIHywKxB>o$guNZ{ z=<}EXy&Hf~-v=G3JtdV(M^a2}xg-0;YT;c(vOry0zMS0xxkQ*m9B`~F49~^rsjlbS z9&rZZ-2g+_;2f-1j0O!^t= zMoQd`KR>#L%7k`5g#NmZf&SMJWr|yEwQ!2h=wxG5-=9m*DXByTTL$+Uwfq&mfLCr0 zxU?MKQ%6OQL;wST{O0`VD54W10NH5lQUN&GC97<2fJ0>yeg;%jey)nHPpg4 z+*p*(#|r#hy~Xroj!DQLx8fqZ$8_$5sbuDNFx>!Awb!IPH^2j;UM!>8{<&``51Pyn z^UYzM*h7WUnOU=FgXaW#Z~93~HW^M61p6#C1@eoKt*4G10(i56uGk6F#m;uV>VMCe ztp=EF1|s8PL9n^TCgsNRaPIbIRhFmwlZ6Oj zr7%i#X2fw_97nmypSpHWwma?xl9yOcZosu3CviP z2AM3ffGuq%YOdIs0UlyCet8jzE*SrRF(XVB<%gZlp(C7A@mEvx8JH0)L)=iqiU&A5 z%o?Z3k7SB|!>Ws7^l2pw>cSOxv`q+a01@QKGe;)a2PpjSf_ifUOoXHUuHt{m@O=&+ zBO6(i#}mv!*-mg&F58hZ?ZX)whuhAdSM>u}dK8+FBKh>$d6!UPB>15MR~aC9nOLIVU)VM zHZz7e8g!llID)R@Dm<6w7zb43LlquOF2a@amh*=Hg1DnuLQ!y~_Tw|ltGA{eF5hOZ zd0m3XdnC}Z>m3nbA3vz;t<{VYJ+G@I4`WB8KY9Ti`!b&$uMmzu2kvPA!Q(EEGf>}c z_^`Z*56qHP@QAo(54^`S#b*C>Mp|$n^tZkCnBrwm`@#BbF7lQeMU^-br!3I@9Pu*h zhZelmwTV0WK2%EAeKWvYN55&{uFuB_qWYu!4kSirR=2o9bqJFaJYr?H_h`hr*wd zZ&G=?_IJW*|7Xr`fPW?(zY)8hQvaEBlYo*-2al+V5_6aH2VkuWRu_-eMf;-AFVtJA z3%zF@=1hyNhjOqF|1-u3`Ub$tpB`5JzR}^Fs_LrSFraL8uJV0*E1^A?Xrh?oZt;a1 zptkeN?~>ZXnysMgfLxyam3Jp@19um8g-5`a@Z6>H?SAI^!H71u^$)6)-~1$k+BP>(W z>#xT$Khc$>dKoE8^gPfvUOpgxbo@uT$4anzFg7Z7n(6HAxfV(h^em9F92VD|T30CCY zfAy?m%HvK;65gq)IgNv<*K!!a6V%{qRrd=hjk6@Cb&#E?Wul-p90sqS61Q(ko6s)y zjb8Mw;L%DcjKJyUp~E@R7Y&RVa1%a>oaLU<>aq^GKk+lM_fOQxFgRRWbh$;^vKA5MYdK0c5)6WmTyTdadsh%iPg+v_@etUvz(A znasY0W;Tet)f>Q*ua+F_V7qGYP zViRXT0B_HIp$Hc5Ye06rX5=bE6Bs#YLy)Vd_a=K^PAX*PJ+e1h+WzIou%F&^Dfg9K z7ds@Kq3p+sY!WJXdo8zP0@H>(EUjukA$b!@e{fSN&~t9fk7)oQ8uVD7fg6f45htR&!kdw&?CC5_hO9@m-WE*IGx&K7iHBs}A!JOAm z{tw%V11D`GoiEqc7m0^>53~7^j%Ws*`MMi`4D5m-LyQxzx_}H9HA0FiRk)`frZ*)? zimnt0CP?1Bm_)A;hN&zS9%Peoa{2tBPEn4r-PZe;hbjQ%$H~zFZ|nRDaKk8Oas(mu zTO4!{sY^kOa6xe^+Mw^mhPD1}MwHdJi=7|#!uTXD=Il)eLic(750 zz5_Z}10^1#KeKHRql;P&j$T_(>fIDMv>hJeJ4CnTXNczHQG{_g_>YQ$BiB2_Z0jhZ z5$)oADq>fkkCSPC{1{6cFOf+sHeXuNY`60vbvN-ak%v9NjG{^qwtXR^VjEqy^z{ZY z)ISO06nh*0#}+oE4db zfIP~8{0>Dl^!EXJsZT|4)^C*ZBnGQ(k1j6Wzigb&o0i(KtBcd*d#;ss?{_H$j-Aed z4qSC%X`>??k;rd3cLO*jQ@NgpM$bqtJOA0q_Vu#vXL;>py^)|$8Ro{P&B>0{L^W8A z*e1E5)pG;9>7-Gu84pttaDm>!YV(fClic5XdX}(;pbp!tt! zSSIs|r#IgNSUkN(!C2<~4}UQK%O9+>TMEO$NTUi6w!I&akM{;U>5mQGOsozvt5bY- zbarW|jLP(t87hj$i)>Ajt0o%2KI!%#C2e5s)A>u~3pLkkrfH92XU=K2x)lX8H^ZYu zcC(Q$?3F(EIjpDV2Oun0A0f)WuDPd<6G115Xo?yKdq4a?KSqmo%IIf|w3*=k+KmLM zYQ~@xO4fPU!QNf$8Z%wx=AQT?MXVoMSK)Sek|uM!>@#6V z)NoR`V|xqQdlxbpdILEBx8Qt=UiV^(|o< z^wpmo9A9M~65ZOWu;i|>!?B`G;IIJB85*2jH(^*+hFL<4GrB;yP=kXj0X@OidlhO$ zi#vnRBLjA)f(lDXT9siS_K4NdY% zNfC3Lu0`!l$2j1^I1Zhg&`F^bie`B{{pykc&0>V{Q~p@}$Dm}%(pq}>-a1cw#5U0 zL6r24Q?2Y?D`0vnF4ey_)jRRj%!pK06}&e7)#DfP!c>G_cm3n@*j<8WU|kF)x?lhz zJpmT9yV-`HGk5cZ@B29oB0g_>#-RTuOq=K9TZtfxc(;B&LG(4)7}PZdSy+|ESnY8jjZguW7-$KK#bzo8te(&L z%OQ(cVA$eV-?e(av~mAK6nLrgNtQZ$he* z0kUjS-L<$)q-0MnzKB$!$}L@atYg-fU03{K8aL(h^XY=cQ`#oC$xOdD^jzm*fs`cI15nb+ z;~!u@w5jEIJiDa9yWI|f6dT^hLEKY=xMK@=-tnyF1_$i*$NhZRjF(Ijuz{NyCjb=C zeMIZsC(*O=i0|OU@OR1N9}FEoC^f#EeVMOsF^hC{6LYqGW;94PXE{>@yX%seDFWa7A7 zxZ45KT}peAsKCd8RFpl${lGDBOPgbL2leITCC9ilnsP9SdF$uN!F-zSPiWWi4KVY) zXMWQ*_y&+%-EkeNgoryEr?f=BZCKb(;KkcJHQ2lrER$Zn!@OzZp5>dqkyh~9w)`XY z8EA2nH6(p2ZLF+Y^JJ5d~rk%trZVi-yU$I@OaHZx4|Kn4CQXGF3d~ab`FT7UeWw9 zHW|a3AT#H3epusYK#uXToRJy@nNJ}-V7}%SAxzuRyqR+#brvPDu6GNmacPA#6vErz zuqz@6%y*&I*sU5Dd=KM2*jK@VQMI6XFDe~iHjlh*Tw_AD%KKiBiB~)NzG7Sqa2{x+ zbzD#nFy+Tp1O+)3FN~MENETHURl12x>%w`Gq*&gsybCMOqtTHAeD0G41v!^28bC`Q zXU;X$)O{Th#!r%)aCdz9iogU&)9Iq$fYKIWDmIewoF%WvZ1^Un7Nnb>2h72^-5v`Ni!t5m`d}E7r8k64VadPp8`(tu#tR#K8Ae;tYzhc1P$ zOMaZNOn#x$XN$RVFl`g87}H^gJpb_AGj*$P#?Z!1ytVnrL=?d0M{;E(Fc~Mozt=P&_r>u=qe4fHU}))4ev5&VprF%66@VM3 zTE$ihjM5n!;!bMN0Y-6;1B>$TiwaM$Ih8o!oVhlW&rWzR3%o1D?(x{Zzv?iW-t`oY zk@$|n=sxZc%8z0&m&#flEq%UglEo+2aKZDr2*%Wg>2@kiRDB#o^fIFd5+?hn(jE;p z88htxlu7K*w>t0jV>Wu977f=>*rcXkx_LC6Gh1=I)3IGxnuYyQz5%M};n)=63w*Nn zyVQHEg^*}ymH(nvpDp?B_Zp_ek5!Y7ENUv>No>Ez9e?$>Ef>Nj{rI(zkU=4g<4TbC z`q9sR)SN=xR*tNPD3qsVXDVwkh3Y;}N>CAO(t!A1WCyAaj4zz_zpsU7d%igY!K}W_mpASs?QI^2=n_Avt0g^W=ux3U4TgcGOE8Mv!7rF) zVv8Pf)z>wsu^MsyrVHRr)7ebZ;Z1Wb#7doD(hi*)Zvr(HjoC+l$XF+j$asEKMgFR? zgCg*bj>yK0t#x}sirw_vN9DaW8Sm}(=3OP&GyY`OT!2?<>9JX>$SG+y_TAq#s!z4& zIZh{G3JtC?#Yn;H&Q$b5GKT}Bx!lw057zWQ&+a5osnmgfre3u{xIBhXCs~ctr)QU! z<2%+e=@;N#;%>h+qJE)IRc`*1*o+NM;F{{`#Q7ZcxaT+$LVmatSeEu}Vb*m{{OHlu zoxgI{0YZV0(Wrp_v_Z$x6@BkUy#eVHL6n@}Ps=_wiWlBsbF^wV$AabH8jTWzVB@@qTNM(?Ps-RW zdNm^x@r+owMMA=Fh{uzuDtDm-P8p%ZaiJUt>b2La?@-uJcJ-?=xF_1{(wOz7a{kGR z;%mwGFZ_9%-HGwS2!POAg?kp7opvq7(V>K&UmkFa%pGF4lzNb?acm^`1r@>+--(st zwl0Iv{9PPmb+#Do4xYfc-ritm4^hX?COy9(%7^-!3_c4W-gD3>$Y^)%YiAUg1r-Um zNp`ydh-c=RkNa9qY9tyb!Ll#zWS%qbuRPu&pRTF{THJ}vKT=0xbBah7bT`zQ19PV_ z`I>V#*rY0{sA%tY@QY4|4@lDiS7rY(9_7mK zYR79_w5Fxzy*sDm1268%H@XeeGZ9+tVDttwjyWmo!UC*>(JBvA)M=T%G$?YFy`8IY;mM=JuWFhXR{EA+T;O#vF3Nh`%%I-phR_@Ebbd6bhb7MC|b3nm=had1sa z;(^*4KG0L4sz-6nan}Vjsh_xxnvw-=h33VyayW8x-cO`jXRS@wGrg*5gUWVS8JrZM zY@=RDCr4Z8U!sHV>^_|HkZRR~)hxT*z07sg|01h>R|HrV71T2#!YJyDl68k?rQ7)*#*fX4 z8Lt>{%8EeZ?is8Le_%)`h$-F!Wha;->(Y)r)ODX0$qRH_cMOy~e%mp-8E^wI_`b7r z&vg(TQ*5SA?mgg`YCeP9b$NKbXArG*@o4n2#`eAay$EzRTu4NjR$MXt$ zD&3;wp-**j7_Yv z!?m>MDz`2f{qWi!O(WY7(@PE8sG6hM6~%Mx1?!)x1uCy(O{ePvRi=xr`1%~Hr_+?* z>{8CvDS~#IUg;~$%v#(tj$bJ8y<%2@6v=BeeKY^eNQi%TNLlT`#%*JbVDCJ)H&MNZ zgVe7NMOZo1xiEjy;5p7#7%PO$(6Eo;yC+9fw>g-tN7`iQSf+sOQG+L`=%nZkJ&y1j zpk5}1DDu(`@+epn=Jc~o6>%>I(k}`B+$7J`kVQGPlrvm^66(o83t_-8Y%oN6fQhil z#dy%$Q@%J2u7V7bVFjw;VCJ4iv_Uw83fe<}O7FAbYtAy6w+~g+{$P^2 z-7@1)6D9S@Z?2muCyQuX=!Jb0{w6?a_3zkB2dQ_T<7V>etY|DKKfw@3tDa4ePyi@8 zDOgQ%0`ilCKE7rm)#_YY>UPFAZK$ab};>@}@M|gn??XVb$aTHt_m1Q;_sFvIr2b2m4$YUr(HN zl6{Ws?w}M>(kNBmR_+CQrU?0-M#~dwaw3O%!qX@;LK7qBI{Be)odmATpuW;JmtW&W zdzom>v}3dFM(afC-z0{t!J8YOVZ!n-;~XNPkX!=9W5>822k)||7PrF175@1{&>s~^ zw0JDc!{)SaRiHC%_mxzwmDNb*E1y3~3BVB(C#4ID2XVmhgUptB*Jnq)_jI3H3hqTS z_T|g)P}VC1lf!zgCdEpdFzslFSV`ed@a)%x-MN~Tr5MoRPn?ObtYw@i&!DXiBNY2P zu$H|uD#9$G0YWoZUEJx24_iPMzif!-FU!_cf_kwmcBm5(58r!R-*AeAA)rbx=mchp zOzWS1(D$*nU2y?9uZ0x0DAuKe0?x0q3`djT+3nh;1oryi?^1noD;gR*-i=9AdspI zk&5M~v9|L~Plz{atTO;+wWlNfRh{rX_{2*=9;!HOA~q=0)GJ{u-~Hs2Wc_zq)4%Uq2O!cD^1dt9C^7>%cs{+)ZXU!FaB;7u_EndSD^SuUD9 zkG8~|>cs1*yFwu^SiP)4^?{(@OY7~nT~MMrWm+4}7(*}7?_V0tAQL}liGM#xZSsFL z-}1+{2>w&>NkVab%~er4LTI+fxWd}PU9f(kzSKyoofU1#o)VF@JQe&mM&D?(Q1H?K z_(a){3~_Xjx^g?(3Wm5BAo`7DI@#C0^=T{W*2`(Bj`{1}eF1|dW%P^d$1pKW(P!)3 z7L)d??Dx!F&Ax2@a!i~71CyAS9sGT-UdmFY*;P2qpOh(335_{RXTN#GYIHL(}0?kfjSZ z#r8~k7hakt@U1F1ZfkLIU-iyD-6!SYC2RFtL)~xJ(J5WjzcgJh*H7Ax<@aRv_m5{! zXV+|HeNL+_H7BNP&Q60i)>vF2l&x1Kzg0vm%$yY*rZqeY)tr4~A~MlKcCMlK&&Yzd zhx&cIesaD^ttU+}SZ!f1&0dW!*^@w$Kt1TYewb&`}@R%r{)ctJKl(AxbuHhcPC9F(Oi((-Yo)17xPLCxF$cfv{Gk=LA8r*yMSn3KixUwfzL+vmvnHb^xR;?uut_^&E<4leGry zru8jHQ1S?QSE?Q7>RJG2H^7P49OUi_hQ{L-*k@{SsUJeuny&l7SijR@4iWyex6Cq3 zk06K9!<6)4_yExg$9?-oi?iC3LJVw!Bm5d?T16l*c#dq8=;53QWo(-EqRzDXCxbg% zu5Oi0nRk4BQhJ}c=V(o4&dba^QHD+E9oD~1qL{K8LAe}*N#n0GgX5_HiB~dRQOvm zyhO5#KgY83{A$>JC{*y11zYAQg>zK=zSwzDEXTQE=XQ;`nOQ&U?Gsd9lECxW2Zc^? zfQbo!t?eWvus$hb^(|x*zzHoKlzMht0ILV^vVSUhA+C73*lA}jpI1xVws>(zlk7An zmh??0QZc1Kd0cuH6mdOmLos{lf%34L@)cMN`K44*7aepjK7isEkLz!SGtSl(^{{Lp~ujK{I8r}!8*M--5p6k8!+)F z8*~%l+p~g3Z4|BtG4^QjUWjAh6Wouo>%bk_peavt?8>0bUL$*XIt9lHFF z)>Fcc7Mpp=vyVR~t>yUG*}Yv#nx8oIak-o5JREnm@~dU57Cx%UW~8!ZcAgZ|?#a%HCRj}`|HW*b^VLeF#&@D)TgWa>?h`0Qz?cO`;u~w~ zy!7lJbMIl9)PbF+#h;(J?shz$Kb+;L=J#WN2L0xdjDb+_LRzf~;H+$If*hR*L5TfDV;?vAz>F(nu)H@>a$=dH6@lrx(*Rf?i8iLE?!W;yay`Sq4z5K#x8UNeJ+M zNpEt$KKvClPQJ5tYP|vm#w{te6Dmo7I(bBJP>PGZQVQ9ymh7$Fm8r4mYKhu=#k}h8 zkn~{6&4j`zY*~%Z;By}3i=s*$B|Z2LSpM|OSyB78Y`z>$=Ly|Oc`hq1c_G~tio4n> z8IAV|`0In_-zaH$A@nKxvnhLW0`#Gbzf$X)bULA~b^WauStP<|{gPX0a9q zXC(pUn&egMaw;3&(!6V*y?K1V7RcYz(vX#h`C$8H%8%{UqgSH)`}Pr*FwVTJZ{|#o zqRC^M#I%={UEiYzBGEn;GYA8`z*c?KKnjUR^OO&1F# zr4lYw5wh{_Qe3)E#d49aUkTtNjdz;v9#Y;~!;HiE`)&YIjD5c?Z6{nR89_5V>RM*;jg0Kg zWTA6inOlC~Lo=Z&H=e|DlfFnf6*Rh_TjrB8#u9OGOZlt5UzyYVT#CG*wT)SySa=aZA|D7eVMRZF_CTZgyH$IF{8>GTFfrl5$D2ndLA65I+PED zx4$ZwnY=ZD@#T9zX^D3mR&Dv|E^Bk>R=7Xqer|R!pj$FmKrlDzF?lgRH?ChDx}%4* zk%g~L6Bg(hhz2>e@<5SW97%!Fh1799*x9=Im@7O>IbkaF_z>x$?AGQ17nNUS|VQwwX zdx9+AaX8~=H7p=EJ_)H0*t@nPwe`{2(X(ymxXmY~=vPoeXGq{WqNx91Dpchfzq!6y zGtR8Y5dDoMu`;PDWw>iSvYXBIC&xlBT(-|3tb-5j*wb{`iY=Y-`nsz^*Oa#5x0>yq zGdlKm@?)(!85lKWK-%rymy&CFF`o(cnCjr#vn9R+m0aGL{qPVSzjN4;;akQUWj!D} z$j)bapHKK_CXW!~mtNpLQQ1@oC46{*=kwpab=0@58)6eFWKpU0^D*H@1naZm7 z>$XnLAH)c)&;V<~-%`IZs_JZHV+>AIr^QuxvBioUNN0qr4p%#Rzu-<*9$yb#=5(*c z!`G`TxT>SnP49WLpZ>CR4F;{`gq8xVlL^>Laa_0%-?U3vgO5%BP(LkW)=_tCK4VCbYqM6Omz!{VlJ4RLAvsfq;{9BE^GT)-m6=1| z;v$pDoSC=BdMjW{%84nWE}^D^hJojR-q@3jbgNBOrFI=#$YPFB1fdmO*OsW-;Nsj) z`^MTcSxz=mhuL=)eOT@a4v>sdoc6EavW37qV@1B}9)4hAqfoJ%fSeHqU;X4r^nsES zti=~6Ba&cb8e7h&1v8%H++jm@#4B$fkv8$rI7&2VbkgvI!ln$_sRvQ@!VHk+&}R-; z*nTgnU(UA5^l#Z+G;nJ7t3lRv%=S2SQY%STID5IAg1mOTtd#QU+ftte@gxqD;5IXv z4roetcgqhkpI-A#^PpA>5NrzPB*@>-vQ6JJ>NCGfsTmt!>e`jX+ZPU>$o6+S`}T7d zuAQa$RM|nE=-Xoe6 zjoeS>#J|erPnF2i2wY<_v`uWrLvn(w!KN%s43W^b$HObuCw~mks*sH|c_Y5q$bzup zu1%_7s*tVReb?SL9F)umd~-f+A#l9kqAj4o?IqW&bdWe#Be^DTIGYz;Y=W>jKqE2D z(m{>UD+!dm(E82JgT@w5oFotuUQf-5)l&*9^Tr^>`7<~8$(zQ(>-I;RP&dsJ(DDSP z7B$$*!F0_M7z6i6Km%IigixsYbF> zPD7~cVbMxKg-#NpNf|I}mw;f0DYNdVMutosdr#gpxiF)j-(}r!KJYf5OZKRe)Ey8) zoo=c2LAX1wC&Rkd7A;2S3VsIbP%hdSh^eK&wsP$7lpub&mRK*R`j3V0@>qHzxbx5K1TMID59{9_s9%a`c#pIp^NvljOjrE9(XgI_8yD=7bnj!xfiPX<$;>}+XJft zZZD@!M?34Z5uuvO4|Mp{2)fCUr1@Q~QO{~5hdhOkD7$U(eGPE?E7XevbNdw{_HGX> z8X#?{c9{zv5OeCd&->^;OL3PA9RBovjDZG2yIvi=j$Pl+5#qGFle-f^G{rRtHmxT= z+Ab%gzwe_|NX`&1lu6`GIPv=1tWmie9&w${xzVc3*WK_vYnE;c{{qKSZ0VHC$~WJG z3e8waY^2`TY}BWSqyPJpqx#Qd7}q59r9tw!R?pfSXl;VzFKUdz;Nn1q|22u-wyFV8?(So&^Fc)DCCdopbxa~_75&fN)D-B9<0=?8S@jT+C8txFZEQ4{ik>D z$2va7l)^L zV|bm$&!8Wi9;=iiwyNOgS4pZ)Yixt)wcf?YWpLLe%DBe6*5-T61*+5_0Ebd;cqe=MNg675FuD^pQz0AqqzA&==+{a@BUW&762gdjR9JTRb*ALuIH6wc&=Ndh7<{er@ zmATrj{R6Uq^5{yAQNf+^L(=x9?&eSTeIW~(465E;jC?oy;yzdA;l8uz5>0~zp(a26 z$MRHHY$Z(xN1(9gT-_~PQEhX~Ys%VHvK+@J?HL7&I~UE>)edR5`x5^=99rleGtmal z{`BSo*L)cDlj92yy`LDS+(ohbL-b4W`WhTY9Zk;9Nq9p{wsTf_8bD|H4(>i5qj|QlLH^no{$7Jv zTXH@OXgzitm6!>ua6VnkSz2D%HGU^`oa=Hw%V((k_e4PxTs6 z?G2sqijmOAY>%Lym({pO@#D1a34a#aReZLdeb#e$a$wjLQn6zZl$$o@?XniB>OlD# z8~qLT24ZV?Q%%vH#PH8Ie=4{zwC2AwDL(37SF!sI)AVwcjl;STD_Q` z|B*qH1jd_seu`(9_oZWsgT?*oGHcoAlPKCIay^-H1@ob=vBu2IymUpRFnW1Y zNMMmEUXiN2ys|F^;E%%?Z(v_5_}xbP{Nm|2+Gb2&*zluf_B~6h&KQ;9G5ZU_4AI4X z#jxF8r1wLj4{yFd*M4$Nsa#(_oWIx-i9!^iOSrtn7AT)V%{Z@HnD ziAnX_%(7pzb1QQtHM*6K5KwL7vdy%g;uZIfxbIFjQ+m_EZ>90OP9@B5!0Ib*K zn1GGO-qtmoy~A(z627g|ek>5Nx>4Qb{f4&2XKLo$kU$>qePhpo_Asxh6yH3`cbyK& z3GAGhto8Q4HM%s^q7dFom9|jIzPVl8*DWzuCY!E<`ySkVlV5OjL>QuBB{idg?;OK{y1!Q4kbmSqjwk`mX)H|ex zLYV5!)YW<3%cLWy1)%cQXEmwL5T~F0=-n@8ojlygJ*}w}zpXHGPH$sAv>Mp5rY zL;F>Z`)>~$#0Kr6>sKXL4;)R!f=sUW6jyn|0&-(!>8XbUSr>P}_L%4P(#4}D4oG220~*G!?o?m1*r zt*t*7&R6$G^7X|<={rBLiNlE!>Q3fk65FJCdrf9m;ii^4R&|`IPs=FimrY^u`X!%@Wn^M!5xp@_K-eyew0i!4AZEQ z)tmkE_@lAy&Q!+Bx|XHb{N>8-3|mDr+GfuXM6*NhPW#K>fhJ1a30uqP$K&~lnb}a; zRLl~}{)8WMEFL4`dqC~nu*my;YShomM|Jp*K93#`KIzL5Ng6!Pqw(M?R*dq6Kp+P^ zafbQ2f(uR2PRMlUJ#IJNc|V20cST7WEu*^mSvIHYb0!T&gGGyj@8@k5z00k)tPvlr zc4b3GER~Ne#Pt{VS&a^Kl3u^fJ{oT(rL5x6Rh!QSB=oeTn6WR?wsYnYC5V1bG1^)%sjOOu(B5TvbX3#q$!^)mQy=*iEA zCOGzqo~e1H3N&6|MU>^hDU=ol5ygZUUuec4uw_)H4VD`InTiiTcG!An&HxZE3IkVzWB#`Gqg@{g?X9$cnmTDv44JUUv~< zPP}1kRfE!R6>%qyT83z_1loH}`8>2zJb$&OQ zVFrv!^VOg}1#@=ndzlfh_ZlC|YOE*6H28a!ZHf-SO4|I81DI^J-)8;2>DFDyaUj zT+*%R<#_?3_A$-~n!=BZ!%^Nt;9$F;1=LZ+yXJE?j2`ddh!ZdSg}dFG)!@JAhH7cn ztFU~t5arb}SQiA9VT<(E;$)rX#%NKLN$OBW%vM?$iEPnYF;bSv)Vt??AHSMA&z0I(wpQ9PhS}VA6k=)-`k9Q|DXqHxj-CH1e{lps z8Fj7rOCiB%ywb)cvn6BV7GvlC!Pk37HT8YlqftZ@DT4GC6p$vM2uPEtDD^{?A~hl% zq_@&6M9dO5=il#&;8ze-|xLK-W!8|$VkpUW$$&?nrqIv z6MYdcYwT#|xdksx(|ZZ#A{9uM;{tl+{$NgtYu>R|F^=Nnvw^#ZA(4Xt5_|o?^Xx(0 zbn*GA$nBnPkri8iI_%XxO`MNZNmOCI&SCx5>M5NL)Rf?dUfvTc)sixe>|V4GTVb=d zeeL&id9QWoGO~U8?Wq;CP!@QLRi&FSGfk${6A*3`Xd?@4-DKY#&vO1_i&IP2LoQ7wmHLoCC2xCDl6S}qZ!=>16Eg~xqUKY_MK^m7 zb}CYg@Au*z!Ko^{@$al>er~+)0fndlyx}XG0Mv9i2f+^-uwAZ794ga`t+U6>cA^l=_4ZvlbIMwz=sNCOT@CW| z5mu~>&S#&NTZ$uA&fAyyD}!E=Ecpy?>j`v2MdBfE6s0k6I@sv58l38fIt3SdwJB@H z`NiKrlTQSUQGI3*zrfRKxV#r9N-!h%?dbx-mve|6!8oNIp3nkM-mvx^UY1!{Ra}+c z01`&(V>rbe9%+DoSPVD<;OL_m1ngk^E^^4*G<+#$YBo_%UAooAlkL%8szFAKNv?b2 z{K*NsmdD9!?Ogb<3P&DhmzS%B^%*P9yMfb_Ly`LNnm78nc%#i63mNX4qt>!z+ZBve{h^BbvB2Mx|K%fi!4w6mB`3AptW5hKenKs5jbaCh!qnG5Yy%<>+b z!rZj3z0%f-Mk_F|>8Mk*iMk);eSY0gkg~d;rad7)NNQS9y1)b$s61xbr#+dc85jtA z)9vwsBmB9pEZ}?+oIDdvEPsfl5JPG+InHL@=1yE&ZmE6gR=74_pIUwVB}AtCnYL%B z%gHS9_}B;nA>SmA!Wk5)+^T*<7w(p`RF_-s#E6>3s3#v5`)B?$teEYnJIIl&_T#x; zLt8LsLC~GFG}~G6X(pd|I`xYh5Y<{p-N7qhK_OMCRI$q3S@Ft)6 ztGbE6TEfw^h=!+UQj1lzsa)>N8YPoo0pOvbt16Z8**@WXjOr)2CNeb05YW*T_by*v z*!G&oE9lK)a5CXA>v=amXZJ%?!$-vD;B3spqxkn7hwpdcJnav*b1soN7lV|iYPWsh zcMm!TIEA?NouiNkU+Z2x`^}j;BoQ_x%+Br1*9&JmY!$$rp-~+hlV4MHa3?7ZvTc$K z{4ci|x+Ff|V!J)@OkGgX!}0;a;q$b#+dHM3ZrNgH7`rH`@y$DG-uNo<=x+y)zTP^{ zrEdh1XTbEMO+eybZA}xD6))J?t~9Lt;f;TRk)8LW>QdXN z4ny9%8nL(f1&0)s2|PsqiQ|^Ihni!?ufy{~o#j>%-kkg9y&`tMmAFDf_XRm`S!whq#zL!c;5ppJ*jj4 zhL6t~3hTe$@3|M9ff$IB($LmfV&~7CHA9Bntv+8+GyYSI(+NF^KTLAm{Q!ys%C<`3@{lyDnH%ZSnxBHAzd&m zxKK~gn^He~#kITn4P5kwpz9RCIJ4dkKk(n-(UF)UgSBxN1K*Byw{Se-p^ zY5zTA@J*sJ)BRYMhDn2#ZwHFohR<@o-bc17QetO2pft5Pdx)#s3BS7WlKq3j4==?{ zsQtE63X^4Dxzyta{WtAAc5m|Gke`fi8~ygPQ9*n#M}{M{jW}w2*4A!B^KZ)1qEk#Q zbf^Z1XmWujGC;@C5ZZnM%3lJcMrQ?un}tju-G{2ca|D4kbWE_v z54mafoUc99(P#d5l@Wc-)@ef80mG%imJz1g)Yl%IzgzjfdGr?k&cs3kVw`8{jSj4B zvLIL-{Za64O*bC9FjaJpiap zmPZi84KsS#nWgzO#@Afg-&SP!N`#CMn%_LE?)}U7mFRS+>Ryo{-%C~_9}ac1#pqhY zyjAONZsl)8ue^I-=6JqSCha)7BQt=ZSPo7iGubc%3#sDZc_{ z!d3N8ufNaTpnYGEM>mMf2yEDxw;#md>YPXCjFR4(3h|{=G4E#P{ntzZ%ZJP?(*cjk zF=Tl8rs8e7^|oMf=RsDT_SCr$@qLZ#-1BikzD&Co2>?T3fS_H7ELcc)A4m&h!835` z18q*d^}NW2ZFN1#lgzwx5Ob&2Q^@w;Lip*0ud&V5>W7V$W3%K&D3?q2~gAn?q#MC;TDL|zHjKb8m zmAmndT8Iho`w$QpqF6ti>moltq}U`!Q-wl~Nsdt6*+EntoEWu41v$}GWcUjz-j%=j zpnGX1wUWC{D3=2JHU6{i<&e*{X_(YP5Z6h5EG8*-eC{3V|KBhX_&jhHcCu$24p&1# zM;3QeNLi@0WimVbSM}Gwplv3@dDs!;(T}A&F@y^j4B5R7;FH%h5HrrYBnT{0P|}@< zcaIVOUpGvZy+@E8P-aQA1wqgqr*OPbWGqHI6;4+hl|$M(QTWVG6g{AuUiWRkiMOZk zEeDbTGaY)O;28&VdydD==AkZ)_zm?p*3K>KFCWS>x~#=DVcseE;$14B6lT6()*eS!0gDbLFcI2%li-{aSivQ<%nM$-VDPO)g3^I|`q2 zA6Tu|lqkF#fei(uIR11P#H0(no1!($buXO&OxveVs7%jyPu(}4;7X2h$Y{8Q zwl!4Q_ZXQCvFDM3aM9~>T&^CM0Mbjn|8o$UIKL-j#FzPt zFnD)!aC>>-a$Y=UW`{rhI*{$%=v^tNFF~@;Dl*`x(5NtP?i;6`TH*@y^y+d+FJ@@F z#MW!WGoMmo{Q~V&tk+ZH4SJ_<8yL6iW8TS{K|h9YX!&ViRXeO%@Ze8s-0vdtZ|&Hw6nzZUElP|k@w!G@*j+j%KII?*hR*N-W{m?l>OO7r=lu7 zt9uErr>Y@z38G>84^DI99bLe9uZMcC!~T;Shw7Sh2q#eJdbHWk!47KYbK2=T)J>;= z^@Yjhg#`=q1vxuMi>$@@Qn|Gx=xq2VF<_9qO*SOX264BIE7Gp$Fa^CFZ+!?zBv|8X z_l*=Z1*cr@Hcr0H`$AizT*tvKK~r(LprU}bBD&ALDpR3usEm*A2kJ7>s~9%fH5Ozy zenJaYnV~6Kbd|p5y?tQH#OUvFz28sLgUwJ^Q=Zt_o`tu-4tBItlWNJ_F#G3?=?4=G z9-hk%Q@L$E&2{z3dP5#U_tU7a8++VN)N63~$s=8~4d+o(fbE?1z*&hle?j!{nq%*1 zEoCG7$)S|k##fd5{c-EJK8*~r&GkLLu4fV131-1VEl4FH40yK==RlbRbh4)GA@cK+ zvptivDwf|6A8LkhbK74+j_jF9ru{9F45TaLPrf#4zx$e)$^;5_$On+b{|ONEsdYK$ zk0De*DG8A7yK^pl$II9DSx>E}$^0P`bEDA^v@5>Fc&wJ&>0lQcI&YXhszNYG^q-mI z&b--<75fS!&kCl!d%%eNa=1n^#$3~1JsrYkwT6cdV*hlPY1Ht6=Bi@pX7t&0@~$xa znvLJe!D=5yTHeAPEAe98qU+lD0OLM+cy!@d%7%LKiHnp5pNF4l?UF&e<3rIyzuO*n z-UE7i)($W;F!?zRaWY9O`!9UhgZP4^=1*foqkPsYVxzrMZnQYQby;G6D!@iV&q&mm z@kYccUB??1Zo6BW;S>7|rw*&q(H~BI=NuyLcQUXWG3t0xUv*@~e=BC^vB%b|fTNF* zSLaM&gl@k8yH@Ee=&M#G?kRzYr6UXdldAcB8XTWyg95W}i5fOcR(!D`oJhF}BST;) zOlpkvWJI!8f#Zev2eGQ6^bOHK|Ki3b$JIbs0uM%AH!D%;ikc+wR%!h+6aO$;`=Oee zVgJgd*@<`Dhw3K27pIZ_BJ*=jEPnuJE+0l4JK43B%UVtZ-AgJ$UjbtO+1IdikWjAP z6H2Dfjky%NFJ6P0VJ3np(~=}T(p1lsPvM3X26KHA?j>uS3GZ@IiH(LkKCto1zx z%%VD@<@KIizcBtB6o44Uv*TDGhj=eza8~BmHmIU;I?(|!gqK@wM4@tbV$SD&3Q_#= zTdis$+#{+0eMMv_ltDXHFh*K}M)hu9sS`s+LjKn249Ycy^EDbBNZ_Si@?qF3ib$mM!`0(#fQc ze|u$=5|76pc3pLm;~T38dhAm8bp1uq-0$t9(R-E#>TEu8jqWN(k<`8iK%4j%XU`Xr)4**j+*!1hYPL#piV)I7@6sDJLoT5$ zs>|)8%tw)XKtr09sDOpW9*?qbBa6&;ETwQf#jf!`do>jXdlRoeRe4Q;EB_?v%Vsn z#nC6}IYZ<>`M^1#da~g4hbZh_C|M-w--|jS&$k0Q{WLhfY!f!j9nLdj(X}fLNh59M z!E|~og&J`xag@E){`d5k%*yl4)dmJ8gm^=L@y35e1;H?=9C|AF&*i+opxKiKa-1nS zj9@&2oyv$kAb{IpqWb#9Q5$|U5Q2M~(4mUZAyhjA!=(2D#4{B*eyA#!OA{BICdI9u z${XYO@o~pB=Qa4gm@+w}@Gq!+p6I*_zX@2{QM?2n?>-{Gb6!PN!fyu)h>ox=2=b?& z*`DW=xKqkD8@e_lf3}bcfjc}pg*uyu$5H4{dHe-|U&5DN$kLhJwA!=!%?cb9e14IN zxNulr)%@R6!=ZXh_5P0mmVt{!&i!X>2lB*J0I3X0z%Q>FGy>1z|2P`F_T=BAkqK@% zWVzy~DyQMR{S-!v-Uyk4hzWmD05OD6!2SScoEkM7@tZ}h|VTL>sPD9HxiKfWfF=2HG|0BMJjGU03VtJT}9Zj3G6f zp(zL_rCN}#yxz9@lN-d6?e{MPUp~2E4wxLZ8lrQqy@ETz%8lyA_xgBT7W%0f6)vyY z5NOiR!%j;`MXDW-m6p4*+2R!dZ?^UVTdgrU zTsud*OXAal)5!H*v945JxR2@!z262P1!_F;ET+`z?$)IF7M%`dg3TG-ITv|xnH{UVin^y=XRN$vvaIHH`Gr^G7ENnrr+zLCk|Fv2&qPQeWTcoD-yKoTAsI| z1c^FT&PG>hA9V#-J8>j0X{0^mt`NGm@JBP2#y;Gp%G+SDVC2fqsT@C>oKobw{rDDJQ8h1d#a#u3=A07v={*!+l)K($aVRPL(Avb5l zq%&-P*w>{AT+?}iRu>8WLIE9iW3NQf;ZwVn7^iQ~Dn2M#a5Tn(69fG@9#J$7WIsKi zzvBM1(mg2HQha(P{b-zAg26z(su&w(q!>qK$sNC}&BqJX?fPxOHET0Hpf||_;5xC< zo?bWwY)us$Eq3SO3lEMD@u&Q~tKVMVE6@DY%8&>GMH2$+LK0Fpn5qquowT?Af=qLI z6rr7LdkC;%%l+_Tp1GpzVWm!G61;kXPteG# z$$1k_QWkp6U>H{GXKx{QN3? z=VA1!5rLH`>L{H`0RJW~*8?D&Nic%P?Mu6H@)>#cVEYKDUsfY~B$)B^|8`KX!~D?* z9o$HpU-6FBD&6oE8Of16PCVTS6<-6hYfbQVUDe52y3YYdGI?c#Rw6w-Sw6nh5sTD$ zy8dhmS=w;w?i%}Zy~)c>FERd+7W8^VK0pB>EE^#|LSA; zKNF2&Q0PREXzq@Z;Oqg*v?S3Ub^mJBt5r|tw^Icc;RhdV@ z?a!@_%GYQH#NNUnyXO@yJ&LZo@lisbGu9Q?|z zOf+jrGu36tJj}-N=kwQ#$h;H!C~@&iZ`HbxUY3*?g_gXQTe5#a?>P{iIh2Z&DVsyij zbXaHIkWb;uK-(xdo#b2xrw9Lbnu2wixzTOaz?6gz>mJdh z$;vVEXQj92$we9(!5DNK2kp?Ig>hn7(xn$490$vbmWwX(Y#hy~M6mrFsB7sZSSX9q z#J&SW!I|k_4F&whjurWPCkkV_AJSd0BI$_ zXL_+HFOUUMwDVH^@eoV3l!wf0qNDa^LV?8|^AA~MVJ!-;m#$RlvdU><0BfG;7qgzS z>?K*hkcjebM*5Ihf)0Q7NS;&pi1uOLy}Y)q>&jvkYo zohMzf#D=Y^bk5(_qHKA=r^B4-_o&F&(dLhgx_g4_C6L1}mqVyz$dY_fanS-^XnWSn z^gOD>^{BvRG0ZJ?pHH^G$2)`_pO6o=eiu>i}qyAD~ z<+456_&iZuk&%pp_0rBIcR!oBA+&+LhjkcKfcn;FrmrV1+gnM88zEGY z1La0t&+6D4?rv1pe~M1>bF0}O&n5w=#=mDH0gz+hYd{8E#*cIR$UVnAY@X!*-m~M!lnP63;q=O%+r;i$qX-&8FRD4%$zHSxUPFhL!;o_116OC!T1&{|f zBZ(4d2&a=NUmxbipmU(8N`&Dy{w9tvN));BCSk*wS(roL)%~Hre|q zQKtwWGN-Vcnl_Z-vY*?S0&UmfG?-n_e!u#~M~Bf8uVDB+Npq$m&n!n^(MG*;VV|*k z_w1ePHQt+Wj*H7CiuM@#-;)?fCmb|r?N|mw@9o<(qneyyE48sHjns^CH)_j*THQf! zyvBH1^9I-=*d)o05hmm!7pmjRX2hb8_-ys#}`w&e`V z?`XF8ke;`|hDv?Ke5~OXR*uw zl+#RYuQE4Xr9n5ky0_xZldhqAgpoaoc|rhusQu4{t^F@Bu>R=(U4O@^JvveHqwl3h zBQLIo7j~2_;OC(+YPKn83{0&1mc_!toqMkQN4X83&}kqAf4QC@TY~cr>nVgmQR!Zl zKm&bI(2`m5h+qn^eu~slH|A!K!>4>Pl;&y%8xY?Yh;boS&6UV!EkN3uP z-X^FS-SCU@P%<0#Rq&+r_uDsKMbHwT1E_eb9$GV{>06GLEzNlATW-7H0mC11S5^}G z@}$@7@Lx1O=dn9ggm_>q;bE02Q+dzIytKvpkK1=qgRaGC4)RO2J$_ECRJ)=8Wi>E0 zvd%@XKBe^w$p$DEvZXE!%(mdYzMdjQY43)}CCp^aR9pyl8J|U3C1R8(%hS)=SguJc zs7{`+&Fsu<$V9&?x~#Sq*E^$X^wajW8ZTCr`J9Siyi8ODbeKVc*yyc!rx1!-L~nCg z?mctIKNQx^@BAH2+Ad?6~sOLr*0^G^*YE&ZNX_7O== z_!=|JD8eH8xr5DagsS84_a>Suo~#%$0ob!rMOr1PjU$g^ETsADqbm5CYLW{G5BVtA z?@`$5=!*VX66<=VQBWw;8XO}@w>36iAp?GNhxz7Ni;J_6 zb~s^cdQntO35+GD<@)dYkTRSp8!}gE1ry&tN*K9eefRqTVfILZ**HB(;~$dfy--IB zZbVm;G-pPorGi2p?d-2i<$Jk56Mm}A4Hd+`b65NMlWjQ2haeI%JORMZsjG&YJt$UK zUq=9w{tGMd>I#jwK4!mS#Ad#J5~Va_`)Tvu#!<1CRy?ShQ~mffJQwEH(;R|AgISfm zyz+OA#GHMY%NJkpfBN}D-ap0G_J!4_Usj(qGDDx9&vrtp?UJ#u(K4bauD;t|?Tw_bSjC`DZMZfLTC+*Mr5qiu_IiJ8 z`pw4}^iIWp<TA8S`zZ~0b= zQ(t^nDbW$z7v-~PtP`o7q>PiP$-DDxXaMJ!dX*_5#@n9a9u@nxB~Va8>B$|aBON7P zfYzmqvnQw41@^I@Q5_YG0JiGsI&~oU-)YiC&s|!HHdy371+GR1Pm9I1LS@oq+RCAe z^VCKO*S+!Y-&qD2jgz+mXGfQ@KcTn0{(=OBaTMggpjW^K3(}zA zVRcIdtQ*?zV>`>Xd_)#T;v>g0V!bWPRx{2c&);QAd|CVH7;l5uIiof!P>6=HWoZeg=Sq^W1nqXlxiJ|`i^pM!4GG3?h zFaX+H8TsCR_L2F8C>MJh1%~$z5?Mp8YJq~KLc}YKo)TOP@jY#z}YxMJOWVLP}Im#_WlHUu2yzoWNLIImkwQl$+6HsQ2@gw_O zJSPA!>ILfCWEE}U;&5wqm+Z)!gt0Od7qMV78t;1BxVvx-vA)dmCfmzy|Df+gS_qk5 zH_URTfzMw75|j=Mt<|2irt@smGW>x*$F;ND3)lJ-GeM%`71EmU%eC$7Eo~?q^R84w zf=XW1L4K{klRxwn)V~e-2kEY=NpFl^w`sCCKsIO@Zw1*2@~wXm&vKF|9Lh(PTP=#)NdEUHZ(`6l2$Pa5+DG=a^GJ7B;25Jp{D1 zEqh0H!X20LGuFG`wyC|j&Q;}DAz)l<;@Cj*R z$4iH?-jl)!6U9XqgVzN^mTk^}9XV)VEy{mE$Xz!QM;+*`XH!rNmt0qbpmYr_KzmAFa!On7f**`P6EWMpoGEMy(3;PX1?9(|}9M2?^94eBy_ zAzhYesK{56Cp_>VK5nu9x65}6lhejMvwUqsf7e8p3%+BK_(o*L%NLVM*LFm4^yav# zIzKd_oZEF3H1o3L;Rij2KI*Y!}#f4nYTQ2oR^dTiKs}e?JU;ko-&=;mLhQ)ZG zvGl?YoL!MQZ-;z$qOM z=e8>9yOXt_p^JC(cZ=3lxnGo*BDbMOvtuV358_3C@RAD1+~S4hTj`7Km3ceOGTGAh z-`gg|S$3Q+&(n6_?EbD8UD@^S(cSB|mtMs{r~%$TGBSryd$7I_+Ku464F zgJApgd@Geku_1x$-9UxAAAM^)OBr^_irTYMgi$Z9YHz=)+@hYz&Z86JzTxTL@<#N)#<6HN3(~56J{{^ZZjy zO7~y=!)I}Hryxz&|Jje5Pw!L%Os8cuG+qh33uPics3ak~czB$TQ7weW-686elCCIU zTt(bBVTedp*TDwr`{)3@`57|B8}*U=u!&SFefZ&z6KAWa?Mfi+8%_z4+w~X0Dlq=k z;p-cjz0ob2pV@xudO9x=sa6p*K^jQ;Js^N;1HVZWLg5Qn0ar&O(dW34 zB)ZQ_e6Wgi&Xj13u$lIsgMXaFL2))mKN5)5pL{o|rf^UOBGc;WFH#|D_kUn=fPRr* z#}o*#V0bq-T}bb{|ykiSAe(Wlha&}9uPpT0eI+OQw2TMIuO%; z8AJQ@@D>h(LPW`nOFT1Dqw$Q+egnOICf^$ZRmoh>LWOVgq#0Rs2i$+Wi`nBK$F>Uv z31jVh4rx&lLH15|@3^tU?AIT+W)qp+5Dyx+v3wQHGz!OzJKbvG0XS!N`$@n`mL*c19ywYyx zg?JKdk_i=;;=J+kd(Q`3sJJBE=BS{8D!a6GpQ#}kS!8PZ`-y9==Agf8J1>r;{IHZ3_{ZDrX8nWIL&d7qaQZEDLn4ix0E zX{(#=$QU&nCw*SGKWLaWTbN($Y+y;O82^SI4->v%=qN4cLnGcRS_6$Phd6~7P0g*- zhwIIn3bLkXw2DF8&5z2LyLs*A6B;vb&9pD#T?mXsD_jM30hY2DBvAN(vas#EO5^PJ z+hX5CM;hy;)(d|_r#i0tIL|+3!GxgEs$SfIMe53G^zg1qOx?K+=}=S^(_(rww`*Va z;4kRSFop3nFfuKW%WJ6;ln~t0!&C3v6Qt;GUik~EKPbl&j1QpF>(Fa}d+Wm$R0Pn> z<-x`^Ek7`BX-TDhJQaPPLxqja{8Jgou>VK@n_zvE6}sZ)LuL9&W( zaV!YNO{jU-)85Z_@;DDBT>dD{diq*a&S0Wxc|IGDfj>nZYR%lKBS5(;J7r2C_FP>fvgR@iChxpW8&GKMBuLfAvn?Tuqx#w$(8WhXz5q8hH|h=}h`8m@gh*8}J?eT>a6cKkOR=OY%>%)kvg;jR8S z&Qmy!zUwe@iM?r7I)|aK)se&8WWMkl5dMCRoIPToZ;qu(MTu=GIT7G`_%JlBLZ7K| zXUw8DTaX^|mBo%xifzQXSVcB}a(mNfX6{$~A8b^9tUKnmAI0qf-8BM(vaxzli!BftIwP(`Uv1q zZf+oJPjbStCEM@0%q(oqn8P)W;;c_y@X3KKKjSBJOmKfe-)Ezux~AOJaz5G<6%gfj z(a&_oAEIgf%4hdeJL%8)+pqrxedEda3o00&-SjOd+zzs88BL;%8m;^ZCWwV31w9^j z3j%H&An<%iEg{t=IWqIQ*fZ*NI0H5?HY`j>sB0=*U9Q|AkcZg@Hgs2{wOXq1%0%6C za~8Z!!j!6xB3^Whettx`SLq5CiP)<+9%X%1m2YqB+2DAi=Gst6e6;!HM<6a@G4I*c z_A8{;w{MT9v9$3yZrqi%3{Ik4dO~XBPaA$1nu8vPha`Ig{10chun;AD^`r3R4jH}0 zxCd@m2pjR-Mb>mfZhUgT0*sS0b-nVWDSh|P^H+$^2D09kGn@pNPYVowZ_O%baoR(4 z9mZWqd{orI{?WsZMjZc^=3IQuub$S z3LIIbKlv>B#O;QbW3x<4_#0hN=E0n?Sc4*uq^Iby8JX>HwU2bE64;eYlAbJ%z+0mb z+>q-3`M+I1ndYB2xut$v7v+#2L{_SIBSPZ_)}cw7ImXwQ&l1*BhQ;b9z5W>Mj4RsG z<>2nh0shwCVTmSd-(Nm19ubJ_=lqy+BdmYaHBHgb8DP;&Z2kajXTDEbE}LWjsjoQI zAHGJ=SLWywihu6+~% zl;RUCVA~AADq~oCz#I-24IkcOvL^=m`V-c&?vT*Q^PmU)Kln1Ob1MB!>Zn8{*W#=C zHZA$q83A1Lm8tnO|7xIUgX`(+^B_?)x!iKGKufHyAoZqM+f;(e$hTfy_1k)%qHS!n zhG3j&#*fvwW@Zuy;z5?RbN1ir%>&GP;gtbZ2rMT?I$M$O>MCTrnd8VEZyE!u!7XG& zc(DO+Zz5o3W%B1yhg(=mh+JxSwj_Rjr3tVUt=iH5Ili#iBO>=q4uIDkgxcRUB8Gq4(fyooE8j`v<|bXN znd3deZ|$*=?r|j!SXG}}4vjTWqA*e=-Aw2576&4_)!2(Ct6;0X#&XGA!S;4TgEIej z8AEoS$176Mc@dCpO+tM|54k>@}8<>AqY8FnyFnVa}%?@V{3f958@R{$z}Nd)e{K z)MaWmzy{6OdHs%Uwt`uIfsyCHOQ*)&#ws}yB55g0z-s8jE*BH3aDZ{*Go0k?`9)wl=xyEFzu|`%6n>=IJHqTI0gh6PdYieh8_gDeU5GV zK`pm;pXxI5^ee)Nq!PiA#Xl`u=sW+tZhjhNbmCU}r>0amNE>Zwi>)!m8&)}TwDonW z?Y)`Ioj%+@>COo6c>qTr9h*A z#sWQ&6{G3)s5?YFLm#1>w1eY>tqsPEL%2bzq`PC;rs1wok1O}}i$zZaLQKii7swBL z)d*Q3_?rN8P(o3~F-EDk`=H z`hK-Oz>L!3XEj~iIs-n)7ltmF&z)ms@Iy0+wYNkXl)pc><+rhMS3~x+pg!c#6RlQr zZtZ}nAU1qB)h=tZsiocS^r9BJSP9qc9cA}s?}yCB^$`~1-MdGh8dnd&Q}n;xRPksu@P`mV zd|^p_9_>id>hdiujDjX?I&5Ck?~2|+)b*8`ex3u_zBWyQ5)nMDMCX_-^KDxCn$K07 zaZDYGp?j=cD4N>{E`7&k(SnLbl_$A3WRwiS!DB)si2i`_{fS`vWUDYnRN=8Wi^1i& zB`qa(6Q;7zr@mr;82w97HUK#v*~cN+y^CPk@p4LZ1mA4cDKeGnzD1i{IHtyH4!`2u z84bD^Q2Uy7#fvw=laoeLvY7YJ0z<-~au-c3;*Myoswd+m5jD|-EqApGj>KpXqW|N7 z;4hl!LSAzsJAcHn zEmbGWJyzqV!*3L45MFkm*y`LEoqSV`<-s5C@x}}9SFSZ53D*BO%Ca)@p-*}35vs=R zh45pl+q`ENWu|F{Lu~|o_FC`faLsAX`+2hH<+bRTzi~3DyfOR;ot zV)r6>qtfaI@w?drpFmqHB27?rqc6?V%{xs|#78(*{U%`EBPQg;PT(vm7k0(h7Riye z9vtV|R9LopsYt%V`nijp=oz`pN!LLrkZ48_;s?yFEG08DUv9YU{w|l^E`SF1_~*zH zl_$))ETzYSbb}r%HKBh>ZA`2;)Wi;oRdFeL-m|_E3o6ch9k41(7Oi=S7Gtrv(didp za|<6atIP7~)xe(^n8taMd4UF+yZTeXfoQF@!n=!fM1bxPhhFpi&?tQm;|E9KX9wV= zfb#mP`nDxf`;n}T$yarQn^k!c>momR{Ea%OrWg{I+C#8Ph>8KSJT`nPASF`lv?Z`a zx#UNv^wC||5(JP)7>QrHBJ`GKEiC$iC8^yxB{}5zgc&(9A#^C~n!yL_5a*zDKTDnQ zOhn|ddh6|?MQ&x2Dd`_(L(ATZ=LVPJO1|~7#p9{9og?tH%@fUy;(+LlSrd{MjE_amgpm;Y6gWL#yrp_E^d%H;3!!A>q;;LQwwG?L;x1wL> zOVxzO3%7pS$l#iLFAMWFS@ebXW7_+e>WF8bHgfcO$9qfu$b0H*oVLU?pIbNu_dcTC zFBwVRA4(^T#=sr9ANG*%V;^Q^|)o7xJa_%qiM8`K{3-|t&?kdJu>jIm|yq=Q@#3kq=ET*I8xrx$v36!#>wgC~#RI3!^GAkh~J3w<$9NW%G z)WL@lgq-E4()EXN6g>^TmV&L5Tuh9?*$(^>-Q`yA<0r07T6IVYbFoNYP^h*AOt-4^ zj2IN5IXgXa%8fixM#cQa5|1mqRBso2PF^ZSjx2lnTB>dyhSwpl;TyE3i6(#?NWL#z zqIQw)lyk66SuJP3Gw>s^p!%EI*{IT{og>@bXRP+&90k_Y&x)o5L7M#B$ z|Kdw+bN$r4&F^Vq7e+DfkHE6WujPS6Z8yFW@gHJy1~T zSB}qd&Gy^wTtCR1TBzEa>34HoitGcFhZr1vB%qPkiL{ij$8G?-ritY{1(@|0o3VF4 zdZ_5emiC=d;I$GlwoHzHg)2BrwUBwuPOf-HShr}^UR$x@pee&LrDdtMDSnGo%R&?Hh`Z7#Xp3a!(`z-693KhCH>g~Fd05Y9a10w_$S~(c;PLRKsB;8 zd>kRPxFkhA@oDHu#~To+q4WpEmQeh)H_BXkWqd6BAdo=RJ&yyxes!vS%I;r)^cp5! z)>_T#DVM|LvXURVS3wltkV3V1)K?3l$CT0)RaCo=lG)#(`I_1mrJj;^=4@S!*6# zF@x^CvJ2PfZ%qC3{I{Ipe5_aYF6K zwXbP){6{a*kS6?TaIP!wt#ktK39yZoD*6S8>-g&@{V{-4`*bD+K!r?&&T>J<036yt z3=C4?$$m78GBKDPm;TeRcSuHJR2$;zMoDxJpMake#(@X}s?9#88KVT<6?QnymjdZc z{wvf4d`YdB!lnSM6&%h4@|uW#s`&)8Cro8(3hXxyXoue2d^cDZVGw)M8Rypnt<~uc z79~?_7CEq5UYd3Z;h39O&wY@plEs%US7?}FVlke4zYz2FM)*0+^-N+f%3*gyIDd3M z!%<1;@SQj-`C{4%GLaM=nEtX-Hb^XgyY5BYvs~)3`=FIe$Hf2+`|yF|<1O}FYS2IW z3GV zuQQJGIAEKD8b%<#G&)MQ1!;LSFKQvFDw5Px>voiSnFJ5=x6iL^ouC||yXY7mZ_eU3 zP+)bfy?Z<2LX$A$)*Q|mvbpMOqATGD5qNcGNgpT>JhNzkMhP-ft|qB#UicZm1>+cv z(2%@Y@M?AI=niGHgiK;wK7QP`gM)$p-2;dI9AYmymxOBo0Sl(YC%Q`2jv{kwK*8t` zg-suI#5u);-rS4LCxqF}KAWz;Q8QhJZZqv^ofWP&AN>T)fgiv*Vqd|E6u*=P5V0m0 zvl8M~S3yOrwOYh&{UBvdxW@7CO!*fhBe!_B`Ka${Z-fW_Kpx5bj+p95OP>d#&d zfkfHSfEX^`2)%DVOc}rPhpK3}Uxvm{oI*W>MvaTnaDW~lp$&igOLBILLqPU|5=fsW z=|0|EIfUSU)cbCywWCz;YxBcDPRCuYc0GAMdzds@y zi6i@wWa>4p5=;jD4W?i`&L>*Dz6-aUO7l)1x2|4d(cy8tE7y4SE3a82V)MlFQ*q-j zk|3WBG)N2fJ$n9s+-~Al8l_`eoQn$xcfBAyyo<;V~)p9YetM0QY|DVnQge;Ig2 z^Azb(rg~CdWn@y`O69BE!OqNu@5cqn#LL*}%SV_z39#D32hbm2P7$CxbwrG;_Ori= z37H@*3F18;?(z_TM{wtAXiUyfD@q`fmt(%kx3gd9_>|3AC-o9L-PDLX?B*&Fig>bpmjVq*=velK*2@}ZPOZna88UMVUJkqh1}CB%~)YA~UK z+_d-V%m*Xp=}?Mi-o(;2yU>wHzJS+-#=&h5S>3%ay*nL5H_JDE@P^%t_A-!%dYy7* zPs3635UH^Yf#*MLZ{2=smP6Q6>z;q=q_q)$@W%DkkvqC|hyx7Q@0JdE&!G;ZrsI&} zsJ@fH8|ZOd>b`jSpn%4FDwx)PTw6{P3h0E^kZcG%MCC7~h)vN@S4#S!=C(fYes6+M z{aLw^tVkjhA2z*l9)Dzjc ziYu3djnj00nX3%WT!)^Ih5)_xGs_a0f}4xiQhu}p5Xv9`p#bV4fD0j)(-OYHWm0vPe6-bwDExA$yK?fo>Hpq#Tg*|PgS(7RqO$^$oRn-ieq$wNpDW1gU{o0Kr z+WlJceCBqrBI1L-xJi2a_qc+~Wk9_?V(*w9(WN4Q#2d)3ML>)h4>_*GodYk$KB{u$ zUF{j}84}ZSKX8=)UbNmQXIueJOECbUP27rDOrp`MJ~(hNWR}OD4bwj@eqkD-H|aT0 z#ZP-%0Vmm8Q94OxS1C0F88!;$R$*b6YuetyFuIiH+KMFrSGOU@F4xlQ(#G=P)kh8(<<6TFLIe2f6UOrS+K|@awef^oc^D6}K z`WWuKm4P((uZ!Cs4z|zcZhO$G^bWWQ+tbddeu^4XIw-!HuI8Uev>@N393fy1Q>qAc z0BJQ4=91~S8$X5(@n-d%d8E22XLAFq5F(y}a173hKDQvWJX|k#09puph~@LYM4GFS zX11i*PR&;crCuIXa!I=yqn#8mmb)gPAr zsH}zUQwimmu;qtE363c$t3tL)2W3vmGW5xwiEnOhvWg?ibZu20y-dNLD$5*oYWE>Y zHY6G!eU~C6T`_pK={z%&hsw;I2OM34h5kyEhD?85Qfa->LCb}nAp_!uS@6viO94w) zJHH2IY|lMDF^>d~-hv;fk{h8u>ZMwv+%l@(s!QTs;B}9&;vYH#&cnq!4&NRY?zp;9 z`o#d^XXJe4hA$v6fE4+4kAnmP6B(3LV&!;&7WqFg9E&CFO6ebn@O`;I5EFekjnRr4 zPRQ_=;XqaR$54y$bJVB_iv8|u&;y^+XRkPkdB=PY$WDC~aD+5KZ)hwAJ10Hx`M%I& zTVLui_drzZgwi3x%ML?`*o!VRy~^-N{vmGCimsYzcf+}>;j*53mjz^kji|SW1yOVV zU^cFDQMV~-bnX8MR-oZ~Y&;Z)b=W8W6Ldkz{l+lX5)4A{dTA)3wPUTn)H_0XR_t%Y!iP!005+XI9=oa{hD>GEO8j$%~A z#8E0Nt21pWMZ9-eCeLu_UW#}0bzw$j$>Z8r4){hRx(kPKCt%JT7__GJA4q-|fD@|U zo|*lzb8jK50d5UtZ~ACvAyHdfDK4Qbi3n`p+=N*G%j`$xtp?o}y`bf}8q{5(jA@|#k%)VN zIaS5$h#*YmA~8mV-1H0m`76|smqtqC_Ply#Vs9Ux=BQ~1)bDP9E-4yKUjgN^O#VL* z&3$Um_j_dkML#k*NMlQ~pdZwsE1+u0l#72L*S^GqG6@(27{KK0fBhv;0s66R(qHe} zA1IVqS#=E5I~FoflebU=_{)GSIi1~hvz_CpdM9wBb-t#CMug~`xhmr(?@p+1h zXx0g>tUGG*xx{bC-=FIsb-}?kO@Wd-v)Cc{?+v=H5`YNWuE91xFZHw*hT+ev+>mhc zT>imduda=VL^>A{I8(InR})zW&?3V>?fG@-)HTAb_0_YNi%9{?v^~0&+r~HFqUxI* za2uW3iy2>qXdgx2Gjw=4d6ug?Cg zfCVR@CbD3Zq#eIvQStvZ2k%f{@4#jg$8pWOq5D&YnlEe%^Tves-|f|MVY-b@m^~Pj-4I+-Yhe>V1jJ$F4S#?3d4)i1jS7^aq<2 zNn=mdKW@K#^82yyQm;pO$%>Eu_f^}hgAz-66pJmrogaVSBpWhZ)>NrPyx5C;b(_bb zk@o_wQydNVTqBB0WU79Z@#DZk3_N*KL4qsN`%vrPNIx>#C zb!6Kg$j$x<|0fhne5atlIYk0L!QOCkVf@fw{@RvO?B&Qe2KG`cCPo*0zD+)zeXjWd z(BO1B%Oye9m#|T|!@2nn3o1#0@7>&8S5C*Yc!bX{K51<=ymH`faL=c|tN{XuUL;vS zH@BdaLHuBS=Z-iXoceFY+jO)qp4p1B`%%wCFaH)w{l5Q$#;fU{zBXwO!Z%<(uqi)( zzCV!9|F%n2uC%XSYeBf8?YcB#>21nSx2Z~&MaS7HYhITWecagrU6Ul+5ffo!ib-sZ z#}iqUw8p}D)POUlITD3eAfV>zQP2&TWuO0{AH7vPLyIWRGqLbj1-SzTZk`Aifwqja zAsGyhHstVwhk};ZTIQI!@(4e1LphB&HtIEEuOTTv8MuZGNi@3O`tn0%7oKJs27VV+r&p3`Xr6`@wm@U zc0vW^6_)qcKX2Uoe*9Vc@jC>09>Xriperm89&E zIQ74nZIHFvH`~y4RVO%w3oPfC-*C&tr%JmC$?~ICw)Jt>&L3|P1}skxHENj83~qq? zaxJu0W=BMjg+RZXw-88oNHtm0@s%8f z8L|6{3%+&l+2`9x)A-ux2w4^?sH|UVCCteY8v3q zy@ez4qSo)xKi~ZWNrL-=MBNA=m+Y+@X0Mw#I&YP`!n&HAVQFn7*ccF7$0(FPbKtbB z(3v-We9K@Xf>CCU@I$v+fq|}qr~YF-P5*D}`PYO%N&ndL7bI{AX#W1!TxvMMSYUYj z!ik-2eIRU|_y-~iT|C9Uld+9m*90t9z)S--j`ol31^3AQ9TNEOe_zx3e@!M7?wBId zP1Twc5Ve;XBCoX;5CZ=)x7Pi5*r7j=*$c3x36Kl5D0T0_WmTx)Z|qd*Uz4p*&jbc{ zOj=Al$F{U!HP91t_w1{Nj#p|!)+hzc;u&l#&K`4sQdpgGgp`lL(=NE2pED+vhIt?L zdUvjH%a^EZonK3pY_-H-n3gydZlODiBRR%! zy|x%h)zzL?L#1mJ)7jH`fuS`73ePY{WV_ zdP(@h%vpE7?*i6LNs!&xD*b+cEFXQKBDtLg?5y05oY2=ZA?&LYoK-^eu)~1>(RxDNh^v0s@WtUDW!ASW#n!j8V_*Mz@p|l| zgYYqN>F0q=v|M$2q7796q|xW2FbHpRnJe#V=R!_$yO#NEVZ@msH*~igmr9V4P9kh1H_l-lcuZYlLUg?+NiU zvE#Z&ykvQKIc1R@gJ%Ld2XN}RyDk@}WI~J9xaR|TiHYIrB7~g>MO7p|WLHtr%(4$7 zyuhYt`h5G5bGJ>yf7+-#9QBycq28rr5!ONl&l6u^Ua22J5u$|54X$iG>^<;ky;bra zbG*WRr*qL)%CCNr9+|jgcq<1I=yxH^jWhv7&8a7eOpQ>E3W2F)#nm;@!f&y|y*CCf z*sbaoGkos!V!EE^KI?W)>t(C;OY`=Cl|5lAe||6{4qy{IfxT{@B6HX(opd$vZbU;_ zw2$>?FZOfpDyWOl3xRVjB~`NBU;u7B6r6bv?2%0LQ^AuPK(yWZ${^sVtoJ=wIny?7 zaN5{L{X~IO$GBPBkzw5V=9+63RB7;=wd7wD3`_{@KS!%mfWg~wz|AABH z`)V=%pi7-k(mAdi%Y8Wed887XrwPE`L+7D;7AmCD)WqwB6!mPhqu;>F|EYFS=)GH_==e3N<7@qq6tD3Q~TMQ-@H zq^;$!UE|rxAhnkVWGy*WI&V&H&w}8iR^t<@OKF5o&7_$B_~dz`(If4MT2UmLqO-}q zd&n9!@fJ|(8i7Y_n4yG;?HO2Y$-zSr=K`@tI~Jo40AT);18k_|uNUIz95e`e=g`RC z|MYvQoiXCfZY696?%qTRo!WlkZ}E8oi2!pPotClqq_VQTVqm_z?PR2UOf&&!(&q^W z-vw@m$@i+j*ma`m;*=<2TSG~CAeyTPMFbbG)9)|YIph*lv6XFY=gF88sRxoSnS|p! zcr2DLbDG@m7teT5qLAu4f`^VR&bIegq ziVGNa;3XWPeA&1%sbQhlFC%@#I^s+i>(Cq#?lO_8rVbs{Z_&^I=quTh6Ixv1uM ztnthtQd@o>Od|YBjV_Q!KK_BEjn$jtPqxe|o3*4iqjP%cx3>jKsY1KV@Z}M=y>-l# z4tT%-D&8)ov>hWNpvs#vramM>*8+j!6UMjzEmf`@nW|b@Mqm)?vvOz2_M~8Ad z#s(Foc?M|Pcf@*6KoMk3!Ed~bh^5z)=J%))6;aE6=cnVIA3~gedk+>a5zkv(sEYb- ziR!FCZ8xXyaU~iRHvNJ4hwt?HKOteF&~ZeIsoAt#>mPT^J8-zVU0wRp*B?>XIkP?m z^H)4mr(#3~dgRKb&$YD0vY#>XW^!-cXevZ!3vaHf!8)rv+t@WN#f& zKhEnDaohU#3u#o5cUTMJh5LaA*;eVnsKmJ0RhH?mPe3h(v9X>Z0$D9^sN4K`YQEQ5 zNlz{LTf;-~f%_|_Wx!n zUI`c9O@>axJ=hT!W=tMFLe0Dci;;8q_3IOgENP#ZAg1_){N%%0<8r!*O56ALH$$Et z%TB2w#Rwk(AD*zHj0jl+d!hCfsnm8g>UtCzQ#>-!?y-Y*Wqj-&GWL-azN-_ygStsc zBpk(z2-#AuC=O$e{-%gFS%y!v8Z7x~epdY1{IGHTm}8ognzlS>4}5|-00K%f%n1em zMghqMT`Xedmqk3q8blPZ2H5EZI;XW3sXbj)k0hjZRHP+bH$&A2BVjDeC?{IYTM`)@ z8NPM%DPZ>rKL~DLNfMO&8RYu5@E{U10Dtu$bI6kxc%2n3fcsu*z-^5~X=343K4GXA zPX4pFA%gM8ZK4kJ8h$cj6C35$nYTG75hn1Ojl|fNNd6 z1C2(l-@G4i59E#OcLM>SFD43xneOEN`v>0v-X*}KOsR48QO$C!90y$SXBC%8BaIL4 z>Kd=hM}a=n4x}RWpy9_%5gy6X-?Rc#J=+ff5&`YcUQ-5%4s-zS1`$mOj_Of)p7(QE zh+4$6Q$UDue*T)II{BOJkbH^v9plvk6e&9CmXP5D6e*Di0SmPIiE9T>m+ z;gEv@TA+#wA!D5>m*}Op!7qo>QYtJm{48oZx5`Ud*(k0LlUpgE~9N*d9jW|(E<1G^w1(dudHqsWRNp2rzK;1a=* zyvjp*%+Ydp;5-#h_`IX!6j}=PY}%ry)%>{VhX%>4;m`FJS37TPd^*8Kx53Eo4W8XI z@g)6cufQh5HmhJC3o9g$!T%=MWij|dkUmDN(FC{Zy}3sG_2=-ga5PN|i`(=YJR!N> zI&r$vXzDSFwCIcbW%#yg_PKa&^c9JK`TZWAH61IB?^+PX=+rpQBCrp1Zm1bEz^h<~ zL#bBigQ(vp4gV@V5tYMf>y+y-d>>MhUj0`*2REvNou;u5Q@Q>?eCtuK0v$fH*qAJf zH#zT}+!faN`pNjxCq}gM!9&m4cGAA4VHH>)hVfH49^Bx<^}jdN>RpQv|NDO+@C?*X z=_j`!1ipN;I* zDy%4e8t(Y0FE1_O(($R7?(}21><0w35ERd*ltU!{UTdULz17oF>GF^mDeT!Y$G(q@ zwW9YK)EN;4&z_!>CC6rp2$G-f{fb8_KNLZk<#Kv+zRSqJXiVOx>t)cEP|0Y3oXKbq ztc`s7iQSc={0AcE{x>Gnw6eTHXL5qt08oK{M;UAg+Ckzqx}HC%z5@PV(ZI9dwRO3i zbub=di#m|_aD`3T`Cox<8B>A%%2#Vw=2Y{U20LYmfDDH1V5(^4c_2;43XINY5XHd1u z2JiI=zZXWlPQ2Nhv<@vSXwJ?^H-`FZTD&OQfNKqcJP5GbtloRl4s;Y1WcI#X5@(vw zRHwoQHO2hJ2P)1^`87)^W?6Nt*IH*dlSeFfUt-K zz>coXh|2qI@UkL3Zvj<_sgmRnJMq<EE@m*Tlwlltz}smD()X@}gJzZMxzlc3z2RHish23)N1Kt`C8teslE zEvk-n4zJC%bUaLSUTwM(^G2m*3Ct_#aV2gqgW`_J961TJ`4oWlH+XUjvrip~<0dsx zZJ|7W}z+fWNltFUfE?EkbQR7t*2TQRr%{eCn!pkG9%e zIBI!?7-~b&$e_$N*;bTPpg1XYwxrXCMxJquEXIZ1kJ`?rlXO@k^LOJv{)+QGE7pn*01A17P24 z^-?$Ybm%G|31DTGcFA9&K*UsTEaf2qJw%vduV=NUpK!6itPedQaV%*(UaH(YH51SJ1M$Hib*))onrOw zEh!HcaT;6G=Rl^v#U+g|FD&Wsi$%@8Gp;ys?B`8(4hpKl_ZA<(XQ|oR8eFLrqS z2)aW56_a32Wd(l75zztRqLruSt9Ass_<9#LSx_Z$=j%d#~64nZM|s?zDo}vU|DP zW-)9+EF3hOXZ$uo(J)_e`a=*LhO?#FDFlbfIgo%yoS9$K$lpw3T&`-M9Qf{*psVRP%X=WJri4mNjKWLO6-@a^T@bH`f(y#4qhfy z@TA*mWO)01r6iS#=;|@HC#9zOT;44l} z$ql0O8!dl0QlreB-h3F-e1=^@wljsDOS2M>7F?J(;V(TEz+cmzjf9(rkPXaixxZgm zC_8g`VBY1zfNUDGGQA`}lF93%BUT(`Qa z8{GV=^(}9;OVg!BUtbHZNn7Tc6ak9&6e?^umKH+LE#S7`KJufeR@Y9{f_Eo^`@j>! zNjbZALbJCdmKkx1&mc1C)yzsdnWV*EAL`#-z-VQm#d2isBjJI5qGMz6;_6SYw+?EO8905*`9S^nSvI7KDHfh^ zA^kpW#+%K8`wO{n#l9GgR|5*z=An32!2nSMUu6VytnQd_45wVk)t5USlf+AAT}sH@ zBZ2big4~1ZMxwzU+;yc<4t{R44RhKpMHdMwk_kL#*5!2=KL-dk1gcdL{U~VdL>#8{~;(8)(e1n*SVmu(>u(Dl&!- zS+ye~{Gv3rBv!9~?Y6$c!z(iyjCq^y8KS|Euh~X0kMqo)C!0B~>$u?|6ZhgBLUbG& zr}gez{4zKr>Ri79U8KPKm1avYIMZk><1|%U0TZjB8oNF$LExfvOG_TbsMAu ziuf3dRP*>0vvJ}HH{LB$U27`uS{+|t*u%M|oJHdrsg(O!LW%2dm9I`cv#&nwt$uNe zEecgmvZ6U3n@Ys zl;p`VAEWNf^4=m59QR47l0{hWx$6em>p968fo^a89u?MkWc}1n$M&Z626$icK-;sd zPiIxChK+YyJc-rQbO>;9!F~z-Zn2DAeg+^YP^~5?t`c~vq1eWJJnkXNeq!g$unf)T z@8_+AKfpTuesxK(A&R9PDKPF9nGL}fc6YKc|$SN(57z; zlb8~^;u#cM6hjoNfgVKa)MEsI-w0(#lq+VClcmWq3_*7|2pla=;tyG_FlkO$EuAEw zBOX@?P~sXVj;U)nWm6=lDhi$21;&1beRV#yd-eQSC(}8et~0EKPYI$ufj6J~8UZld zQMx2j48L8E5k!_xXz@9qS6wdLmhwLOepQCWIMR6$Q=jL$V%xYDkGXgw-qF-1#(C9G+S{dRhNhWm@3MDwsl2ov z#W1g`iFr`PNqL`Ul;xNdU)L(Ce0=0c(tzVz7YqL2)5?1jzmEHSo=p zyidn)I@*b#>tq+cjkXD!jkY{_!@Lm`JX5v4xR`!GN9=Qx1<8o~B+-@vo1B37EA_L5 z3kU6@TZe^FI8H3zB`mKG3?NgWS)Vql&LOPOv&+_yvJz~b0aZ-H)<`mu-(OhjI^~E z&xJVYjn27tY1*5a(9$(!mjqS2sO*#M5X<7uFm=E(eS|f2F|o(GjbSvKWZQVp?4*z@ z+WFascb}yqD)Pnv0n^G+;vOCs4ti15OuJnD)d(%A6_tg9TFpw_Z9GwPFRjtz%uFKs z$3DvNjM=hcYTd$pQ7a5n*f3W2Oy_dZsHskh&Z_GLtb^^1h4grbgtb=dYsc>|y?}lQ zQk5sBA)7>AskF*w%8Zp@CEy=oY;S0;U^JHV*L>Wb4kVCV4U|RJtgn|;EiCo!mMx(; zw$@J1!zJL=BeOSJfprrGD2$8!Z^gFWRX%$L&sasf{*#Mof=a>Kmm2)S z$=G1rk~e%C1R6t2Oa7c7hk-fr2jT`mOJ#x%+^FVqY}+5m4V`-=v-%WeV(4>kIETLq z@dJ`jZ;s$7>K<5EWAd`Pr2}<*D=%6vMGgMB?>Aj*N|B<3P5oAv5DQj6F)ki=r$2Xj z>j5`$81>+{AIj@dK*4=>{ErU;lYyCX!4a4N?X>SS6;7D007u~+Q3 zu&bjDOdT^Ax+PlLNMqL=79A)%LXpJJ)xloLJGuCX)rTn}dzNo-xWEb?(+Bi*ir%Z& ztVhbPEOp)>8V#eF2ouXKe<1dDpFuERHqSJnE<`|umv`1_I5gOaP@e8`hc6qZN)<6> z)jV+n^sLK2eqI(ruc$yy@!XuAzAf#EXKrMLO+FTcbyx9LWfFFzrA1L-HJnFkThx2b zb+V5P-PCE(b{GSKQO_9^SCMI)3{;TE)w%Eoa=gN@M=YT|+b+Q-xaj?akOs-l1dlrA zM3?Ht_Y~WpF6SDE)ukvBy#(cn86{}Z?+)0}G24Z0eFe!A z{0+n1qtBxAf3h7ppZD{lXM=CQ2=7lriyl>Y3w$=9>OzM4^*+Qc(;^AAOF8g(i6vAR z!aHy!)BY#)z~1~728z}~#!vTDh%UI}p@-WL8tGB% z>9Gy*n>Ygs} z@HM}Y!k@7H2NLYCi>)VwoRsj&P!#d@v;A3RX}i^%!YYzkpPcKhDNHGJ_@s|Q2!sx= zdLe~U=&+(Xj?iC57u(Q&Oj0ib?{nJo%n7q}mQ(s2x65s#Uwncuw&&d`-g+O=YI6ST z0qlhv@za{ryO9_0+U~`GI>e5HL@&*C9OsK@u+j-Tb`jzbe#FI9C{p_xujkvl7pn=; z1ZJJRlZ|rz`oxYx_jJXEjF4o-ys=;Gu5WS&Megjl7v-FdI@~lm7X73t=a!>za>C$5 zUP&o-t1laQx=J?k<9%9_;|0`}^$}03t+{u@0 zP5Q~Maz8yjVM+EHD(#*amI|#rN?HxyqvHIUnNi#81@N$nS^c#*G$ z_3ROdOJA;WV!404)@A)3GP7mgtQp@=QKoCL9xdHHSV|AKB2XVrS?1EYo$ZbhU%lMM zx+4^eIC%Tt->iwAi^Xd-gK7p6b0s=CG?-8oTswUGOU5IzviU$)ov^8b>ox5cMsgRk z4n)g38DQ-n?9!9N>J6boiyX--rsl)sg29B99NM^p9=r;A_y=%XaC?kS6T1cify|`+ z%rI{H?Z1=2c7w;Lo|`azq6KY+5~>S?Uf}mp*Ixm+9^>V^gPmPXD zZ*@jF#JSe&FxH~IHjW_u3(VL@R42+cf?Jq7lD*0IbON7IG?nRVl=k?ze%PblOg*KD z2=gDuo5U@-xKAARkkuad0Hg#Pd|XeEH}15Kbd<`^(jBkGC3R5q-3JQ>4cL>hSKV{W zZ+-4L)m!LOnw%11OqnAI4%SyGQB0mqEghkvrt(C*#}v;dnb_yvc_8_?_5InmuvKz$ zHIC%ck}m-csILL{WM^1Hr;e={K6l}vLX^9#AAjwhC70Y@gE_7H@mGt}ld?@k6d~ZK zxCTluqE1(xsdKoOXERTnH#a|}%_e?YS-h9KRLsPibk^>*%G(wrKX%+DI_`weR~w!2z9c6XUMe#D z)8eg-Lt9?ukV@amYKvsfIY}uu>4lQ*(v~rJza30*U@nJ(B?tFY1rYeLM|=9kKYP7s zPY3IK&xTuGPuz`aN)SFlMHi?D3<idcj38 z$H4qb8Pq1|n!9xUCq{RLo*VVgZzOcEDaDS|;)(_%)H(2eZebTG-im&-b8~vV%U(mP zQ{S8}^a+?_#Ag*9vMyClBs<--oh|(5+6kie~IQ>jm z`|$5<_m2r;!&L{7`VDpr6oV-YBc$BWu)S1w5b=}I*e&KNZCUxo#ytL; zzHFm-BS)wdfpyeB$))sZ`2;pOYR<6q8{n-dcf(;^Gkd&(5{tQZc2<(Vpz-WHNQ>$e zSrYqmb@_2Q(!sY%YYyqseNVMlwq+&eT6s^h=zFlazi8oF6jULh>noY+0{QyYg(j7N zTb1q17f_OUSCL0L`aQyfeK`MSK8pd_{>67$_@x67gxKUP3=iBzSHR6-TKV$@d|oC9 zx~~6sN$=7{5*zUIW1`@R^jNe4Nhe%AARW0^Qzby03%0f<&X%}^6y`?Ef8tXKIzz4> zQIWR^_}pUjChA>FhdW#`@IN?s-2ew~KutJkv40Mrl~cMDpL#xKaua!wy!P7L#qZZ1 zY^VaeqW6ogJt1!}ZufMpd2G;m6WiaJq){gzaY^8yjKXPS=082O-F-fd2s+<93cw(y z&|EbqDLM6?&U#aGesERY>Ea?Mta@1qMo4KgL@g1y^K8K`3l8A&%yhmvvQ>45%)cr% zE9C~QcRt;{b2ny!Sp^o&s`d%nI1z*?L5EX;Gkx$d3H7SjKxvmflzz52_*#!h!tkwV zvEKBLLt};Vj3;)r%MKeVb4Tf^i8GOJglm566^K=EhzZI*7K?=npTEp$G3~Rw zu;;so;X=8f`TbS;(~_W&rI@_LS0V|b(IOcAam(%F0A9$D{^rz9)t&oH8Ymk_rWv>A zm=o$w1b77MG*u4oqO+3Z5DZA$?K8Bd8=ymZfM(0VpgXnbeN5QcS^M9o-3@7P!4~Yq z8%B`?*BZ;wvKJcGO4o1YjXebOpHH3anFf(|=zAdD#@ zEpU+M)8R~@?ITuznSpE4u;W!Pet=@D1Yo(^v1mg%t-vknZk4^I!?W-XT$gMD9dyXK z9Kkxnw58-d3x$B3@hZ$>X2eRnfst+Rri;?jo&J2&j>bl@lxG=tV;VS}EysQn8NUk0 zby>9A9i!}!B=L>0^unk9w_XnT(f?zI~6L?P^X zeubDH`GD%d_;;~=Izm>XR<+-5gbP~$>qJ*X3me%7Y(2QcStdg^Nh7)^D;fh~mW5Wk^1an(7o5Z_s#RCs#%9>aXy2`#kj9 z(fpgGQI2BKl-Be3W1_H)W^PeV<6oG6yx_e02a=Pqi}{ws9rjpV z@AJa=xt?^HhuPK*-c0iGq7q%ICo!T*3o8|Jw==YF4#(ARMjGsYM*BG zbxh08-J9%UNavTl$5nwo-E!8X;ztBB3UhZD$-D?3YO4XkWZh9WE{fP4k_R&3sEhuJ zQo?I7_p~~%l>0+RT!rI@Zr>=7RGKkAfc|*0SF21vF_A7fD4*q+GqGEV&Z7^m=?qtb zO=CpR88%W`vBllKR9V&yHc(|+9rM#yj`Vhb3Fq4xToF^;jRTK-j{~K0u)A+qsd`>! zw)3?BE%Q*K2^@srL;v*5!`%kshpDneZ*1d-Kafb-(O>>%sL$5(1=Wwt4u21t)$6Xe z_6dfPDv3WN5AA_&B+rn z?=5dN8?)9k4pn2nA*PL9R-GVe$L(3uy9A~B6>>hEOW9!}e_+HD6%EQf6~s>3?j z?mHM$U=l(ah^2CnUEE4rxO^~Bh~6hem+6c!qAC4ThKSyxp!baTF^>DSiQvC#6Ulh^ zq|V@`1A%@K@EMmUQU>X9Le=Gv@^i^5pfShB%rSvAC(e;u+OJM#J zBkm#0zH{fy?!Qx?jpej>8d`-= zn15$c8ghoK*xWBOMyfjq_oIswF)h|G96 zl!FTJJNNzuwhX$EWau|w;Xo!4!}BDO_hfk!$GIwL14)*ePY6DNNzVxaYK(#x&J`!8i3q_UK+M~kw)4Fw)9dY zp_>PKQJYDsyFkijZa#2u61z7bM3vmoU<AM z_6^P+c8ZeL2>Q({x5KxrldbSgO`4+LgY@6?nTHPyZ^iQ5`mUlo_{#426Un9Vt#$Tg zhbbs0639s_e<#@iCDlDJxQpbNfxaY^F}o|WOy`kjydKs)M4-q#d&aQ6vgN7 z?VV<)FB3V}v&oTLU3-O7Np}#~t%f6)U;gCJ8?pWH;)nPM#tzFv%tk`|Jt)B!qPN8w z#q^1k{vM+(#%4u8(Eltn@k9)B+{NyK#;1JzqIHJS(gdL@OT-$2DkK!$;u0@_~E6g)eN@xKEfmF zWz#8R7nrn)!-l!4)t+AhN``>E#Q*+xk``cP@_dkTio}*R)zZ-TtHnJ-e@5@z<41zl zaq%5&sxRQ`fo-5u3+IH}4c2Z_ZSI&fesRb>taZd&h5WK!qG9V^ALd3 z|4obe1DVf90tfn#94bnNs?@X$p3(BGLs8@+sE9=7Q?N{ObqBs-kK~i4*i=01d{s8xESKedzoSWF6q{{rmZS{nC&j zTCD27o&z2)cyA;?6-^LFpan1**T23k{qIqBLW!m7;FJgR;2Nt^Q=h}sHTh9BfKS}$ zHC>U!To-lC#^~6hRp4hodwXxc^QcG8Z)z?!ur?G=*55V!Y-?^CI?Ix=JD4WD*!;OW z_)yIOkHZ=@*lR?K;?H)aaR9=y>X>I-Z zp;OhD`AY?jqW3+)?Vzt4**KZL9arJ3)~O8|Tf~apWMlC37Y)YDm*(1XcXozoriS0O}Z7Q^Pu#*wce3+>FFNT4fbeF;Pzp%h=*Y0;1pZ7w6r0&>AVR?YMYf}rJ8LY=ciEDdFnK}5njfa1VS#KE`)BEPZJq?}a%)5%$592PH$^>?C+Lr;4AnRN zhsGKu2k8AxT*hg1+KU8k9hc%SYL7ss}vzXN53@o?bA2=)OTdnZ(~ z;3j}_Io|+6ru)%MJc%vrFO5~*Kagyv3{YVio~@er|CoF8aH!kG)@-QWBB zJkNi>Puu|lk;d<<|yKp`D11+qk`(P}>7rDh3a@t}?3A}0XSHd|u-Svj`y_|7bGz8$n zU8UwlpoB6XfF4%j<_e}p19cZz$Y+2ZNJMP=Zhw7;$E`YHmti$@EJ)ivl?8v2IswHu z9Y9c!AMG1Z&QH!5{rWb7W={-Hz>xQLVHSZ(u_1a$)j?*vA(NQk4R-qZyL@y;5}Bbx z#?Tc=^=+GnJh`@3!MG@#7*(X4rdB66g5e+NcMx20Pfc$#Dhq106@Dx=TcGM{{4FeO z9E|i>F-PnH9c_72k1)>unpGC9Irlws^lQA^(+0nw^gL54cuR{|S%L>29}?YPK=BrY zb-#y!Gzr%d+X#W3fq{bngxH8RWF(^wwtpsX4$OKQY|Y3Rcyz0@s8>;*O+rsc`}n={ z?ChC!jzAnW=)YE?ZTmi;LLkk!J{yG}Lbs{P1_wgL&5ykuF%TS-y^GgR>-Ta0)@PY) zX4`$@=)euLcCYwM7E7|XZcK9kI<|9l&t8lAVdIYPw<-%Nv|9`a?k_z&CCqil z_um7+Z2}Nk0W2wObk7aq)XI+shV26-gNX!X%-8r#%rK5o38A!MgmwV~^!HkCx22_Q zTcpt2z)Kze+r|Jp2W(6)$jZ57B}a3%p~P0T$X%eyhLwB0WhH5iy!I*& ze*M}|qpHP!_fW|Er`s!(<*87&w<~XC$&*6IYw-rhqoN&iT)H#beRk_F=%9=iZOS%x zC4mXQKqQPv5naIKUpTkN#+)it-PiUs{iFYJiRb6?*$2ozw@#C0Y%(6HB!_H#mnMIJ zx;;!pgt%NVAF7mI&>KbAEqrI zGu-_{H+hB7u?&gkBjDt{#89MwE@tlpO%<<(+P9K6l1^i)^QrfyekAWOra$Qu>_)#6 z?ZJhT#t??2y<>|{hkzlTrt=>8o>IGhf7gkeBie~;n`FX5Wm@6Qv_uoexRb7cZ6NF8 zzu$zJL9q4H`Hq=Hy`>so{|eoW^o8-zuXit~tE3v7D$x&pw%hdei_-85J0+U3_b1;D z&=nS(0^gddNh&mq&6)!_a*0^8c8~0bjuKl{vT68OA8Q@-9v}E3Lz0jC4oQ^F92JevQQu%I|J=FjPJ@Fy8~C|J*vCl(%)oaG}(M_U)>rk z0J2m4%oYrFlrmh6*^}w)vak25<659|cE-k3!Y78b_=%w!jV0|Gp9|ptk&UWy(5i`R7L2jV^AiVHphE z4`YU-xvmz|@O!K7$6#r(CxRJh;BkBcr|&cS=0^Ae1SH*FFN2Y^a(rIJKW}0aOPQ+P zYzP)7x|ke(G@^aTIjD88ciMlhtD@azof9ej!tU&>PmrLeFxk(>$oX1Name)68}ne^ z(T|FD7v7opM=zpBuWlI)bNi9zrkbCvaD$fqbC83-+wzHXWA%v5JGsMSxEeYBA1wc$ zfH{S{-+oeYn9X@Tb~82nxHcf|sy93E(k$q{bwKqazcHT9AJkG=w9YZNYIsOYETz8> zr4nuB7w>H%ZxwD)Gyhaf?BPekoi?M+xK==g=T}SI$2f)(rQCVu;;4bG-RnAMK=gYO znHIu-V!YItS?|F0^<>ua(uTXs=OEhc5$Rz7k*~0i4r9@*sGHsD4@NaE)l{1NAj5TQpK~;jVBcR746@NbBzWts7HYoqezeO^Aa2M;o_X|U;*5nw1waci zN(Dn;aqPwvWB;HIV;i$NgCQp}ls*3f%-dg6xQV~PP~kt&;&1o*p#$i5)iKpVzmNw# zZKShVq_Y8#`#eZhTe5+=gbW%clFk4u@atg&0Y{cv*#4C)luDXnC|}=VQ-A+6$oID| zKWp8Y-z~6UZnJ~3mPWm*UrG>&$M7^}JaK;6@jV&uz0IU4zbNz;UnBNF{;D%sN zhq>@FRMSs=q#B}7Z|F7`qS((e6U&-?UjR-iKgAhNKf%0!*u!W8b>P8R;%zs!a?}B; zU2e{YK`R4gT!!Bb;%#dNv8Ni=vZYf>j_l!rifmz{9+YGMRAP@Ahl}_8sS3v1 zX!f}9q?Kc{kaZxfI32peaD6pw+~BwRGEqMO`n?j5kls=Ht2%=Ql)p$0*>qDrZpH2M~fG-Zr=8(-)a;M zPqjA+@A{DHb@>9*dGoDvqOWi_O8iCQcSZdDcnZqb}!mk0wCXPdz$tqknbFr#ecN&gm%w*+#>je zAf375N92VwDfs*!T@e{;+?;!vWe)TmkU#Qvt~F0p<_@D_uV{fcH?@Fb}Pw zn!!rho(iGcXInbNDJKT$hgLM2@Y@<~Pu*wNV48)q9Q&{6Gm=F4@DuaE`v24_w8Zr_ zJdKAdqLb{EkUiu?kc-;dI^}56XH6;NBiO&VaZ~ZY`%<)!xC@|1be_l3456n{?q49g zWyOKbD=|V#GAo4jc}FH{I;vWY+|_28D68Z+RTVXKeX?vZn3%U%4GTtyrfw&HNF(;B zU+8n{d#0oxTlpH7@W#rCOEL9hqSx7daujA^V;XNkw`IPfij1vk$AFNOE_-^Ma{u%X1MUf3dt4~6@y{Gbvz zY0gEbX9H>=9l%}1^)Jox1ucxUHKk0gVb?zBL;BQn;?0!3+}hxoIK#OHC+M3FFfp$b z5iDiLFX)5jDcNN+(Boz@zIeNw(9k@2c4IXAQl-JLo5W4$8U_p94!mqXd3ZmN&2Qk} z@Hgm@?m~#%0>p6{lRtIJXA56iyzy$VfZffYrKhrn^#Q20s;Xgtp(t`&Qs@&P{IG^Y zjY3Dj@lNA*9}0l|(Y3uvmzj_YLo-t`&tQ&ny<7Eh$Kuv~C7-y`^^RrV|M1Y|>bjmE za?g&chz(t%LSfNTo>e#Bjpb4Mx^1^Ah!A5RU9NL}VXBiMW;J(js0A#kDqF|OBtERB z=+Q*NIhUqkxyzysEYq+Ir7*cKm z*{8hDeP(g>kIz9)1p?tgV5;Y->uaziU_8XGxjx*+O}Z`-YT_5?|6r1#K>;0b35FNF z-kcwxfhAtLWc6V6T>um?z5xXLW+fUrURB zGP;OKdOdZsV|DIa@{WiX98%0&Io$a6bL?agUv6_^g1%{AP02Zu%t314^=20l!;CvG zk1g2epdSObrXd{IHtBAFJ}r*kvt^_)w<@`9y zkHLDeRXqgO$DbBQt?%~Dly{+z6ySYacxPJ~XM8v#%PrpDh`ptmZL59@KikWD%&PAU z+O=Hc7Rg!}9jhiixpKXzcu+mgU%kD>sn5GkZa?E|IjK7Q%qY4b&7o(Azp(38sHkg(2?9%N^==V{M^{oV1j&9KO2sKGC$11=Fi}?F%{hv`0!E zw@kImpQRi%DOZtaZ^}G-F7?42T7-5!)e&;w&}b1hL@9KxxA1de+ex3CjN-T^g-V`R z{Tm;==?wdLe01}EcXLxB6AIeb5F)oC;&+|GNE*A7Lx%B z8>cDLM2D{ZkBl?+*D;UF?7uXgxNCeBM_`Z;D$XD;j#tY;KCbdzh{(pg%?LP<((z#) z+7$CKyZ*?HW?6jhr`jF1NGAWU`waZ9ltGJTCVK5IxuQuXeI#N9H{jH66?TQb>GDxU z81|KHD5P$g;|d9U1i7E(;9iuexS zugv-YEtu+1&FQGCR6Ekb!(5a=j4n3lgF2sWPVJ+^5_|>2Ew1yoXHA>6gnU_A{FUYq z`yG;&BLAApI|tDMvu8mxRu(3Lsg_2{k3rOdC3Qjaf6pmH|w*0 zxepEc@vU*e|i&WQa$;Yb|GFT2;QwL1m|IpRGSkA|b2F^#Lc?Jo{<>Wh8(5AaK z&d+I=CW+8hDZ#bj#{ubW@s4RrUCF}I@IIjmwSZFveFL>7W)eovMU_pe#LkLCKzQK^ zb`(=(JdUKEMEQ-R9x=8qeC+ixqn6$JQY+hzNu|XHP7fPQ13-T+dq-i>GcxfY4O6em zgMCJq&a#}{)K24%>-*kwJx(CSH7w%EBbX@OGl#GO$SNmva&~|$Q^OB)6kaLW6N|H8 z11$c@AO`2}kgM5X1EaXr!1lobv6>Ubq3?RxHHFKDHacf@-)XT;ylmSoD5I9heW(tp z>Nv~;g(WG}Mh*i8QjaM}ZwN!@spqSLq?IyTwZeYy5&m0lg?XQSjx>e^ctDTQ+TPD4 z9qI3w$Co@sE=(jSY6yVtd)M##@Av_AoTPRi*h`5yJD#t0qF^+Qbfo^Z z*m|@?mn0PvzIb#gqi2;3u6!0N>t=Tld$N_}4ZUnO7+~jU zmeZTculYmzI(n2s_@4`tjgSK3QNB@n>TyGV(PV6kpt?_Z+aY+WYrB<-DqhBv<~IC)CTJ zHw(iO%Xn(hfAZE@0YXMnCT1qrcYrqd&eQ7HTk2>#kPgdaOfP zXR`cJRwd#3fHttED~B6IRcStC1jxtNFiu}nZ3~f`DEzu^De$x+f<49{pABlX^pbEl zY^CjNG8a504Nxp-{*cTo%>PX+St6;#0K5QIvSx{9+7PEyT$XmZX>w0yO<*K+{3&|n z{*ejs3nK92=O^*vH0QD4Dm!@^@0weP&4J+D&}^&Yb8C?iJ*sa~B}N3!hR{9)KU~WV zMO=LVR!bB?85`q!2$jkiED2f^-D`WfdR9sJI~bHdfPa6 zIX4r;P$Cq`I<9=c0M()IVt=_Yujhn>vxZ4s+2qjoqxr|9&<>5Ki8|WCrm>i6BWl4& zAsSJ;M;a}O)E>ZmexO^tC%3PjeTdL`l=)=9qU>|72D#eBk{zL9hTLV*;uLnfmtWHRNT*|WI6|$kQX5Gz@M5bJ$WsHK! z^3OmU{epfSpKY~#p&q}W6->-Wf@#=Q{ENGax{$>b`SxmPk9bS25crt-&u>UgNd zbY%5iY@8GmT?!YAxyyw`AK#8cbIfA+@Mp+=7;df9W>#-^xNlG51*@h!9jQH>1c!sV zZ)2tI-RIhwg=AR^C#m~TN88BZ89}UQ{%-ZXH^*eGZ^J9G{pn^=yN~wmaBW^_96%MH z75g6Gk64++=J>GJwuLix(hv{q-#K9)WYglWU+qIbPOWzHbpD#b*=~8HsUE)=cOmjH zYeAUlOOT_=rxj2e4cZ#w$H?&AElo5v&2l$+50RZY?`yA5aPH=Eah|C8v{$@3K$(7i z>%76}-usK6hy~xni2Kzw@6-!5lg~tww~t?dO)Otwu8lf!xTq+<`HXe-=@?$T znP^Al8nXe_2mA51w;BE^a7c8@jQk%=T?&THI)^x!re&YOFTEQPo`h5Y= zVQAsiWvyR>>^A`!K_NRmfiz!?nfVum_>7I|7}tK3bSG2{$$pXg!UQ`ea?4r^b*j_6 zaDDd)-f)Ta-tVlgvd@ZHRpdlhH<(>tFwedmOXLMU7YeORqfG-|gJYQ9a`rXi>x#y$ zeQv}8JZ=Q!%$)|YNie^OUYiCR6*7iqI!vih$82M#z*gCB$0uW)H{$WQyveOCwI&7^ zaV_WD#_~GB`UUg62b$YM(Sw_LsiAstL)9%phw zuyqzCMwP7EmT8EAN|Y6rN7pjsNqDWhE}}ZS9p4*D1d1<70pK#9z zr>G5e*a%S`szl#_;}8dF;e=vok9b&}n{1eIet{HQqnG3fy@fZ%L_6HG#8b4S6qFX8 z`m%4ZnY<{14Am@HV2A21)Oe zYNTdjK8F5m=q#8*`EU%g4IR3|-gWKPV=%eqifV`O*B$Mhz>G=W5n@<)WyiL|s-g!N z?VFO-!u^6N?ES0}>39vR=o7(77y|6ro+<^rs&R$Na3f1KN6QT}dm%Jy)G zOVM6exyrN|nPa%Dus59FzKO4^C$fLv_yD0=l0iKDZA-<3SiCxxSuHkP=@`%L}vr%@!5M4XpZwsx6sFA zPH$T0eBiLqvnzQhrX|w*`W(xg4Txjxl5AMyc`(NHbd4cUgx-q_^!OZgqp}-Ur0GM{ zIHTjThq;q|GFur@AXznBq?JX>){Cz%57W6vIj@4q z*)nx|zHRmeG1C9@I1Vk(XwjavK2xo%%m8dqY8J!Qo0`s+R-WdP1tI!K-RaBu$?T6F z)LnO@SwLk52wXIUw=|R9Jd?#bQ{{S>jM9%C9n+^ZyehjcdRL7SJ`@!c<`qXY)w33t z1YT$mzI1>`=iK{JCN8IsIwTN&Xlq!{+f0)KTuZPdc%SJSl|sU7~)Pj1xlI z*O6i^av$cJ&Dbcld+x~&{cM|G_R2>!?St&$FYx`IaXaRg>%>xyi)3%>yAKg;wnL+v zwqK#x`POCaAuqeSV|$#;8(#Nr#QE{tteL8KeZ@FHb}HhGhQAF;Azfw1&2L3= zY3p3L9$c#Ao2SG#r<0YE1{!Wf41bL4PNAf{O-cSSusa3E4XS64%sM2q)%jTCY8Tn9 zQI15K#@@o3I%Po~LeYsy@g;+>b@_8oB?j?%k^C?F1NNP7JX&cSu}aL0F8`K z!|b{EBh=1OZ2fK}#q!6aB7}#WUd{(hi&Pa4RoA7S(A*&06XU3Gk8mFRh&W1@pt?O< zb?X~m3!ubgytf*3=wkyVaqxmDZ3kJ4rH+N~oKlASo0`(7>rVw{5Z2&nS#plFfHzUl z=>gr!sdHLPay?jMolBRPEL7jL>z&)9L)OvMIS&HL`Z_jJ$y`ZzEs6=Kb2>@)rs!OQ z??Q&GX~k@|k2!i+Uw_!_pWA+EyC?>T(q-OeyV!87a;plhLc!J=>;Sb~V%Hy=bxcW^I0)U_hgWpX;Ec!!(q?`^Ij6|n z-5#7(OM(8-1s)u5>y_O<_Qnz3te(_LaSoU}O@y&^J!?>XjYsU6UbT!(kk%f~tVl z!q-BT3bfc&(ew}|DRE#ZzG?(f1@iYqQsw{>ut@f$j!RCOS(F2DRkt3QO z@!|zjp{L5bC@^-RIwC0IQNW?0m$n}~c8))~!U&}fy_yluY`^rjy^8aL{4O!p12g9Y`o!Z- zlxsvI-E)R7Es604eSKT9wBsSb0}ShPw1lLf5KVnCuz-Lp+`M2U{n>;%%l^ zWFTEdO!`>2(Aj4q9e0OJN|&fHebGIT@czAd1UF|KMGjOakf-XnXyHS8>hon z3v=^f+Hx&#vh!vwuksvQUWZ57&SrhHch-3tUpF*CuCM~1voiG<(84y~#C@Y{(b_&V zQn~7TE=v?8PM+Quo7Nm=uishaFmm{YPwVYBV^D|IzD?x-^%8PY6^@B68MHu`y}dMB z{`R1gYS9YLCu{a#M836klZDQd_~E09jk`k5;*4h>XaSIh1d8kxVPlf4lodgOGH+=v zV=bnWQYjIjCP-Tfc(;EdAW~<3n{t0{Bq@llmpnN8EL*u^)WFupR5$E`V)seT=G|=Z z63kV1!!|kAF*&HnEfHitAT6Q5!;851W}}@-1To=Y=uCg(Y1RfuM`R~K;rKNgANe3|CV5|oopN0@G|{q z8b-TLXF{u>&S7;U#*wv-Xo!pTc-!;4p$M=CPw%-uubw6{=WFW;-D92~zN9g8HVsU+ zhw_1H-46N#kWc;7kiYEnBN_4Q(=mn>{?8PJ1`|qe>y)Kc6}5jYCJJez7+>eb8P;^p zefu?2Q*@WVG|2L@71fmlGfsv+O%#)?UF1{!&gLy+9d@eiil5*n*CzxYUWv+AH>CO_ z0q@PWShH|JM0xxBL8TD$^oJ*TlBLB84x2K?UQ=n~pG=@+j4lQ-PMOo9KxvN&$*Y}) zu<639-9Oo(DCeElIB}V}Ydre&>8-%xk0RGRlDUk5M+|~L8;9`je=ymx)c?V>iN@nH zE3+pSf7|uve$;~IPP5DkYRpY?g&+#wKsfP;Gl75+!f7WM#nsAp^u$K!8<(Yi{!LN> zxUFN~LCk1WG$?cjSu+7#PW8Q$&NeTcucjayt<@Urm!XG5Ms4(ha?~~MYNhdZS{1bC ztc7Dsw=-K%XF%C#hCMKO!jfo$Fj|ZBb|Ki)X`|m4%f-@yecI?Wu*@k8dpWvR1Oy~M zlIStO-B=e$nM3Tu)HnjE`vnMo%{|@Gf?L9Jflu(o1E;G~c|lTXh3DMq zhF_RxpSauHCvx8;tr6>FP3zF7j_4{k=AI0`x$KE3m@Zz^zBVP~dOJz%5u7_V`1S9H zwMENNfO-Gw*RyWwk72_Q90VZ)kqoT5Wx7Lb?4I&H`tg|^-!TrdB|aS`$>IEo)uj-c z4X8MJh9w8rxE(}3GrwOFyOvVZb2u@+^@Gdzrazd9_NFFBXltRGM=;po-3)>A;t($A zp&5pddBbA(V@=m((Xy9gC40pVHQI|uH>%`rF@&ym;^;?zuLww0i%h|m#{kVbb|aK= zv;p{8U*QaW;M1)R&tv+W>jDxG2oMnnkhQv*+##Dzl8YUb!=am9Q+<__4zI5oZz<6r z77~nOPI1GvIv7IS+PfvQM#kD}C)vVI@H4`o%x4Vw=Pq9Ty^ARfA;UICpfrr4h@r;8 z{_+IDf7{VDxnOk}GM3#ja++|UUo|OoR>yUt)Mc<7!(8kBiRi~k2 zoELh*<;<3H*Fek}QCIpY>(V8Xi0>2yCk4t(ItxIx-z7MGFNA)l%<-PKqiR8lAaupW z6FK8wo=}9>ZFFqUc$(mM!FvK#K#C!4odI)IN@uu~C^O&$0tFJH{p-hAnu{82lN*!_ ze*!7?dgpA$k)a2m#t60q4%q8D&~!7E0y!nSSx>OhYJrUf@{# zYYYCjL%ET?2E#UOzfl)Bz~Ijfp72)k!TvbXaN0#6wZ9&r1^lpKq*2fw!HS-W)<7fl*deE`86eY^=FL9IqQ`|Dvey;n!mhRu8 zd-I>q;9u_~6z}ZU>?138)O_UiB~{(Gk=;QYdFuOht~ig%{bSYp-_d$GlXl*w5fp=V z3p9MAQ==JCvc#Ud>x1k5m2;s&Rj(5v(fwCT6knKp?5>Q#GMJ|)7)QowM8{R%_3_X* zmAx|BYL&N&=Zlu)i{QQR5to*g&9-30ofH$?y$Tx;d~S@JhaT_@wTZoe$`_WmV>7YXI@0o@26F*?zI6324IZDIPm8Y zhyUaGOa69hrZBZ;zzNvg4NwjI@^j)lP{akAx#3?R`w0gIW=-0UPi|EUCJ=A}Gi7l- zD~=@F?a%MtL4jUMk&8eJ;+RU@pX~^ zdbUl{(5&AOO6*ay^`=ptQGzsxuKljjmN#T&CMS4n;9FwFUg9xD| z=%S3zTat8{QblRP;2Kt8A*ugLi?Vu>ED9&nSno}sPwfeVxQ<6(z2avScggr1Gcc!t z0|IIkJFe_VTzNrQ4F%>Q?m=$RNI{$_sAQ2b-Tw4EFx+Z>#fx?Ufa8g)a{D%sM zrHQWo6M~5UamD}TNdI2|K0x`W-fuvt3`>uLO+u)=XEBUNm?iTcoihU-rvKv`ID+q2 zOZ+xS{{050{u4xTiKqvZBPTt#lAb|!71@p;gj@Mr9&n;vld|62i%W>PE-8HDQraEE z_uJW@iQwRFVyeY{U-+k`1ONBZ{Vy+ig%Z2K)Poi)rnym@^+Z{z{+%1jhD#ge(7Y;d z-cH=ED`rdRHYvT>^d89RP52N|19fu?PJ@i@eCSDb6GEwnT6DOX-Yq7b(LV~&76?9G zqhewupLl*J;VvR9Nl;zE5%v%bdF0R}A!t#G-Jw^hDgjwJ4J z_g6rFZ|c{ui;ZGl>dxHT|KN!)0N- zv9#=p?iu5WC3wuoTsG7MrDm+5hWNxneG~o|wMSmxU$RebF}Fy}Gt=peIEusBO??|K@A(Z`*pyPF#=^fSyw2cf!Y@|dPF!>~f;N>U{ zCZNX@bAW~e?gJ&(K`t|CpHGPkXMe5>K705W^U)7~FqLjqMXqtZa#1I>f(gQ{8JXYc z9^3IQcXKC7f|l#}W>mH3vyShTzt8jO@G15;qVz6^C}x~Ov|HR<{28aQqb`z(;p&c5 z?-4|Zx>AOJ!DWiuQ#rXJbJVo$mIl7~U^TW_#{~7c%AMb~Rkz-N$^WXuW;+Nyc-M)p zykN?xl#r_wUQK4*c64|bzy-jo1=&z5Lzdg)M38iT;Gx*;bCUCXG2el(g~7*Ro1=vD z|1e}L=_x^{ztL1*T|zagX{%P+-0jle3mk%?lPwvQss+C3|k7?57gdU~(>sZjnejGMn ztvvacx$2^J0ZzaA+tc~WOLG1F9f=T5v@}Y@*qw3))bl?Ht@-$-Bv5H9_U6GuF@mn? z{gaQGZXPK;Socj)GsA~6H->wR$7I0*Pu?n{5W{k=Xpn+epVk$p@Yn0tSAz<#8ZQPmoO7KIbXm)-b<&~ z$2?to8@T`b1Ft+MVNsW8!yqD4;x!~%c1Cb9R7u=EkHUBBI=|GffHOHL>fFr` zMtRcQDfvTlj01=rmpnxbo|Fe=cxT4vTw=H`SzGHL(mx_8zQgO}y^xd;{hBWme=z-m zaAT@vFo|CK+d#y?I2cl>dq&<=;P)~Pv~jB0sYBjbpE7^0JkIFuN|BbR5(_SQ+#DRn z`|1wi&DRy#Lp7=fxL~?chUJW^b%>AGT5+WK{tJ)aH3;80j=jNz3;W?nf{XoDroc}4 zueKk6yuVx3N&o*tM8tmx*8gH#{GY`5e+_QPv@&>U+t6c3Z3s`pEsu#fJ+T7<>K*FE z{R#c1xQ@z0m8clGJy!$Gjo#--RVYnj%Rh}}-EycoaDM2ZDg3&}wkLQ{wd(M#j?N7EEjJ zcc%|Fe4~|^wO8ATD9eMv8o(w++Pj11qVGD+k z6v;J+{lR4TyLCK~VD_U&QBt4UM%;uvA+E%IC!{BH1%rSbaYGJPNZJnnIf*ObSCe-D z=+FLDZoH}a2}l6`&@){MyznX1SOCUa(mj~85o_3mEJ z34L-fyQun6YGhT7@52^PG(QyuJyzCrSLkwOM{Z$#m_EdZ{I$R4{5rmeSUV= zymM#TYoj-1VzRD1ZJy<;>LDA3N8f0qj@jK_x70}7Qv2>X8X=`&u(JZIc*&;FD30l} z(MGp@WkJN6hRURor=r_&t)zsMr|&D6r_;I)eV3}lhgA1iHU~zg1nQp(IrH+f$V)NK zdH)v&oveCtw|L*mq`5R(JFMj@d_t?X6OKB-D3Q~r+e9*w6q&4aCO%v6ers^mV-X7* z*?ibh0qc1eh`jzN%V1E$Sh1G;-YDz%_3Nn#asC3U4f9})#rQ&C^W4Lmr=7>qtE+AE zq2MFPGcefW*Jo}ys2-q~#H`%XU`;wCd&H7N^R^S{%{4ldp;&Zvb4>)=Vc8GL5;(r! zW66$Q%yrprc+jrM_(me&uj3K`=$R-DERxvKz zJayPBa_QUMiUSrWuQ@j9IeNs?v8L%s7H`vLV+4m@JLWiE4Goo{*_J*!w~LGWy#PPU zBl%tG0nVCTrWbFgM#D8@?zy|ln>eUVM(3Q=m|k1Brkhwu*q%<$&Yp5)&8liR3k?bo z(ehag2gLR)c4LLaU_y`-eXg;g?x`ufpy#Ce=?_&I2%&qivOE>q+4q}1Ry--SzrGP%z^gpmN*_k8cr{z5S-p{S%#+*(OH@c|f{_Z*bp^&=~kz)EXtQT3M z8km^7qR;K=jI+Qwd%F2M?_GHF$l>IH+u>c~@@Jd!pihKJ&ygu(*e7&}_p*suwpq}` zlUi@eQ%%werzIsFbzbF7SRSv_Wcrjfe!%M02Y6+l+^uYD$=xA(91d*#ZEf{0hB=P2+Ez8(i9U>7yK zCWaTefgwRuXl;cJ%3y*J(J($$K;#0+!PqBFeM&5>oNHR;3dijAsIPDyw2m68>XZH+ z(=)Mf@cP-)+iA9^V%vl?r_^ngzUot1s;NkvA;s8ArMpPiW^@<*^$tbd_-3EL3myw^ zHXmA@{z7T#y%yRn+p~C);AYzq0}l$p_7!T&laq+vv^B;)0{ zIIV8Dqf*Gv$rX&>>i6FQ+h7Vw|M9F1A*&F!Z*Z~M^2_?N)r#`knYsPiBA@m~58l_9 zoi%+Tc-6~IYmy$8b@RQ!r7NLM#}^x2y!(ywxZTgsIw)m5eNpL*brYevpd1pJmd91Q zVir7(F5TWeo(^?8m0LqfmK^3rHaDP!C_GODvCr(0N6KCv4oj8Se=Quy^x`U?`06dT z`^`dH+Qtt|BEpC5C%tq|)~Zy;rEjF>zD(%zH}&wOr7Y)S7zh19@q5b1K=XF$3Bn*& znvN6uvQ#iRc4uL@=FM2#^n*Fe>JN5kQ$NRLm06=VvaLCuo-+f#j)(cH#G*~-t)rMm znIuNFDP)uN`bI1wotQ64k18twc_$g{HXje#eKd(4l0=K0MHh{f&?87RyG@9Z$7Bq) zCyE${og;U3&Rud?1#L;Db+{eO&2gEy2q^We zVVAeG@|Hz7Bj;l$xBE|i509+72PuNkV~M>lFkQbYu$i`=8r899C4b#ihb`!gXZ1$!a28I`9Jw!a)7`P8TIEdR`z5!B$9-EAH%X!dHOfGu z5{#?EabIC^4)WHi&zVO|t`CUcIFN4%k=9ranfKRnyo8rT>-edudC%Kf?!B5&nQBp9 zPQ12;emJa{;OM@nQ{vQB3%5X^S4<)PW4OIDLvo#{y`E~hAG1&7Th@&?evIOlS6Mz? z5TNSXW~lPvJt4AC2AZp@l<$dAs5?>OAa!I zyl_J7uzIzC-NiPhvS*eKZ31^WsXb*Lw8{}aOLM`sQ7N2_6gkFn&*xFEgh%zP*j?~f zc}&p}&Vpru9QVhwj3j#tU$<+@)OH6KW?V86a! zzOmgcu$_=3rt!#B^BQ|qhY<=LT~VJ`$FuCNW|Ls2g;~P7x7-fRn4%>Jl+qUS<;wqbj-UUGLKpa67~jZbAU$- zkZ6H%(4bShml}GNrZox)%9eeiKm$~XN3iF*T*tk%^P|4+7VOYsh~VT|Zw;V6u|&xX zZ~5>M2si~1 zR7m7=hN|TiEEToww*<8-Rz6W@5_Z0^ceeXx_HN_Oq|Np7F2kqKTA>yw1>}YW6-q^| zuYHgw*Hb|RmLY7Z*wpE4@}O?iLAKUy4OusEP{=RFT$w$}?B#23;i(;^i!cUTd=Qx`_o*4NL!V46Xo#T-Z>D+=J2v|S>^YkVlN0hxMWsAH5^Kr68d z;uku&WVAG0m1 zEvj>5PGjWc+^9%Kh-awCt=ay0ySH0=E*AIKJbfuq=pOKCzoF%JZs8p9$(7+g=XkDAjp49CM85L52x}W8c zzx23J>KcFjkRC4WJA-f9i=b3Yd+GAtX2#psGs;3wq@=f_&)E#^QlK+_utxOBfrllo zY)H4N#T^-8_|u~rl4jkWYS!}0miWXe{nM!+ycYExI~4^vt0Tp7Pe-8sY_olRPC#v) zarL$P51i8C;!XQw;MTJJ*155o(^7X{si>CD8ihRa=ns8_BJ!zUAIsUADL2Szaid*U zcc<#ubSWBe=&jUsu2{>1iB{#7dTc#uyW_vyL5|z&60UQcQb_3iTxW1=8BpgUI6d~- z)@e5w_t^&$uis^YO{HZL;G~Fo<8>Hmt zz?N0oamUgtX22u7%7j&p^2WXX!NX8+AlW^fvllrL7PEaXA5vo03UqCD?iEjdcKdaD zkWS!L)Nc;fD36;m&4EtAw-2A$GwlT`g2@4Sl}?(|o0^7(Gf1MjVuzF#SPva9k< zN=3P2W0uQYJ~3)q^kA0;sHeF?>y*Yv?9CED3?5X9h(Cl%&f&z5LAS!AZJ z6vYwUh?-S@Fx?s6kR&;~`f8eyd7U^z2eyB7hSXq^qckNI@J8?H<>#;ITqIU+q$)K|XeC^<4{~vd69uMW) z{tu5-Dx^fVmVKcD-)pYQMYEU(w|dOd&ihpTI@>paivIFI8vkLCR~o#MQ-DkYn#bE#EEuQ`6Y zEq?UK(mln43d{!kO_ObS%6si}Q`)ssKneXa{2TOWF({Fsiu9~tj!^E`sR(B1BKKzA zdwM!i^n9(thejXb3D@YML+X3qsaOCal@T5aj~7`=j)$Wr`c9>v402NJdV;h_gKv*| zp0L(+oB9*Seah-hJRC?kqqrg-*x^~n*~j|N3Uyr&s;}BtSGK<9^HvdF87GHcG>miZ zdzyiQ5|F-f1_8Lj%rKNpmV2o9RbAI7(% z5H`twmd0Wu>e)6N6S0^!_*(Z~QgLEwb&S{fBGQ+8M3I!9{WV&PwJmRo&GH64-^49) zpfwVj;YU#HWFhq)6;IWG1%JZF1r+vu!}m2Av*yV;iDv2ivqS3QEpA(d2i#H zYK{)f)x4nJO-Im0^ekdwICD|ltbq=3{IWq??G~`SgzYKH&2ZEz#%^}qcyDH%j zsv2Ge3Lwl71A)bRGdMyXXbjf##+p61h8!Q5B$(%S`yH54`SvK2DhIg!sHzUvKh4@) z=~d zlxiprS-i|ig~}*gPMlO@a6h#LDu3!#Me8dPgRu-rj$$H%@kh&|)0cdw)Nb5LUq7C* znoPt5@u`8+fEZOH&AmsQ4}TfHqN~-@KhGZ*$al{s`GQV(fJ54y7zV+R1u91>B}%>5 zCaBT!IGWX-sKQR}$%Q=;RE zTZx0T%lSm;$PdWSb1Z*^E6`>5O?396yGIR({@hwcUl>m$V+J^A+Sss8_{u2|T8Xbt znl^_=RUJV9P)#>W_CDmooETQ~E3N6aNwoCAo}T7Yd$mYDUbEBA_N=)9xt9(s7}}ve zM@6+r9ZHKxGfgY95-$CoX!C+%I!LftGw?lCUeSxGe}92-Bwb49*r?4>gNa2ufk)OX z?@HK?`Dj8;R;`0naxnEaX{)Kk=_Vz)BY;5R`@DOJxk9fYM)Nhv!MsT1*e$6hqTh$Z zmbD6XrzMIwQKFtiRVSiWsPLG4`WHU8@Q#& z##7*aUM`_L&pJ@cwa!FdO7R&(pe~Y{sr(gFb&Cf~5sAeOkHx=rAcy%=2>nn20p;Q2 zORWn$X!rE!r_JmOQYb-E-zGZvKK0`AY$>0JvEKQ~Z?9WUYF6I4cvwUX!Exl-cWxdCr<4Y#|PvIZW+!y5Q|Si6JxjTl@zfAa3A$2O-_xw4PuciX2C)wP&?bI z?}ReuazkRMRPCK}x3pDBko4#jar3HmrU$<3C4^xijwfn)_2 zy;DTK2M!C1Zy)E?APvU7{Ov{{T~>(f1?r&aO$Ghh(1idsjSrlLU%ZPap+zpZFfD&G zqh7H&7s-RuMiD9R_?M?bD0;s$>H73$aK1R|uE@&RRuqr$s zXJ}pZOh=;6Si`VLBqt{eBH}wV5h4^*PBBVMFI^isPL|KhwXF!KP0^DkXzP{lDfQ6k zKcNtFP>S=Z#NLCN&U01|jqPt;=a#Z78`Lavo%OHp#Kgefp*m>cEkRyV6BO?P7T|<1r8ZE&?|K2yx6EJI!AKu z6^hlK5*0hUS7tMXoof$e-n2UlNM#PFksETFZO*-1#}eLZkp*WuSCkp3oR1|xZV<;Z zVI|G;lGAHYdh|5=yjS_W<(@G~|4qyjZ&j)>1?QY~!O@z$ceK81aPZ{2_@VDz226qH zy{}Gk`I@YjT_N``Y0-!K+lLqXDkeUc^l#JJD!H?%F zP1wnzG#z{>ZjRmS2jn$hrPJbiD4GC&F=Yc?h{M%OwqU|ArWZaoO0J}1=a#7yH8usn zgmYv(x0Bv|R7dSJ#R4T1$*bSYtXf=6lsaks-2>(KP}MR-M$r_Cb=WlAwq@LhE1 zoqufkrXTN|rpxI&9vrg!)+^_C6X&eCW}6vOODTHfqmmG8{eJ3gJIB{8wqhB{E0YSj z_f4!cc*1=7I~_t9gVrblM$ilzV}Dm=Wi;VQLC`Bgbu1EzTcMiUrW z?(hr_UcpnVB~h*kmRjd2QRH37UGA@STUMs}QR4Y6>77~TcLwlt?`1s~Qlw|-MHDya zm(*w66oEeoYJo)#5eYv%2U;o&DwRmi$XflpEUZE;g6tm=^C3-wIbGw>gH zofft6_cO?s&l8RDCJkp=czrbj=l3adagR4d-(q<-=HDWeVU{!&6()hKf@if8)tB9S z4I-PCU0){AX{RuM%oT|<)>7thb-Pt~b~pDU83RYBwu`;%X#;j%8F?GLtKNuf%QwQ4 z)}vCmi}2;*a&JbzJie)CrqR{Zx2l?(POVJW? z4rP5@a4NJ|*K@H1uE*cZjEi}9xxZmPqEyP3q`gt_m8;_*Lpc>_)fhdZA)Y2s5k-qt z^$10AQ~X^V$A$W-NcJugs&PJ9wyU5uN#*@y;pGoCBQie9Pn)CXELBM(uZS%BW>P?9 zVOp<#9#Z^mVGCW^Alt`}TUT^A+Wl(U7r)@fS(LCSC|+)&)oo6~?6S91#Ejec%3dAM^(YeEa!~47AkM2f^PDy`Y%_Zv zNbeuG5!vNDC|8~19!7{yBYR$>xV)Mf@W$LrJa2z!NHtbhKzUNi=Su7Dh-kb>Dzr=w zVTWN+6Z$^FSVYoHAnG!jAw(}0hR&$+oqP2D^&AryHk6^+E7_yb;vK~*&klMhgItr8 zH<%9lDm8s*K=V90j)NAHK)~1Q0ETD{UX`&XxcJBx65?X9Lt#}Iy zEwU=MWMe!sn&Wl&?Q>EB!kmj9J^Nk z+W6k;3@Duzicp|15caFF52M79Y=glPShfBKfr(p9Hd{LCI7IX22SwTo3MbhUsjU4>7kFq1yd@MDM^up+{6skAuKb=_SLQA^ZLT z%Mj;>VY@80i$j|Q$QKuj8rTX0_}w5@v1Urm9~f=AWVNd#^9|A|vFcT2eaAm1%Ef&v zuTm_RZjLi5ARUN=K%AtR=MiJoAjOB`@xGt?4hF*c|fjU$=;eQ|cpLGB2Dem^7(0EKI1*>CttYEKz>w zQ35|u_Bb~0;A2z*s-RTl+0ZkSQ&Bf6zj~ zQ?6U2itJH=d}@ecD1`>`iNb^1$=uMxgFLZmKOlGa-R*r5*e&!@P^|QH{IbV!h&;90 z+n(S+qjUGekH-g*dRW*7!@Le|AdSY0@DdZJ?7F$U`yuu3TX;?AA$|t;L>ZsAIz3~n z8LpnF9vfdPDOLE^3ELCVnzm0fEwH8?<~ZLb2P>@JkTV@Mes9%?Ib3Clx=BoC)T}J+ z#@Cu&HVxlJI4m?cOt8CA$tqv>{BZJfz&)=j+E>GY7}-I*k&Qayp7hbI{rbKISF;Z! z0Xvbveb;pa@9|@j`1zs2*k{z+E^pj}K+tf(1;c4u!QLM!{x+?IyfVmejVQ zmcH3#JFh9akwvRva@MjJ2Gl5^1DX);a}e-(0R9ySrHvS$lr==_A1#!B<$w_D@4rf^ z>J;IZb@WJ7>l^uS+l_CpcFULvjxyBl;JrBii>XrFW^BFZ&xb!8jhl_o56ao}_}!R)W_Bg6K?$or)*w`N25ES~+Uy9WvS5 zcz{Lqp&}ME9@-mhNTLfRf45bqvXs>n1;$5ZK-f|;^CUr^=_SjdZa_S)R%Ez+TA04Q zMjG8BLI$VQtY68-CY+V`&r3Gj{`s&KdU&MqHr+VGRtA#n9ZWMP6SsdrSZFNU~YX%bHwM>kcC60)3X{xD9HNz9jp3@)*vaO*V z**#wnSAg#f7JtBDa8fq8jU!S!aoj&&|Js64O{R*6Q!R1aD1EwitHY!yt|TH6-)e!; z?2liwifR)v%Fs_IE@YXKV?8^ji=k(S?JLq(4d`eRly%x6WN>MT!b2U;vB4)>s_mbj zJgr*m(LBcBnh-i6_r&M;G&GSXSJX>MRQGy&ObcV% z!yB?;Q*p+0it1)D5akOntGQdXa>h4A=lo$hz(Dg z)YCHXxLIR|_L%+nIqB=n@aNXNs8+UDoMmUrC6JE8650q-9{a_G_?c-@Wj?eg*??f+ zvjS(gSCi}kFcn>c5@BvCjsw-7@M8Q+Zz6>r?tbufsI1k0C|t($9oaglIVvEG;B}q% zS`~Nq_0S=+xrDjgNgsN7tF20z<%3T8no4b>{@mCwDu3>GG%MBMRC(5nW8*QCD}xz1 zru(3uF0g59i`}h_4rzmM`0TlWB&8AdjD}JL3D_gGLPd&5Q*A|~{ktV9# zGZ7Y5>9Pn<8XrZN4q}QVsnKu_G(HynUH1kbZy(j!^EHL#LJjG2Sr^i(G;(3gd^>f+ z@M~4{5sqg`GAB4OF@M%wLqojGu;gmY3=YosFVVdnrPMfQhD* zM9We%B6HZT+pOI6X2uy6f-5?%^t6cY89GSeEj;!MN{-BzKm_KcquHVePSF;BRif+G zI~+eOe(kN$!wQ~{yBEgYTnEkXqj#yCW2r~8P!)q3kJdq1;98!%wY}z5DVN{ab|=i= zVv;!)-FNHOA)RH>TZPcahEq5k(#w=OT|e*8=DFSZYVOJ>>D==&4@(z%*LmmX2C~=O zrixS6`&IQ_d0@c`que#37eL8{VL{KrQC{W|-7;>cxss@frX0>aOwQ#UgfnL{1wx7d z3%L;4HE5iXk&)jnf7b5@WPTobV2E??%Mr5;DV>wq(u$lzpV({ykDFw)kfs_d+`?B! zIVolhNBf=KQy6bxjR=jIhg)p8A_Pktrv3f&@k7!cLY!}TZN}K&nsNlpSMAR;NJUt( z$jBvc9WRsSiLD@Zo;QZKKQOo*1F3Cfk!!YF2u=OZSo@+yUMWo4;Iu z6>vN#IqYW;x0Nqz4p&QvnKCR`FES~a9v5mD4cR;em>?)=dXa+-+)d?tu^ASfoZ1tm zNdgwSyPf2@*p8*%Vr1u)xv7^mW4WKQOknnG#1pTDT_WjPgjhYfz<-YYtJB-mdC9Uf zMp8{(9=lVfEGeSd6iof0`h54-c2zeWybfmU&3}4w--F{kcP^G*KXB5xv#l~-Fr-E9 z7WcpOz1Wmbyj0>(3 zB6T*Iu5K5a&ZeB~en6{dAng`g;lWtusPkKa=?_JCdd z#`Cmj>YV(i0CN#Co}bZT2+l+`WQd@iBWj~G?%xIa>iuE)l;+mMyAH22#tnTHC_TV( z->7bkoMZoh6;!pY$Co?-6o1a9qdr!S8_wy|IZs`$P811Rd(<}1AzKsC&i9wZ83+lD z&f~}++#$K7!Tm*xN9^M_t=penZ@l}}g+CS3Az6Rg^3ZY6is6LFCc_&>xg;vd+<48( zlAeSQM~|*QJ*T2(Gf4etYGcP4cjD~6SNoKM1wak8se!NMiLeje>?AR7;gZ9YtA=4$ zLY?1~+nEjzXm{i1Z_&jo2djizaSCj(gdk-*52G%VuV-dc+BCZISK5N_zb@Kw>K0i0 zrCvBBx*A36h$;)8Zy+@+N!15uN7frYu`d^ru$bV57beJ84VYect;w@Xv&js|NVqfG zm%OhrYuRD{O-VaG-3$iL=0t_!p2tN!YE=~H+*#Ujl0IO^Wqq!(mhHgaD{x4)LbLGr z!vKMp>#hENZq9i64A(aRYn;0@xEQx@03^L5r5<-m?>`YSrZcW{^UjMqu~Sj)>O6L< zD~)b9uw4&XCtoqRu9z*BUR@gMQ7KD|F1Q}seZP3RBxnXc_ZCLz`~pJw?RZ+a+zQfu zqiqe;Ud6zy04Rq7eDe&xyO@e;#4X7*{D4HlUu@$(;FeC+q5WV_T?nBUhK%r?(CfUm zYoJSD1whhl8l|j2yb$96)m)jTGxayT@-op6FdL-*0HzknL`gXRS#F;Tmt5x&gwh;x zS3!A9+NtD1XQKdeEXv^*yw(AZ*Mm8+NM*5A*z&&*O8#-YqXkE)i#yB zRQ8GoXZm_7eSgip?dRUYltMy&6#2k%V^5=4$}w_t@FN}>wmWCU9?kk5JMh74lJRpz z;wB;!z-r9NfiC?;(6_%OqIL#Asn@DQAEd-BE&j&vhRvl?Mt2~*T~-f|{eiV3((HfN z#se-m+Gos?$s%mFAO8cGk)-O`RYro%+;Aq4koz=|O1$8E535iqwuhoV{N^s=Evk@tULxYY6k zGBpcO0<39T0icyT3egZlP_abtr`iL$!dC8V63v0K^p4 zW*U7aD7n+3UVMw80f@A{AmvKlR|tSINH~%(dP!?)y_kIlQ(QW?OilpVxBBv$6I7O? zHPTU+uAa)c?cyLbkXC9QpF+pfHW>i*I>V)T;{g}r_-J7uZXC*vlIo$_ZD}DYf_xsh ztj8dfzpmJgcL|h?7Dyd^t^0w;&#XOqU*|)ao9G7pR>g9UQLJd`25oO=?a+E`(pS;7 zNl-Jd?0srMcoyzri!N~Ut#j8=xt^;bbRRr#i4sufS0VifP8N75`dc@OM|IGr+I_iPT2W zam)n&c?F!Fe8AG~T>1N>7^vsPK%Edul6nA2jh-CcNwQmlH|-?Z8?n*bTlkqE@D`t= z7)HrrJH5;MN(q-b@Q#RU^Vv&L4{WiyQF9(-m2kAJ9c-P3 zrb+35QG1YK*tNh|z#B9L>Z{mSVYB(5g6G^}HD%DY2Z~hG3$}1$7~##&W|fS*=Z-4R zx-T3)s2oa*E|RLVe;$%Wcve_fnBK@~-n6jBN{rhgMoJ=H|XP`e_ zyhv5);eOo0n%b0anUGwX*j>c!A&>9Z!jaY< z$~P^yQJrm2d(nl$EjG;NG4v96*)aGh|N4V{1xh|#fCO^@svC?TTFr`fLg5g)OMzy< zLer*PWplxT%Es|fgYARx4e%)!z&hOlA(snLw5vc9zK)^i;d!5Cx}RYpebtjX7;~!Pax!#^=JHm7-yK;b$aDo^5TDAoM;wLY=5)aZD#3jwx>iOHjOZfrW)%`x)2~}zEpS*8C<)a}UZXx`jTmkf7GwFJ zUOs5JI8p<2THtBq-=@y4uze8rYa%cL|J~8$ouU84UxAYMm-*9taZBz%Y1Ji{zpj~u zy*K2iX+f!(KOpNGLR2Xo@VNc|TNBUB;}h?FdvFQ4I*Kj%Q)~CQ56TibDzul-0eo^~%i^sYR8Nv+ zZg#uNE6Rh;I+iyBihD(`*A!KH`g1n#y{;iX>kGf>iEoB7JkqmoTWj2Y?NT|cMi|xa zo2xonl3;XkdQYui2if|twkBRGo^CC+=;=WjxK+Pq-By$#52#f?YOgutbrx&a6XITSV?rS@-E@aDNRzmrga;DOuR zlQ0Yl8aK2`WlcUeuc=T{G+{-5Gy!;Npb79*{@aiHBXO5DasL}SlFy|6EMXDAUbK8s|Czy}kR-T7nIZ-W8O`Nl1vN#owqQR_wLjJkm?bby;=Wu{4XDJIlIbWwn0)IsNNTqq5%9@a4mY zaV*ZwX+O=OFn&674%i`qeCN@vOK8yffdZeB$&C!7`EeV7xFS#&;U-_;yS(`j7^W(A z$soB6hpBQz`2%`085K|7D|7(fZNmWBRKwGtFYtj-uxK6hHbcVt^#&&0{c4?J3Zpz= zWEJFGYO$b3+0M7K@oAn`oh4aq-dDAqUCEdR6byXNuDHv3vq6kJd)CuaSwd=5fOh|h zIyt&SA%}IPM3bELDN#gZkhgqQ-@U@5@sUim%-vG23x=xJI`12{OsKO|@|QItD7m#z z7Epf|oU)f36~e_3LKWMLwLO=A`LgNSU`yrT!Ld`@ot6hXtD8-M30ixGm&=nR2F*=m zPA3L5Cg*4GaPbfVO(K=(F6F4ZFwxN!&`sq*yppu?xYCZ@^I@^yfgcQ2>eiuL zfq0U2HI7{Y{UUdP{=7=S{nam0y@j9GYGB^A-l(=8ucDYKE!dG^Cw92#NzUGA?LEP} zeP?|TWAJ~M5S%Tk4OTmT=JV~4oYzD7*zagn_r|)7Kp2p;Qgj?VeJYXL4GZhMr!wh3_Yq&I_P?W|S zOZ$?NNy7c-Vk5H0X+-6hvf=*6-bzEqr~1%4N|^{7__IPrU_u4ICZzPY^|TkAA#U+f z_5bd5{-+-bI`#gG&d~zi&M%FH{+sd#j^RHY?f);}~+ia?|_ClISc@SpW1#|3ibt5-(>g1_$GV$_ayvlCzVjw86xk z*#n=OANse2`sszw{!MAZe#>Im-(6dMTI=YN)?3`{B=B-Hxi=NCJyl6#nZ7RQAhlhk zq^5E)8H;+^o};DrY(GYqG95KmTqRW7<6e}-9zIYW@VIE0@NK~BbLSl2p}M|TapPJ2 z5gD8sa$nOKGT8W-p#kne-2p4 z(>1bWz84P`vfd~wD>p^%^U*MTVo9ecR4xZ(>>;Xpytb2}5YbOH9Ml^mTw^8d(5U-O2_qQOY+_WCR++{5`n)tCptEgJ%O0ZConp&td&-C9ac6OlY%Z^yEP_Yap@`DeP(p6XKG5VY zd|3TT)e1fPt59M3$z+&@WR4>HimN|Cl7*6aImn#Ullh!`~58dY)_7DT-M$v7j(`z@^h? zb)&_muS(qIRD>h{VLxl#JMRoq2=3_&$3h{S3L`N!7TgT;#-y!e`i93^jj9qV&_cw! zACNmUjas|Kx5JV4^O2fcvMn=2MOLjXLyB8{xEM{2Vr53La!k&1&N1`K)Vas7J3sr5 zP4UGAZnr1In6%vxmdtHEUIa%yjh_VijaRZ<78&j1D>_~{$*_NZNgg_PoYz%{%NqHi z|1*RQG(8lR$s8!9I1(^lV%4}qVc@|Pw zU1BB0Uhg47?ZkzmX>Kv;na!sUYUl6A+~n_EPv4i8eSDXe6gZ#Y=b4GXzhV>enFowc z$6IIkBq~_ViSI2$_1(l%%Ho&y(nGQY<{_K9BUPSFFsK?Uh5s6e4>?~a?{@P(uE>~f z_r4{2Cemw9Tw>eV7NQ_zRTU8pE2O<50CeHl=7G`h!^@Qh#R*%CI?dMd10IofCN+a$ zA-;yYIVY%_}vXWPk>&FJB2e-NNCqWI$H;gHSJW9C%=TkxW zlGexPygA=LJTRjsS#hY`@=5cX8!z4nZNHc(IXgQLT|@8g-&h_?6l$=rY3Q!GFkNvK zb8z~i3#1v?_xp}1xpLn^we~o^+&&J(xPc-xUJ6c1fh8;%R2XRzGd^plZFjwC;H(Os zMt$qUb?jSAhc2s=KH^nRkvPve4lB)q(9ZdF>ER&{ZMJqtsZTNGP>;dz*bWFUgx+Kz zc3PcrQAZbF%-0720N>rfF*01#0G|XKvVsT$=5`M=StC2{6C-I**O8A{B=z}$=6tnG zUR}DRBhOZpOEku4D8|Dv+ciz zeNH>6HR;_qy0-xK`8!Pl-Kz)RPV`Yr;%Z{CTmf{%1E37RCXvB(e=7Zme>>-EpM%dl zyM$#SUc%Gue0`W5({RbMIt#FW^>z100@jbH$HMdKdl>p7l?*{n-c&VfGZaEeAa+ty zIOeKg)4v#vGXYFLL%50sIcQ8FxJvm=^+M9P!C}i0`&iMFVF-^$8@EkvecHe0i)8Em zsSFn=D~;7Xwh7MX&5SbYS#aR*so`zI*1TftJUMhp&H^^~bUWFYi!*m=Yxqo6M002% z*_n7Wn>0m`tV8d)F-;mdM`pQE-YX#({laqicwx*pm0J&QJcCSyUt#Ftwe6HWSad|4 zp_(C4N&~_xyqo>lL(cQRzB%2zNQQtwv~eYPFlcQl-(GU+PuXfiqTJlAtyP@9_T251a)7)hY@IQ-jV}%%)`@-8w@x+nr}u7=H9_Z>08Yb*}=t z7}HA-;j~#mf_3msqHyXStTg2|2~!W<4;mw^#%;>ose1YLYR4rjLEvCI{>^$j0ODqMC`Csp-P|zkBw#}#JH{V)cRtMI3u-48L%d4o;S#i#9U#Yl zfZuq5sO*10Ve^kfuC`H{8!O;`vRqXfZw0Uj`jhVhE~wqTmF9QNMXi{KCiVV+97Ioz zUIAVrP~qeO$mW?XL%1S{mK#tNVM~jEcltSS%;%v$@ANbS^vshmnlfPFBSe{Q^wPLJ z$2+N8zprXhWx)^gU;lYrD|lR}YVH(_Vh8A&;LHE^xbnst5UlpL0ukRLx)qDPHIIN2 zO{KWt=Qg9ZH=;oHrW>TaV5F()KleSOTd`(e&iwfz?8Z7^Wd4Mvy2jGD(c^e94%qL} zXFRmDc**ho7FCVMYPZ5NQM5 z3SVy~$skx_pR~O_6>Z+jk#XZnW26I@!CKP#Raqu5By_7vox<&=+-jy2ZT3uAOChn5tK!{ z1kFj6$wMi;ENFTlCyFeZmiTrzu5T`U?W6w|o_YVOq2+q2dsID5tmhVS})po3hdgIBC?`!Pw4MH#FXk5z0p&*5@rVku$gyNl6# z5$kjp6ET59-`9;Cl!Fo-?rBeP%@p!yF9_4G@DJ47*^V?r&&`jgfrJuw8MRpVmE|rR zlqO6$v7Cx{kJ9da_Hpzceva59Cx+f7PQ%m;`O-cI3stUTMJfE`T%ynuHLhvcLHwm2 zDqlPY$D+imM{v_}_eF`xaz5j9Njs+de&%~UL+JQVsW_@b^QNlZ^{cmZP$#=r?NAEo zGmC`IL+3kr?7VsXEbfs7HHNHS-0#;YJ-hg@8=H-dXU5ld9(l7EzPT7$BTyB5)XYMM zP~2n2z)ur&l|ms6p6*+1Ibse2&SvbWX|bX+hR8I;Ulfkv;sUhzrpyR#|?1*t8v)>_m87pzVNJ zH`~&GrHYS@?xR3;Em&$z1>^lQg>*d`-REC1aCUr*Gda5$I<1PBGN9Nk2EqP)qf~_b z^%!CL?{&g!*q0zOo=M}ip{f4kb@;!%?t29@3Y@8#&yi-aU;mP}UxWH72!~ELaYnws8_GZt|-%V{p4gaM)loAntHv4=Q z$Wi=FdEll-|8}Rm|45DVukG=28}mp1ELQCy+tfo3KUkPs;mo)4lr?cyvG-?sC{;X| z2w7RQ-{{n*R1$P?1Er=E4#DEou1AaeZX(DXtOg}Kxna|J$z0SKiNf8!mtqa9IzO8m)8(&^tUSUYd2Y{C+#-mtJ!o5JoryA7QTuYTR43WyR0<(8^AwfZyQE-IzfqRo!aTIntIAY<`vOtpD7@fdJh{ zK^TSesWcSD&vd_h?9+_wuNh5+Y#D0OVHxl++o)ac=o{7Dl6i{}?N55no%)z2Vk%QK zuXgL>Sz;(;9R(7dJKtdkhyBlYu!R-vsMG$x7W@iB_Y=eU-^%&#AI$dGI(CTvB@C%7 zufNLyEh)!PpO3Tot>2=4O6b323vfGgY5WuspcsKd`YpF0!1-He{q^21hVEx39lvZv z<=}5!58?_t3Kg^BrS+G9f1k1YOEA%$C4YqHk9hs-#sL<9iNCWy`kmMQlDc1$03xCG z*L%TyQ~Kh5EBloGf7*l{q5eJMRF*XGyym8C-~r+e0L-z3AeCDnwE@)tP9TV|?~L=$ z{ElHo`fKFeVMY}EI7{%AdRN(MhV$gT$x21Gf(qQp9dTlNIJ^(+E==+e+b8(&pbiB3 z5MwX;Td4o1?ft##&o}&=-DFq;`oHhD+1%N0hqa1PQ@_RpBmdvEhyR)ln7c4G`L7N8 z$C~~!ZeYj#*;tTG>Cj(mj`#d)$$xEhXIj*soeJ^d)sy?Z)3FhD|WZj7}H9r6xY4G*j z4`y#^Q5dxQ7FKd|J(uO4^_-u$6Q2M5OYW@=|L{;C_B--c{KsSYpFNxZAAHH5!soK3 z_DA^WR&IlW>)*om$JYFE4*&RUPClwXAVz^pM1zXb-h8!YH!=Np9apqZKF!{@wV$r! zigZQes2*9!q9K)5?ONX(e<@{DfLey7nQs!n>j}s zEhfe8+C-`(2eZ~>|I}hm=!t*rfS==ZE&c5>`kX~FeNV&(!R%S~+j{fkZOHPu$T9H; zW_jn`dyc5!xocA9j5ul9X}cXFrINGQj^1H}zE?$zy`V`_?jQ%G+XznszuU%-uvK&i z3#YgZWa9kB4keKb?`jDtcI*`$=$!wD6ac^VHozA9twsO7G5>8?YGJAP4h*Redaz;} z^T%H9+ywHuoqwe2f8huJ<)s7j{>ztF>XiRWt^V?<|EakL{y)^*%J~0WmDZoD0$h;4 zglcE~)?U|m7HwyEL*W+nHq14zCe4+a$^*|RNPXyB1Q?3@R{&(5M?xKr5hY2AkIMZ@ zJ-r26^IE{7J-pTsB~|cw-4I>N2W~Ovqb6-0-SP?Y$?d6D-c@sACvkVWVmq0mk=LXF z^h`*+iP&CEFfP|XzSbLR0KAGMK_k+x|kePBQOwl~u^jS`swNvId-F&K%O&=gMH zy_O;C2Dqsd;ORVMS_G8k!@ z6P<3|fUN-qL^u_6rZQARTf;$v>ii|!MGc5YX(LO>0IUwl0vanKo7SduBjQdacHcn@RPNE6 zS>i{1dU(KpF;Zq|+O1$l38k?7RWGxut*q8@Vk5hguBtRzCeqz%u|osZyJdrJsiK6@ zD!u?qf6ds2aG55TKuqvgIsKZFzpUL;b58GJrxd}gu>3$~U`_j=e z&JjT5aD2eu zgDluWufbC=h{GEr*S*oAuESqeI=NP{0k_EOQE#>rt7M3!WaiN|!t9-XGoLFw!Ehr_ zrvi!L%)>r(ki(%XKsPy+&KnX>4_?QeWERzbdEztm#0ypGKlzi-QGHl{^$mWt<1B~ z0ukRfH|y67iRPdjaCQK?&D6y(h{^B}s_W!LiBqCrAyv*K!pMG!gsYf~pXGKJ9ny1i zPE)>6{6>VEAY6JTq+S~Gi7o2+b)k7lviIm1yq1NXss*dUy(>O5{Z3@jGvd|IOj=Zyef#iT?90vb#z@fZ z%w`f&jD4c?Y!exVQa$##@#zzA;=->z3Ud}u%inKZ7Q!u^#w9@CbZ!N=Q)E~K4TB!6 zQY1^(vw9@jgxtSX6Ew+m<+5+0JU@kaKP~drv&h#z%B|FV3Q#|SJPo+DLm*Dj9f-Sz zSgIrhQJ*1MLAB_4;!d^-GXB4}0oYPjiXO2+S}f8+zFm`+_B$`3AjKO<*lSQuKT=it z0duqn3!P0I@y)Q^j7X>PCg&kjrjKhA%DA?_oR&!t+C6d3-0hpqXRllWYF?$}i5rCA`leNF1G;oCpwVN5Oc4HSF zyt%-{crty$(4?)2P2{B0%_Q^zh}uk!+xnWRMMP6(wtjqnd~{Vo+ld3wP5xq^4Adbl z*rzB4GEXB-l#Hosn;kyy?m&g7S`N(|4OREe7dO75%S}987V`zY$_JQCJPGBXuo1FKF}f4cgKVh!SKw@O5mSlcgHD#FvDxRa$RgVK-Rw;;%2c zI%)yqT1q`X$Hpq%9xgm{!$(n;uGL+c076?-Bw4Z<%~XWYjddYy6n)2HIfcFoU<`eR z#H7wRH4-j!(OqGC$0c;!-DZCEmAz@yTnhnu#P&?lK&kJ8qD3}Shnc)Ii)LpTmW+y4 zdw{BJ0Lp0&eIMQXjr;==kwM%TVes^z>x$jRd=<-e|Zn!c;Ap zAkGbnB>RtZ$aBAB9T>Ve>Z*OuDCWePwE#CHs(y1Jfso9gN0tvcTi#pjY5QP4gIB=6MOPRSP-iLFbgF@u z$rj6{K5@_@knz4eo_yk>S()SH`CXUhZeG}>ZGoG&fo#U1prpw~lU_Aus%yV2ucm{@ z56ES*WQgR66!wVcs>M}Kdm6HcOeVX1c%YB`+Yz=(xCT8QBZ@-asC(1a)u?7y1QLx9 zIe$F7Iymw4i7QD}jSnn_UsgUoJ*WVrJ%DFZsau^# zjTaPV`d}?5ijIWe%9ppOD1B5N9o)=DIe|!r&l>l%9**5SxFj6Kq6cOc*3PE9EKl~ z;}0})Uw2B+65vx;F-brqm3k1gCI00J~ z!Ulu1+27YP_xi)lyXaQZLigG>XF$+DK5uVKw~Iqfr2y+w9-Egi-DyBwazHpO5h5Bu zNhZzK!;aQK5#XGLvZ5>RuF|p-Osp zKHkzd&4q%i_It?t$(_FP*0eo2WwgNl(EEw_8<&CvV(yv@ce1a1c3*GSjS6>#u{_H^`jayZetX^0X#1%4By@Asu0i*~Ix*5a zw~{{DE#HFm%eW`=DLpg%pPU+o4r%c-NdvmF7h4y1dSLs?en8?&MmOaU%P}}pl<~NO zP?M=_#le#NwODBx5&61Hy!Du@b`KM(3>wsNZ>)m*KLn++bX2GnZ-P2Prg0LEb4#$3 zwp&$_Dic(Hu)h>WIkZBzf!C_`4n@~1P$i*DvyPqHDyl*|kD#rULVy1M*YMR;lkb_} zH+59v+|P^X`m-oW5U$dVqV9VPq0WFc-+2VMsOssdBdfrZ|gX-jA5DB~ksSPhRFE56ul8+JMF^JHXW z>QzNerlbK|>*Jyj&ARf%=xY+R!}VT778c6ndc$VbCMM`aJfqX-ox>Fd4K?<8SuGF$ zkMh1e9_s)5cSMvWS+X-_iIQYX7=~n-Buk}?t!$AkN(RG7*0F`86v~n$v{=Gq?4pug z#yZw4!%W0G#(eIJ&*$@99>4o|{O-Mv$M4?zxc6V)-rlcs&g-1#>%7i+9+=G8NoFC$ zz(V@=su@Ou+lvAjkiQqTBRGhMHc3v%4iDvq+8d2~e}%ZNx1fv5>Guh~=VoRp=7gVT zDk^C2CRBg;n)Lb_ao^YK5~J92j8Ar;h=Qrj;vhHD{Zh7LBi{4yy6!u>OuZ^><#xTd%ksRJ>>N`jpR?`39rWCC`@hx zau3nXPTx4V*g~y`N^EMw^sik$+oe~0#jySwAt7K_=kMohCxuRGIm7)+Pe2;f^adO= zL&Du*A3~HbPC?DY1koDOJpH8ryROdNH~qNHRd#@x9VU^jyBE6L27lk$kR(dy1gN12 z8C`KFW{h2NDb=G<)QLf z_L|l>-Ti5f1Wp(R7G|^Z_@8wEOdy%Ls6 z!3YoBvF0$WOKUlydu?;<&oIIW@di%&KR9{530!!s_<%IEyd?n_OpeQDYgt?4olA^q z^E!IWhhwj$fy>c*!oqABGvixNumV{;w1DsuH3iFAQ(D4jd)rv=GYS0D-FniVKbk%$ zy+E5zSFAX^M{b7!TGBgP&^V^wz2g*5P&{_rzb-sON1}^JhIcEhriS1+fz4Uj41Yy%=J8b@2u!DvAcl}F%{L@5|wxeAX{I6iA3(vfrwjuwg29npd-*Myb ze^|U7)xkUb3n|3O3B~pF-vdAX-8wXI*A=6b-rO$#(Ejw^TGTUcf4Y1F0nB|r0L{II zKCk%=^dV5k<9x2^)D%3L*+e)_m}!zUWOZLke=(6wC*BzrvOkr#=_JID>3MCUjiVc~q zrIq(F*ytA2F-i#e!;24HWf0)${hD{Mc8k?oz47HSC+XU36+u5HCqACEONVPmd(^bG z&^=!MnG!Sd3mi~x)+ojn#s8-LdCD0VY2|nK^~Co39kux^Z)(^bg0u+NDzbS=I`TYC zzg61%hI^*_nRH7>_#ju z+#Isb*NPRIWXV#xg!>Ee>{tv?AQ-Li7Yxd(_zP(PT=uugz~~FA{{Mo4=6wG`K>(xl zrvp&Z%k;a?WZRDZ6*~PRrtdtxZALq2?oW`o1D!w+Ba)-(8^hastdnilP9_xpC;XTS zdAJR~M0QN^pTG;e#LnGg^}=>U48=|U3DI``&m_a{uN_fDed+x?CD{EcKA<9yC(R}P z$ZhGX%7At_G1o77p8M>U=s$MZ4iMczp#NN{R~Pa6S5bW`5l=YeF%javh;@9_7F$>5~B8)xp+a;reOSByj9Ihlx=&yk79W5T#DWhlmz zlKgG-chVw2#sI=PEpAI{l?Y_SsfGS)$>V1A-K#!lcAb$a?DgvWi*iEp4yiv>pVw4>+h2( zfpttq1~3@gocjO$K>dFpIBHgR5BzoJ`0od}*~{dAwe+2s`cEtU%hZ{m0EQu#3S;{& zp|d%(I>cl4GuSdT81sUsa6r%j?>;->7L^PJpxtF`6TdRuw)idXZlj30P0TLwfliM)J+pH*Aq-#p&dw0 ztcS}4w+I+f*yeZ)i<_)ziS#6VUt&#q{=LTN+BLDa0`Vy-h_Aw=pE}0vGqt7bxk7-y zXxF5++m_q15m^`hNB`D_s)Zr$EKDP}6nz#AL$aT>e~{_MbPHtIXHvr09d5{Nb3xV` zKyjNI3u}OA`ZZ{k{kG9ej6gWt^7UeiDK3$6?JnOsb&5BnMCkYCMLATIe^He_f_ESl=qQfIG z-_fw+4F0qZT!Ryt=MR_?czo~n<3$ykjL27C`bn<2UV%po9q)s0HP>xpN3|FWKjXr7 zhscYpY}c-pi1e8EDR>pwx@ufc5Oa2RR$=abl_Z2QrmEHOqJA8A`~x=ek~uej?bYRt ztjzp|<7y@!iX~l167B+O$Mg>FYT_K1ke6ccDejqN=s!e4d;riK&=r^M$LC&uIkyq{ zmhh$;KLayXHju2|rAco$p8V*XBn0U*?)~#&nFC2q6I{#}o({QJXO<6tS9m9!Jj7hs zkm8F92DSWi??vs5E|ix)f}J(dl)Acl%8?7^x#ceZ6oB2<23dyF-*2)JBqe6Q&|6D5 zJ1=h>w|h+GwMWV+VMkWXGmh*?@tx;CHy~HV-h%W0BO>&w|84K>ElWP%7j^~#`yFe@ z$#wzzeAA?zujj*TbE>`p1<3rVfIlUA$6kuc$URlwQ*L5Y7$Yrx!Ei};{$^g59J6zm zwP5X`PSrNX*TNfC{i#<^w!JVOVyhLphX;s!Z+ZmohlUeyBuY5iMW(J5Bm45R%a%Q_ z6(T9Wdae3rg;G!l_Rki4F&67Lq93*gYJ4`5#9#p#_GWW76l8fxi_7GEpQJ~E^1S?q zG<Pwe<+g>73Lw#*fuzp{A7j2;_2oZBLu$6re~~eG?iQ z0M~)?su7XG$OP;FwP!7hIsjfSp#L1h-Jo-%y6Z4XT0*j>Yo=$_X}Uxg<^AP?#;pcQ zO#Cm&cMQ4EjttQC{Y}IU*U(0r$q5yHb@~uBD=kw~tCGFeXDw~Ybawd6(%$`r3A(M8 zdk+dCv^UK#whf8sg<-o``0~c2UzK z5DMc+z~F9DA7ok*n={uoJw&o@deg}J>$e3=ST-dn1tpzo2M+@5XqI!HGPpF2WtNNYj;6X47F3X@;&}je=iwqa?S#$1d^s2?m_oM53E%+|dZ+1Vvz_ zG4x1}S9!wGEzSUe^kix3*=WdKefJg6Q{zq=v2n052zEJYYcnj_CieEt?b|om-@W;C z@ZNnRiwmoz;g{EppjfKkPrD0iXTS|x#Av^VpVBuN zQ{6_YEv>&2^eFff;|YN|cJmuw^UDj?5~8CNWP7L7l^>vL+xG0TmP~m3M4I(y?_1&V zu%Ltkv=Jg@Wm9fRU}G6PS!y&^*1xEDyCy>FRI+16?;NPYlT}+5g`-uihD5f6Nb^n9zk)+L)N6x)?+=&g?IMg{cgT8 z>ojL0-U1s0z(}4@B%Kd3C`A7l*F2hmtY|VkmY5rBDyIQOa{Cdip|HC+ET|DoWSXcz&7Gkv(K)WMZ<+d4`8o`GiJ#?_|;|p z<`=5&w@D!l=MTB-$-&)E4f9WG-}LD4_Zi|?LwEYyjBuc#0`jJU(fKSBk4G}ZSZBQ3 zmAsgFGtSq0qLJ0{OA-`4D>6JR!d_$-K%~xz#Y_z6a`3lXH_^ zylEy%uV@LP=saMWhojH_?y=u00URaV91B6wQ3N9zQn#nv?r5NbcX2^&tV}rON?vkx z>izb*aPHGb&Gv0mE6T=|55YMgr-?iF3g=0AZ9X0zyI0MQ^%&ob!S6Fs8fVglp9$r^ z4T^=6X|3RntA5V&5R^a3rPA>2RV=#qhTsvVNBU*P7kq_DcDLq|?SX^kdb=r&1AMwb zh1Yge7=No!ew(R6j9B!I4J&wvradC%>-Ts`L&XhzDrglko}Al>8+* zTuYcvj_TzHbaiN!8a&CezBa}|aC(7xVW;F2oZqu#d5gW>`1B^K&anWlZyOK;bm=n) z?m#IZ*-W>fsq1zB3OVi?KHopFA^lL;oSfYIG>R=86cuoP6%QKd(1cThHx)5Y&^>ew zV4iqQ*5|KoRPvzFHu^Ze&xn-|iKn{s+OGTYGxcUHGo~EZgSi`iw08B8wyuA;k82H&JFlXLL5E{q@*J;0~ zP7}WunV~xD^gdqY|T0ppB z3?AeVo6#3ymQ+8goU0{P8M$!s|K6{9^}gQp0!PDGIlLJeYcJRRI#e9XN<-KSf2h8z zSYFDjT^s%ejR-UG-JQ3%VoNz~ebG~4QR5l+`{|TZPJu73Cbw{?DAk#3NT;^!#u^nd z21K%(xMM8n$G09}wFR4OK58!U50w9Qb;D}p>k)*V)RUuDmaLU|4snGCU^ngL2SQat zJ8A`?^_T8ESl#~eScF1>bhbP7SNpHrF;CyHTx~P4EHhQ69M#Mt{#FB#52P+&tOiRWJZ2y9 z#Jx5==lxwcZg&Th^yJ~PBTD`06A?52-CE%wYG zXweB+;a3>;wfb3;Om)kjGmkj@IlIsFFHT(f&dU?UvHzx~Gjo3xIKM!?xbfWVcQ9L9 zs9}R-v7VdSNc!%t6Tu;=0u8*t9@#4W19qL5P%96xpDZ~$_PCne^-n3YZK??ivZy|W*L^3BL9tRv+ z@$zzsnYzQ3Jj}jQ3SpKP{lSH0`51$51Dp~McCrmy=WT$C#>-F_b#go@@*1tO>Faq@ zdBYDM61xs3cFfb_yK^-a)7<&r!7d3+y<`@Wm1bVS(GdY_pz``TX&E5EEs%h`@5a0& z;&9{VSj3o!ju_ULS~jW*IlV#^xvi}$qY>TKBi1Lfq>WsDE=@$8-c_Ahf9L5quU=~h zoJw2%z5p)KL5m0(S%Uis97*G^oZ739!=`g+2T_<{!*(}R43jZ=v46?wp>w#Y?8Wz% z_e*AVy!u{SaPGV5Ap4A^FB?BrzP`9lHEPzcci=;ybkbf6Nmc)J>MoaW{^@dzhw$0N zI(MpY@{W;jcbLw@oLX?78=V;1Y74fo7syUUFhJ_(liDh-373H6V(>x$H8FJhqr}gGMRln0#%|pTVKm8&ndHMoY$HN(>7Z`IdB8#+KO|^{R5`*50MUZIqcC<$ErXy z@(}R=R=&@sB){ zxW?iopQ`XR&*C@_>i5&@Lc-GQu#FT@=RC1axlZL9H;eTm9HN;uU6>OoAIWhr;19CT z4J!7LPB~LLV?AUl^tox$-K$YpiiK|)x1$h7vsrN#gx{Do>q^eb_$t&jSmPzQ`wSx@}7DRD{}@qOth-4B+(JUf0;+pM*6RBJ8uaD z8-NBv_n3r^05-T=@z+=+no#{Q@$spO4=FbCpLySDT?mzdkMHe5ESxUePA%b*;)1|CFbfm-_~6^D;HBG)+f5bjaboQL~Tm z+!!~VZK5ba->xq{oW#~uZgc4tznA=s%@gAZTS=wA1T?2}BD61iS17Ybf2c52II(v% z!I1IYu2C$=H5V(wOA&HhyVG; zR4cJ(8VWNk$1DjzXNP82S z$z)}NV)&cyD4JHYK!#ZL>%!fcgm~xbsuaULIii?*?UTN+ch4^|B<1fyL*q!2$pkCW zB^P@PioHf%XyEc-WC!faT-)+CNug97R5?+Jmy+ld<5sfqoeh(M)HY@4a*#%m$)sxH zax+7V0A??i@uKah&HgNR+aallR9?f4_2N&g$fb>tiqR%*?0mocmA5 ziCGv3vB$)GV1ESziFMZ9wqS%e3?2!ZM03J;A11nOfCk2at&Q9YcTJ10FvX9cJ9K9p zC~5n*W#&vq-Kn6NH$Qg)R8=|QwLmkiyd4RBL!jGwtS(((g{xHTmAFvQ0nc!qaPF zen?f_>f4Lcc{Y!HOrAuv__0J8$!5(E*cd|0YoIGM=R#g6$gk4cH@U%AV9XlE22;ls z>$SZXnsDq`K_~LZ^J~K-Q|*^Gmzgy`IcYtPQkTrGvw|n#0Dv6Ko=A#Qr2B)@M>Oe9 zW$ApVMHS&GYQ|)$<;@F93EO9pqludA8%Cw#a?iMv7l~K5*(J>g_srizHhImyUXq_h zBh!!m2si>KEE;-{bdG}ilFPJ8)ca0`JmfvlR3U-|6^r5W&zhYc6`r2-IMUH=5EOkD zfSV*7ISlFKupy&qG!4#EVx_@Up9XjoksHgQ*ofQ>g$|*pgoT@>1r4HYF4fRi7T{;8ng#`@R|EpxUG3Hn%-X@>S56pWU?RPKISMDT%7Np+ zUZQSQ#;O*X@XAg)^5)Mrnao-;A7cDi#F8|<4i;F-0x&LYD*^Z8!5sVnk?vRf zst2q8Y(c#`F*9#|zy0tWZ;YZ;ltFUwjLSZ$bayQWcdoLP@V!D($NLC+&ws!j8AHVI zx*O_8u6thhk2mp^laU0BUCU(@an!i;L>R)1T)J4WsYREdDbf$aISCxs$Xjlw*Cyt_ zBOgzr5*AjpIK?<=7FeULRNQrBT$XHlOExc-tzcnenAXZSGQ1bjl6a_`pK|j=?%n1P zp3%f!Cfe-wdcD91_5x#iKIlAoyDb3NO0iOBaAO(1Rv3L_fccvYr;lO!{*`0By|IA*`OOEweULH(N*H75xMa=F@F1 zr+ki>erP)5b&#HW_bWzrM@R#V1Al+!qNW`47m$Jw1%1(GXR1NHVd=ghjK?`bmHI6` zj}KBEpOPQ%n`o7>FtYEHjbQAjyVmg{x)cHI^YZ;tQR4EacS-I~&3+^Hn}6HLLJ zljyzkj3pmbdciDK^Cs}HBW@d@;8|!QarMqKP?LT{ zhpWpa!|M3bhW~pN6|of5xfC<$jPiUiqVLx8F@sk>Y7cb3R=g_Rs{JvZ=WIkmUQrIK zFiRr$S<1Fo_1jnuM+ih) z`!o3f$3w*!-TR{e9@bh-U5;KZvmn=u9PbY~WO*v`>HS=rJrnf_h@6vNrnzn{tgM=s zWx&kIdt~*ky_&KN4lg1!5~Nrz{*pPX6Dv!0V9)D<91P!gb&uaWIyltoEmjJ9lW_S| zTZ_lO1+rPg=Fnpw9|Szt%=acmIt5iQR8#Zzvx?I8`m$csjDqq@o&u~gbDL0$Tl8WV^-)kN-(H!JkWgK-=a$1|RiVHiIXG_p_0p4R-mhje*aMXU7NwT|%ik_3d0Xx`lSVDVq(F zx9qv?{+z~t58q~4w zxwW4tsjAN^Du-lZr&8Ng7q~$>h;y~(lfSr11KJ>eVJ&xf2w#y89!)(Tp_y0N%kb3H(W3a-!acMqwG^t=Cet+e_G$IF;;tRGB(F7+ln7 zdb z$vMEfR8>DaEG8+oZwlVecK!JWtZLfHb~>OxX&fIn+l=-k+L5woh? ztdM>~scM#B=b~yr6}!yIJ8x|s@0xXU5-IUbjbHuh;y^zExxX4UExeJ%+pTzf=?dY! z={10_HRiv3iAV`f@%S)jq{`G_4Zx98fxZ4qcgFrMig$6Q3;xP$g>K_1{VF?Q^8;6P z4<_mr7vmZv_-I$cFu4YA`*&9?%h}r)`A#plP_i5=hR3gq-_(}R0r6;gdx;Jkytn^a z;yfAb56Y@OxhCXzzxKi0GoJk{0;4vweu284BtJi;d#N@$z(a6@W80rrRQ&hus$7M$ zK=lG4LpIt7X4 zvi|`NadzqE&4dZybpTj0kll7097dkqw?VF zT<~QPymZkmcfmF(ntqF%%{RX>5M5S7gq~o6u5tZ(`m%FapQv*&N}VZQWw!!<(Sw`& zy3f9WiTm?sVnEHWg7|wBaNSHHKh{C#jvOGv>IL*4{@@FB9y|r04;0M;zG^WG5Z+*} z_}#LqurdpP0YEYEwmoc5bI(Z!dH1YUA{sP5%^Aoj2EmSKW`I9h%fV~^faL(3n7h$= zDtAG5xA+{OtOepIRsg9H&5R?vVf>b>IbnRG=?ez@fOsV@e_408>;)50iQM;djFL}F zx(sn`Dq|KVE=TVhxzW+DWvBh&uvX3Q+%mj*@%?j$9n{@f#2+0T=uKNur^Hb`ETPR! zHE5Q6C+a1sw$Lr*``&$h9yJ=d)sgD(ni*9+Weii*q$*R|9faQINvtGQ@NTUbb)2W4uJy;>(jp%}EKQ zf#~Q+gg%BLb*D#Q+7(5JNzUEncYx=l(4@I@;Nu63<)ssBKAYV13p9AMS=b#usC${h z0JTx%XG>Zf$bq}uw8P9`J3hqKL&zF-1wnG;{&^}vbXBj|Kn z&)?`WfWglm*;X#9x4UbbNYw&pk{_54y#H=991W8LB@l}3MeGY&A~&8|M2WtFz6BDTs52VF`uBZ?mN z+u9u{m3EJOxagT|G=dLg3HWXZq7^6)-Ve`PQAgj8q*wDEyywXLR_e5k`(a&Uz4g{c zI47SoEDV zMb7$5YC*8!`fKo(TVU@UWyAIPA_K77le!Vm@6oCGOAh8bgFlJlG-)P$L&J;%>s8rZVVBkhp35ja~bZXBGK` zY1%LzJIx$+a}*90Z<^A*sIKp6spAHtv1?lJ{kC8nbkr|Vzrv7$xpl+-iQpRYVW=S0 z10~Y(oJW_%*hZ~<6NQPIBo|OrHgzxyCbzs17YZFP$V=?3$0>@F)?U2lXFpY$DXY-6 zp5AutkVxI+ht}&EM`)lnZDoQo8(qr>Ox1H{QiT4b3bmoRk^%h`OlaaP$s%FpzXA4>vyw)x~%N{(X%jb6$7~!i3JO~ z94NeRo;GO4(zGxR-ZEN!F`M0F&hlU^U$(T@^7=hM%e5SY7A=@Iq9(m$+=gWaS+35V zJ)#9R_ebtJx;yu8`VS@pu?uGUzj6oWk@+yq#Ik#Z8$C7V%)Tb3ZEYL)lrYiwuCqsS z3qfs4x8qZHZifayoMj0zIAAzc)hvylNk)ARweXAo@%WYj%=Nol(8;vGmp3%M;!jp; zzK&BEPCjUubx}_!<@7V}n|MLrSvTfWyPgzbaQXv`)~(8GyP=*E-k-x}BA+Cy;`gUK z^9)<~K=ICfIa5CLzwN8FMjuAa{kEV>%_B2mvi1)?YFgsY1JAdtPeqF8GjN(P3nTuX z&MmiJ8%cAz`*Mp9&WDyQ^}>cNd;I)DbMl_CPO4hs-M|lFWseF(^arTVnDhCgiM_b9 z-hp%j7|L|RGHA=-Fa~L%yKg=Jd3EST+@6i3SzS&{_a)N2&@NFHXBXpxD?bd4J|YXw zTA)V5gpyk&Ch+`K!{3G4=29Vc+<%F>{*Ntu??Ard`3E_vz^}APC0-gl%68# i. This prevents the flow of information from the future towards the past. | + +- inputs: List of the following tensors: + - query: Query tensor of shape (batch_size, Tq, dim). + - value: Value tensor of shape (batch_size, Tv, dim). + - key: Optional key tensor of shape (batch_size, Tv, dim). If not given, will use value for both key and value, which is the most common case. +- output: + - Attention outputs of shape (batch_size, Tq, dim). + - (Optional) Attention scores after masking and softmax with shape (batch_size, Tq, Tv). + ## 3.特征重要度学习组件 - SENet diff --git a/docs/source/models/aitm.md b/docs/source/models/aitm.md new file mode 100644 index 000000000..a15ea0489 --- /dev/null +++ b/docs/source/models/aitm.md @@ -0,0 +1,118 @@ +# AITM + +### 简介 + +在推荐场景里，用户的转化链路往往有多个中间步骤（曝光->点击->转化），AITM是一种多任务模型框架，充分利用了链路上各个节点的样本，提升模型对后端节点转化率的预估。 + +![AITM](../../images/models/aitm.jpg) + +1. (a) Expert-Bottom pattern。如 [MMoE](mmoe.md) +1. (b) Probability-Transfer pattern。如 [ESMM](esmm.md) +1. (c) Adaptive Information Transfer Multi-task (AITM) framework. + +两个特点： + +1. 使用Attention机制来融合多个目标对应的特征表征； +1. 引入了行为校正的辅助损失函数。 + +### 配置说明 + +```protobuf +model_config { + model_name: "AITM" + model_class: "MultiTaskModel" + feature_groups { + group_name: "all" + feature_names: "user_id" + feature_names: "cms_segid" + ... + feature_names: "tag_brand_list" + wide_deep: DEEP + } + backbone { + blocks { + name: "mlp" + inputs { + feature_group_name: "all" + } + keras_layer { + class_name: 'MLP' + mlp { + hidden_units: [512, 256] + } + } + } + } + model_params { + task_towers { + tower_name: "ctr" + label_name: "clk" + loss_type: CLASSIFICATION + metrics_set: { + auc {} + } + dnn { + hidden_units: [256, 128] + } + use_ait_module: true + weight: 1.0 + } + task_towers { + tower_name: "cvr" + label_name: "buy" + losses { + loss_type: CLASSIFICATION + } + losses { + loss_type: ORDER_CALIBRATE_LOSS + } + metrics_set: { + auc {} + } + dnn { + hidden_units: [256, 128] + } + relation_tower_names: ["ctr"] + use_ait_module: true + ait_project_dim: 128 + weight: 1.0 + } + l2_regularization: 1e-6 + } + embedding_regularization: 5e-6 +} +``` + +- model_name: 任意自定义字符串，仅有注释作用 + +- model_class: 'MultiTaskModel', 不需要修改, 通过组件化方式搭建的多目标排序模型都叫这个名字 + +- feature_groups: 配置一组特征。 + +- backbone: 通过组件化的方式搭建的主干网络，[参考文档](../component/backbone.md) + + - blocks: 由多个`组件块`组成的一个有向无环图（DAG），框架负责按照DAG的拓扑排序执行个`组件块`关联的代码逻辑，构建TF Graph的一个子图 + - name/inputs: 每个`block`有一个唯一的名字（name），并且有一个或多个输入(inputs)和输出 + - keras_layer: 加载由`class_name`指定的自定义或系统内置的keras layer，执行一段代码逻辑；[参考文档](../component/backbone.md#keraslayer) + - mlp: MLP模型的参数，详见[参考文档](../component/component.md#id1) + +- model_params: AITM相关的参数 + + - task_towers 根据任务数配置task_towers + - tower_name + - dnn deep part的参数配置 + - hidden_units: dnn每一层的channel数目，即神经元的数目 + - use_ait_module: if true 使用`AITM`模型；否则，使用[DBMTL](dbmtl.md)模型 + - ait_project_dim: 每个tower对应的表征向量的维度，一般设为最后一个隐藏的维度即可 + - 默认为二分类任务，即num_class默认为1，weight默认为1.0，loss_type默认为CLASSIFICATION，metrics_set为auc + - loss_type: ORDER_CALIBRATE_LOSS 使用目标依赖关系校正预测结果的辅助损失函数，详见原始论文 + - 注：label_fields需与task_towers一一对齐。 + - embedding_regularization: 对embedding部分加regularization，防止overfit + +### 示例Config + +- [AITM_demo.config](https://github.com/alibaba/EasyRec/blob/master/samples/model_config/aitm_on_taobao.config) + +### 参考论文 + +[AITM: Modeling the Sequential Dependence among Audience Multi-step Conversions with Multi-task Learning in Targeted Display Advertising](https://arxiv.org/pdf/2105.08489.pdf) diff --git a/docs/source/models/loss.md b/docs/source/models/loss.md index 1fd13e6ab..e5d81bae8 100644 --- a/docs/source/models/loss.md +++ b/docs/source/models/loss.md @@ -19,6 +19,7 @@ EasyRec支持两种损失函数配置方式：1）使用单个损失函数；2 | PAIRWISE_LOGISTIC_LOSS | pair粒度的logistic loss, 支持自定义pair分组 | | JRC_LOSS | 二分类 + listwise ranking loss | | F1_REWEIGHTED_LOSS | 可以调整二分类召回率和准确率相对权重的损失函数，可有效对抗正负样本不平衡问题 | +| ORDER_CALIBRATE_LOSS | 使用目标依赖关系校正预测结果的辅助损失函数，详见[AITM](aitm.md)模型 | - 说明：SOFTMAX_CROSS_ENTROPY_WITH_NEGATIVE_MINING - 支持参数配置，升级为 [support vector guided softmax loss](https://128.84.21.199/abs/1812.11317) ， @@ -71,9 +72,9 @@ EasyRec支持两种损失函数配置方式：1）使用单个损失函数；2 - f1_beta_square: 大于1的值会导致模型更关注recall，小于1的值会导致模型更关注precision - F1 分数，又称平衡F分数（balanced F Score），它被定义为精确率和召回率的调和平均数。 - - ![f1 score](../images/other/f1_score.svg) + - ![f1 score](../../images/other/f1_score.svg) - 更一般的，我们定义 F_beta 分数为: - - ![f_beta score](../images/other/f_beta_score.svg) + - ![f_beta score](../../images/other/f_beta_score.svg) - f1_beta_square 即为 上述公式中的 beta 系数的平方。 - PAIRWISE_FOCAL_LOSS 的参数配置 @@ -159,3 +160,4 @@ EasyRec支持两种损失函数配置方式：1）使用单个损失函数；2 - 《 Multi-Task Learning Using Uncertainty to Weigh Losses for Scene Geometry and Semantics 》 - 《 [Reasonable Effectiveness of Random Weighting: A Litmus Test for Multi-Task Learning](https://arxiv.org/abs/2111.10603) 》 +- [AITM: Modeling the Sequential Dependence among Audience Multi-step Conversions with Multi-task Learning in Targeted Display Advertising](https://arxiv.org/pdf/2105.08489.pdf) diff --git a/docs/source/models/multi_target.rst b/docs/source/models/multi_target.rst index 9012aca9b..2263a27c0 100644 --- a/docs/source/models/multi_target.rst +++ b/docs/source/models/multi_target.rst @@ -7,5 +7,6 @@ esmm mmoe dbmtl + aitm ple simple_multi_task diff --git a/easy_rec/python/layers/keras/__init__.py b/easy_rec/python/layers/keras/__init__.py index 0e59090ce..f029b9c66 100644 --- a/easy_rec/python/layers/keras/__init__.py +++ b/easy_rec/python/layers/keras/__init__.py @@ -1,3 +1,4 @@ +from .attention import Attention from .auxiliary_loss import AuxiliaryLoss from .blocks import MLP from .blocks import Gate diff --git a/easy_rec/python/layers/keras/attention.py b/easy_rec/python/layers/keras/attention.py new file mode 100644 index 000000000..d7f717cb5 --- /dev/null +++ b/easy_rec/python/layers/keras/attention.py @@ -0,0 +1,268 @@ +# -*- encoding:utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. +"""Attention layers that can be used in sequence DNN/CNN models. + +This file follows the terminology of https://arxiv.org/abs/1706.03762 Figure 2. +Attention is formed by three tensors: Query, Key and Value. +""" +import tensorflow as tf +from tensorflow.python.keras.layers import Layer + + +class Attention(Layer): + """Dot-product attention layer, a.k.a. Luong-style attention. + + Inputs are a list with 2 or 3 elements: + 1. A `query` tensor of shape `(batch_size, Tq, dim)`. + 2. A `value` tensor of shape `(batch_size, Tv, dim)`. + 3. A optional `key` tensor of shape `(batch_size, Tv, dim)`. If none + supplied, `value` will be used as a `key`. + + The calculation follows the steps: + 1. Calculate attention scores using `query` and `key` with shape + `(batch_size, Tq, Tv)`. + 2. Use scores to calculate a softmax distribution with shape + `(batch_size, Tq, Tv)`. + 3. Use the softmax distribution to create a linear combination of `value` + with shape `(batch_size, Tq, dim)`. + + Args: + use_scale: If `True`, will create a scalar variable to scale the + attention scores. + dropout: Float between 0 and 1. Fraction of the units to drop for the + attention scores. Defaults to `0.0`. + seed: A Python integer to use as random seed in case of `dropout`. + score_mode: Function to use to compute attention scores, one of + `{"dot", "concat"}`. `"dot"` refers to the dot product between the + query and key vectors. `"concat"` refers to the hyperbolic tangent + of the concatenation of the `query` and `key` vectors. + + Call Args: + inputs: List of the following tensors: + - `query`: Query tensor of shape `(batch_size, Tq, dim)`. + - `value`: Value tensor of shape `(batch_size, Tv, dim)`. + - `key`: Optional key tensor of shape `(batch_size, Tv, dim)`. If + not given, will use `value` for both `key` and `value`, which is + the most common case. + mask: List of the following tensors: + - `query_mask`: A boolean mask tensor of shape `(batch_size, Tq)`. + If given, the output will be zero at the positions where + `mask==False`. + - `value_mask`: A boolean mask tensor of shape `(batch_size, Tv)`. + If given, will apply the mask such that values at positions + where `mask==False` do not contribute to the result. + return_attention_scores: bool, it `True`, returns the attention scores + (after masking and softmax) as an additional output argument. + training: Python boolean indicating whether the layer should behave in + training mode (adding dropout) or in inference mode (no dropout). + use_causal_mask: Boolean. Set to `True` for decoder self-attention. Adds + a mask such that position `i` cannot attend to positions `j > i`. + This prevents the flow of information from the future towards the + past. Defaults to `False`. + + Output: + Attention outputs of shape `(batch_size, Tq, dim)`. + (Optional) Attention scores after masking and softmax with shape + `(batch_size, Tq, Tv)`. + """ + + def __init__(self, params, name='attention', reuse=None, **kwargs): + super(Attention, self).__init__(name=name, **kwargs) + self.use_scale = params.get_or_default('use_scale', False) + self.scale_by_dim = params.get_or_default('scale_by_dim', False) + self.score_mode = params.get_or_default('score_mode', 'dot') + if self.score_mode not in ['dot', 'concat']: + raise ValueError('Invalid value for argument score_mode. ' + "Expected one of {'dot', 'concat'}. " + 'Received: score_mode=%s' % self.score_mode) + self.dropout = params.get_or_default('dropout', 0.0) + self.seed = params.get_or_default('seed', None) + self.scale = None + self.concat_score_weight = None + self.return_attention_scores = params.get_or_default( + 'return_attention_scores', False) + self.use_causal_mask = params.get_or_default('use_causal_mask', False) + + def build(self, input_shape): + self._validate_inputs(input_shape) + if self.use_scale: + self.scale = self.add_weight( + name='scale', + shape=(), + initializer='ones', + dtype=self.dtype, + trainable=True, + ) + if self.score_mode == 'concat': + self.concat_score_weight = self.add_weight( + name='concat_score_weight', + shape=(), + initializer='ones', + dtype=self.dtype, + trainable=True, + ) + self.built = True + + def _calculate_scores(self, query, key): + """Calculates attention scores as a query-key dot product. + + Args: + query: Query tensor of shape `(batch_size, Tq, dim)`. + key: Key tensor of shape `(batch_size, Tv, dim)`. + + Returns: + Tensor of shape `(batch_size, Tq, Tv)`. + """ + if self.score_mode == 'dot': + scores = tf.matmul(query, tf.transpose(key, [0, 2, 1])) + if self.scale is not None: + scores *= self.scale + elif self.scale_by_dim: + dk = tf.cast(tf.shape(key)[-1], tf.float32) + scores /= tf.math.sqrt(dk) + elif self.score_mode == 'concat': + # Reshape tensors to enable broadcasting. + # Reshape into [batch_size, Tq, 1, dim]. + q_reshaped = tf.expand_dims(query, axis=-2) + # Reshape into [batch_size, 1, Tv, dim]. + k_reshaped = tf.expand_dims(key, axis=-3) + if self.scale is not None: + scores = self.concat_score_weight * tf.reduce_sum( + tf.tanh(self.scale * (q_reshaped + k_reshaped)), axis=-1) + else: + scores = self.concat_score_weight * tf.reduce_sum( + tf.tanh(q_reshaped + k_reshaped), axis=-1) + return scores + + def _apply_scores(self, scores, value, scores_mask=None, training=False): + """Applies attention scores to the given value tensor. + + To use this method in your attention layer, follow the steps: + + * Use `query` tensor of shape `(batch_size, Tq)` and `key` tensor of + shape `(batch_size, Tv)` to calculate the attention `scores`. + * Pass `scores` and `value` tensors to this method. The method applies + `scores_mask`, calculates + `attention_distribution = softmax(scores)`, then returns + `matmul(attention_distribution, value). + * Apply `query_mask` and return the result. + + Args: + scores: Scores float tensor of shape `(batch_size, Tq, Tv)`. + value: Value tensor of shape `(batch_size, Tv, dim)`. + scores_mask: A boolean mask tensor of shape `(batch_size, 1, Tv)` + or `(batch_size, Tq, Tv)`. If given, scores at positions where + `scores_mask==False` do not contribute to the result. It must + contain at least one `True` value in each line along the last + dimension. + training: Python boolean indicating whether the layer should behave + in training mode (adding dropout) or in inference mode + (no dropout). + + Returns: + Tensor of shape `(batch_size, Tq, dim)`. + Attention scores after masking and softmax with shape + `(batch_size, Tq, Tv)`. + """ + if scores_mask is not None: + padding_mask = tf.logical_not(scores_mask) + # Bias so padding positions do not contribute to attention + # distribution. Note 65504. is the max float16 value. + max_value = 65504.0 if scores.dtype == 'float16' else 1.0e9 + scores -= max_value * tf.cast(padding_mask, dtype=scores.dtype) + + weights = tf.nn.softmax(scores, axis=-1) + if training and self.dropout > 0: + weights = tf.nn.dropout(weights, 1.0 - self.dropout, seed=self.seed) + return tf.matmul(weights, value), weights + + def _calculate_score_mask(self, scores, v_mask, use_causal_mask): + if use_causal_mask: + # Creates a lower triangular mask, so position i cannot attend to + # positions j > i. This prevents the flow of information from the + # future into the past. + score_shape = tf.shape(scores) + # causal_mask_shape = [1, Tq, Tv]. + mask_shape = (1, score_shape[-2], score_shape[-1]) + ones_mask = tf.ones(shape=mask_shape, dtype='int32') + row_index = tf.cumsum(ones_mask, axis=-2) + col_index = tf.cumsum(ones_mask, axis=-1) + causal_mask = tf.greater_equal(row_index, col_index) + + if v_mask is not None: + # Mask of shape [batch_size, 1, Tv]. + v_mask = tf.expand_dims(v_mask, axis=-2) + return tf.logical_and(v_mask, causal_mask) + return causal_mask + else: + # If not using causal mask, return the value mask as is, + # or None if the value mask is not provided. + return v_mask + + def call( + self, + inputs, + mask=None, + training=False, + ): + self._validate_inputs(inputs=inputs, mask=mask) + q = inputs[0] + v = inputs[1] + k = inputs[2] if len(inputs) > 2 else v + q_mask = mask[0] if mask else None + v_mask = mask[1] if mask else None + scores = self._calculate_scores(query=q, key=k) + scores_mask = self._calculate_score_mask(scores, v_mask, + self.use_causal_mask) + result, attention_scores = self._apply_scores( + scores=scores, value=v, scores_mask=scores_mask, training=training) + if q_mask is not None: + # Mask of shape [batch_size, Tq, 1]. + q_mask = tf.expand_dims(q_mask, axis=-1) + result *= tf.cast(q_mask, dtype=result.dtype) + if self.return_attention_scores: + return result, attention_scores + return result + + def compute_mask(self, inputs, mask=None): + self._validate_inputs(inputs=inputs, mask=mask) + if mask is None or mask[0] is None: + return None + return tf.convert_to_tensor(mask[0]) + + def compute_output_shape(self, input_shape): + """Returns shape of value tensor dim, but for query tensor length.""" + return list(input_shape[0][:-1]), input_shape[1][-1] + + def _validate_inputs(self, inputs, mask=None): + """Validates arguments of the call method.""" + class_name = self.__class__.__name__ + if not isinstance(inputs, list): + raise ValueError('{class_name} layer must be called on a list of inputs, ' + 'namely [query, value] or [query, value, key]. ' + 'Received: inputs={inputs}.'.format( + class_name=class_name, inputs=inputs)) + if len(inputs) < 2 or len(inputs) > 3: + raise ValueError('%s layer accepts inputs list of length 2 or 3, ' + 'namely [query, value] or [query, value, key]. ' + 'Received length: %d.' % (class_name, len(inputs))) + if mask is not None: + if not isinstance(mask, list): + raise ValueError( + '{class_name} layer mask must be a list, ' + 'namely [query_mask, value_mask]. Received: mask={mask}.'.format( + class_name=class_name, mask=mask)) + if len(mask) < 2 or len(mask) > 3: + raise ValueError( + '{class_name} layer accepts mask list of length 2 or 3. ' + 'Received: inputs={inputs}, mask={mask}.'.format( + class_name=class_name, inputs=inputs, mask=mask)) + + def get_config(self): + base_config = super(Attention, self).get_config() + config = { + 'use_scale': self.use_scale, + 'score_mode': self.score_mode, + 'dropout': self.dropout, + } + return dict(list(base_config.items()) + list(config.items())) diff --git a/easy_rec/python/layers/keras/blocks.py b/easy_rec/python/layers/keras/blocks.py index 06ce11cbf..13cd14612 100644 --- a/easy_rec/python/layers/keras/blocks.py +++ b/easy_rec/python/layers/keras/blocks.py @@ -4,6 +4,11 @@ import logging import tensorflow as tf +from tensorflow.python.keras.initializers import Constant +from tensorflow.python.keras.layers import Dense +from tensorflow.python.keras.layers import Dropout +from tensorflow.python.keras.layers import Lambda +from tensorflow.python.keras.layers import Layer from easy_rec.python.layers.keras.activation import activation_layer from easy_rec.python.layers.utils import Parameter @@ -14,7 +19,7 @@ tf = tf.compat.v1 -class MLP(tf.keras.layers.Layer): +class MLP(Layer): """Sequential multi-layer perceptron (MLP) block. Attributes: @@ -74,7 +79,7 @@ def add_rich_layer(self, l2_reg=None): act_layer = activation_layer(activation) if use_bn and not use_bn_after_activation: - dense = tf.keras.layers.Dense( + dense = Dense( units=num_units, use_bias=use_bias, kernel_initializer=initializer, @@ -86,7 +91,7 @@ def add_rich_layer(self, self._sub_layers.append(bn) self._sub_layers.append(act_layer) else: - dense = tf.keras.layers.Dense( + dense = Dense( num_units, use_bias=use_bias, kernel_initializer=initializer, @@ -99,7 +104,7 @@ def add_rich_layer(self, self._sub_layers.append(bn) if 0.0 < dropout_rate < 1.0: - dropout = tf.keras.layers.Dropout(dropout_rate, name='%s/dropout' % name) + dropout = Dropout(dropout_rate, name='%s/dropout' % name) self._sub_layers.append(dropout) elif dropout_rate >= 1.0: raise ValueError('invalid dropout_ratio: %.3f' % dropout_rate) @@ -117,31 +122,56 @@ def call(self, x, training=None, **kwargs): return x -class Highway(tf.keras.layers.Layer): +class Highway(Layer): def __init__(self, params, name='highway', reuse=None, **kwargs): super(Highway, self).__init__(name, **kwargs) self.emb_size = params.get_or_default('emb_size', None) self.num_layers = params.get_or_default('num_layers', 1) - self.activation = params.get_or_default('activation', 'gelu') + self.activation = params.get_or_default('activation', 'relu') self.dropout_rate = params.get_or_default('dropout_rate', 0.0) self.init_gate_bias = params.get_or_default('init_gate_bias', -3.0) - self.reuse = reuse + self.act_layer = activation_layer(self.activation) + self.dropout_layer = Dropout( + self.dropout_rate) if self.dropout_rate > 0.0 else None + self.project_layer = None + self.gate_bias_initializer = Constant(self.init_gate_bias) + self.gates = [] # T + self.transforms = [] # H + self.multiply_layer = tf.keras.layers.Multiply() + self.add_layer = tf.keras.layers.Add() + + def build(self, input_shape): + dim = input_shape[-1] + if self.emb_size is not None and dim != self.emb_size: + self.project_layer = Dense(self.emb_size, name='input_projection') + dim = self.emb_size + self.carry_gate = Lambda(lambda x: 1.0 - x, output_shape=(dim,)) + for i in range(self.num_layers): + gate = Dense( + units=dim, + bias_initializer=self.gate_bias_initializer, + activation='sigmoid', + name='gate_%d' % i) + self.gates.append(gate) + self.transforms.append(Dense(units=dim)) def call(self, inputs, training=None, **kwargs): - from easy_rec.python.layers.common_layers import highway - return highway( - inputs, - self.emb_size, - activation=self.activation, - num_layers=self.num_layers, - dropout=self.dropout_rate if training else 0.0, - init_gate_bias=self.init_gate_bias, - scope=self.name, - reuse=self.reuse) - - -class Gate(tf.keras.layers.Layer): + value = inputs + if self.project_layer is not None: + value = self.project_layer(inputs) + for i in range(self.num_layers): + gate = self.gates[i](value) + transformed = self.act_layer(self.transforms[i](value)) + if self.dropout_layer is not None: + transformed = self.dropout_layer(transformed, training=training) + transformed_gated = self.multiply_layer([gate, transformed]) + identity_gated = self.multiply_layer([self.carry_gate(gate), value]) + value = self.add_layer([transformed_gated, identity_gated]) + return value + + +class Gate(Layer): """Weighted sum gate.""" def __init__(self, params, name='gate', reuse=None, **kwargs): @@ -165,7 +195,7 @@ def call(self, inputs, **kwargs): return output -class TextCNN(tf.keras.layers.Layer): +class TextCNN(Layer): """Text CNN Model. References diff --git a/easy_rec/python/model/easy_rec_model.py b/easy_rec/python/model/easy_rec_model.py index e45010553..f2408ba47 100644 --- a/easy_rec/python/model/easy_rec_model.py +++ b/easy_rec/python/model/easy_rec_model.py @@ -120,6 +120,8 @@ def backbone(self): kwargs = { 'loss_dict': self._loss_dict, 'metric_dict': self._metric_dict, + 'prediction_dict': self._prediction_dict, + 'labels': self._labels, constant.SAMPLE_WEIGHT: self._sample_weight } return self._backbone_net(self._is_training, **kwargs) diff --git a/easy_rec/python/model/multi_task_model.py b/easy_rec/python/model/multi_task_model.py index cff58e079..fa6ce8948 100644 --- a/easy_rec/python/model/multi_task_model.py +++ b/easy_rec/python/model/multi_task_model.py @@ -4,9 +4,13 @@ from collections import OrderedDict import tensorflow as tf +from google.protobuf import struct_pb2 +from tensorflow.python.keras.layers import Dense from easy_rec.python.builders import loss_builder from easy_rec.python.layers.dnn import DNN +from easy_rec.python.layers.keras.attention import Attention +from easy_rec.python.layers.utils import Parameter from easy_rec.python.model.rank_model import RankModel from easy_rec.python.protos import tower_pb2 from easy_rec.python.protos.easy_rec_model_pb2 import EasyRecModel @@ -82,6 +86,28 @@ def build_predict_graph(self): tower_inputs, axis=-1, name=tower_name + '/relation_input') relation_fea = relation_dnn(relation_input) relation_features[tower_name] = relation_fea + elif task_tower_cfg.use_ait_module: + tower_inputs = [tower_features[tower_name]] + for relation_tower_name in task_tower_cfg.relation_tower_names: + tower_inputs.append(relation_features[relation_tower_name]) + if len(tower_inputs) == 1: + relation_fea = tower_inputs[0] + relation_features[tower_name] = relation_fea + else: + if task_tower_cfg.HasField('ait_project_dim'): + dim = task_tower_cfg.ait_project_dim + else: + dim = int(tower_inputs[0].shape[-1]) + queries = tf.stack([Dense(dim)(x) for x in tower_inputs], axis=1) + keys = tf.stack([Dense(dim)(x) for x in tower_inputs], axis=1) + values = tf.stack([Dense(dim)(x) for x in tower_inputs], axis=1) + st_params = struct_pb2.Struct() + st_params.update({'scale_by_dim': True}) + params = Parameter(st_params, True) + attention_layer = Attention(params, name='AITM_%s' % tower_name) + result = attention_layer([queries, values, keys]) + relation_fea = result[:, 0, :] + relation_features[tower_name] = relation_fea else: relation_fea = tower_features[tower_name] @@ -222,7 +248,17 @@ def build_loss_graph(self): for loss_name in loss_dict.keys(): loss_dict[loss_name] = loss_dict[loss_name] * task_loss_weight[0] else: + calibrate_loss = [] for loss in losses: + if loss.loss_type == LossType.ORDER_CALIBRATE_LOSS: + y_t = self._prediction_dict['probs_%s' % tower_name] + for relation_tower_name in task_tower_cfg.relation_tower_names: + y_rt = self._prediction_dict['probs_%s' % relation_tower_name] + cali_loss = tf.reduce_mean(tf.nn.relu(y_t - y_rt)) + calibrate_loss.append(cali_loss * loss.weight) + logging.info('calibrate loss: %s -> %s' % + (relation_tower_name, tower_name)) + continue loss_param = loss.WhichOneof('loss_param') if loss_param is not None: loss_param = getattr(loss, loss_param) @@ -241,6 +277,10 @@ def build_loss_graph(self): loss.loss_type, loss_name, loss_value) else: loss_dict[loss_name] = loss_value * task_loss_weight[i] + if calibrate_loss: + cali_loss = tf.add_n(calibrate_loss) + loss_dict['order_calibrate_loss'] = cali_loss + tf.summary.scalar('loss/order_calibrate_loss', cali_loss) self._loss_dict.update(loss_dict) kd_loss_dict = loss_builder.build_kd_loss(self.kd, self._prediction_dict, @@ -261,6 +301,8 @@ def get_outputs(self): suffix='_%s' % tower_name)) else: for loss in task_tower_cfg.losses: + if loss.loss_type == LossType.ORDER_CALIBRATE_LOSS: + continue outputs.extend( self._get_outputs_impl( loss.loss_type, diff --git a/easy_rec/python/protos/keras_layer.proto b/easy_rec/python/protos/keras_layer.proto index 3b7c0d34d..a8b92d1a7 100644 --- a/easy_rec/python/protos/keras_layer.proto +++ b/easy_rec/python/protos/keras_layer.proto @@ -26,5 +26,6 @@ message KerasLayer { SequenceAugment seq_aug = 15; PPNet ppnet = 16; TextCNN text_cnn = 17; + HighWayTower highway = 18; } } diff --git a/easy_rec/python/protos/layer.proto b/easy_rec/python/protos/layer.proto index df51009bc..c0a01686a 100644 --- a/easy_rec/python/protos/layer.proto +++ b/easy_rec/python/protos/layer.proto @@ -6,8 +6,10 @@ import "easy_rec/python/protos/dnn.proto"; message HighWayTower { optional string input = 1; required uint32 emb_size = 2; - required string activation = 3 [default = 'gelu']; + required string activation = 3 [default = 'relu']; optional float dropout_rate = 4; + optional float init_gate_bias = 5 [default = -3.0]; + optional uint32 num_layers = 6 [default = 1]; } message PeriodicEmbedding { diff --git a/easy_rec/python/protos/loss.proto b/easy_rec/python/protos/loss.proto index 5c913bf6e..5098518b3 100644 --- a/easy_rec/python/protos/loss.proto +++ b/easy_rec/python/protos/loss.proto @@ -17,6 +17,7 @@ enum LossType { PAIRWISE_FOCAL_LOSS = 11; PAIRWISE_LOGISTIC_LOSS = 12; JRC_LOSS = 13; + ORDER_CALIBRATE_LOSS = 14; } message Loss { diff --git a/easy_rec/python/protos/tower.proto b/easy_rec/python/protos/tower.proto index 580708825..14cf64c63 100644 --- a/easy_rec/python/protos/tower.proto +++ b/easy_rec/python/protos/tower.proto @@ -58,7 +58,7 @@ message BayesTaskTower { optional DNN relation_dnn = 8; // training loss weights optional float weight = 9 [default = 1.0]; - // label name for indcating the sample space for the task tower + // label name for indicating the sample space for the task tower optional string task_space_indicator_label = 10; // the loss weight for sample in the task space optional float in_task_space_weight = 11 [default = 1.0]; @@ -72,4 +72,8 @@ message BayesTaskTower { repeated Loss losses = 15; // whether to use sample weight in this tower required bool use_sample_weight = 16 [default = true]; + // whether to use AIT module + optional bool use_ait_module = 17 [default = false]; + // set this when the dimensions of last layer of towers are not equal + optional uint32 ait_project_dim = 18; }; diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index 73f05836d..bf2052dc5 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -650,6 +650,11 @@ def test_tag_kv_input(self): 'samples/model_config/kv_tag.config', self._test_dir) self.assertTrue(self._success) + def test_aitm(self): + self._success = test_utils.test_single_train_eval( + 'samples/model_config/aitm_on_taobao.config', self._test_dir) + self.assertTrue(self._success) + def test_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_taobao.config', self._test_dir) diff --git a/easy_rec/version.py b/easy_rec/version.py index 68e35a53c..2ae7769a6 100644 --- a/easy_rec/version.py +++ b/easy_rec/version.py @@ -1,4 +1,4 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -__version__ = '0.8.0' +__version__ = '0.8.1' diff --git a/samples/model_config/aitm_on_taobao.config b/samples/model_config/aitm_on_taobao.config new file mode 100644 index 000000000..c67f1d677 --- /dev/null +++ b/samples/model_config/aitm_on_taobao.config @@ -0,0 +1,295 @@ +train_input_path: "data/test/tb_data/taobao_train_data" +eval_input_path: "data/test/tb_data/taobao_test_data" +model_dir: "experiments/aitm_taobao_ckpt" + +train_config { + optimizer_config { + adam_optimizer { + learning_rate { + constant_learning_rate { + learning_rate: 0.0001 + } + } + } + use_moving_average: false + } + num_steps: 500 + sync_replicas: true + save_checkpoints_steps: 100 + log_step_count_steps: 100 +} +data_config { + batch_size: 4096 + label_fields: "clk" + label_fields: "buy" + prefetch_size: 32 + input_type: CSVInput + input_fields { + input_name: "clk" + input_type: INT32 + } + input_fields { + input_name: "buy" + input_type: INT32 + } + input_fields { + input_name: "pid" + input_type: STRING + } + input_fields { + input_name: "adgroup_id" + input_type: STRING + } + input_fields { + input_name: "cate_id" + input_type: STRING + } + input_fields { + input_name: "campaign_id" + input_type: STRING + } + input_fields { + input_name: "customer" + input_type: STRING + } + input_fields { + input_name: "brand" + input_type: STRING + } + input_fields { + input_name: "user_id" + input_type: STRING + } + input_fields { + input_name: "cms_segid" + input_type: STRING + } + input_fields { + input_name: "cms_group_id" + input_type: STRING + } + input_fields { + input_name: "final_gender_code" + input_type: STRING + } + input_fields { + input_name: "age_level" + input_type: STRING + } + input_fields { + input_name: "pvalue_level" + input_type: STRING + } + input_fields { + input_name: "shopping_level" + input_type: STRING + } + input_fields { + input_name: "occupation" + input_type: STRING + } + input_fields { + input_name: "new_user_class_level" + input_type: STRING + } + input_fields { + input_name: "tag_category_list" + input_type: STRING + } + input_fields { + input_name: "tag_brand_list" + input_type: STRING + } + input_fields { + input_name: "price" + input_type: INT32 + } +} +feature_config: { + features { + input_names: "pid" + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features { + input_names: "adgroup_id" + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100000 + } + features { + input_names: "cate_id" + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10000 + } + features { + input_names: "campaign_id" + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100000 + } + features { + input_names: "customer" + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100000 + } + features { + input_names: "brand" + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100000 + } + features { + input_names: "user_id" + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100000 + } + features { + input_names: "cms_segid" + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100 + } + features { + input_names: "cms_group_id" + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100 + } + features { + input_names: "final_gender_code" + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features { + input_names: "age_level" + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features { + input_names: "pvalue_level" + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features { + input_names: "shopping_level" + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features { + input_names: "occupation" + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features { + input_names: "new_user_class_level" + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features { + input_names: "tag_category_list" + feature_type: TagFeature + embedding_dim: 16 + hash_bucket_size: 100000 + separator: "|" + } + features { + input_names: "tag_brand_list" + feature_type: TagFeature + embedding_dim: 16 + hash_bucket_size: 100000 + separator: "|" + } + features { + input_names: "price" + feature_type: IdFeature + embedding_dim: 16 + num_buckets: 50 + } +} +model_config { + model_name: "AITM" + model_class: "MultiTaskModel" + feature_groups { + group_name: "all" + feature_names: "user_id" + feature_names: "cms_segid" + feature_names: "cms_group_id" + feature_names: "age_level" + feature_names: "pvalue_level" + feature_names: "shopping_level" + feature_names: "occupation" + feature_names: "new_user_class_level" + feature_names: "adgroup_id" + feature_names: "cate_id" + feature_names: "campaign_id" + feature_names: "customer" + feature_names: "brand" + feature_names: "price" + feature_names: "pid" + feature_names: "tag_category_list" + feature_names: "tag_brand_list" + wide_deep: DEEP + } + backbone { + blocks { + name: "mlp" + inputs { + feature_group_name: "all" + } + keras_layer { + class_name: 'MLP' + mlp { + hidden_units: [512, 256] + } + } + } + } + model_params { + task_towers { + tower_name: "ctr" + label_name: "clk" + loss_type: CLASSIFICATION + metrics_set: { + auc {} + } + dnn { + hidden_units: [256, 128] + } + use_ait_module: true + weight: 1.0 + } + task_towers { + tower_name: "cvr" + label_name: "buy" + losses { + loss_type: CLASSIFICATION + } + losses { + loss_type: ORDER_CALIBRATE_LOSS + } + metrics_set: { + auc {} + } + dnn { + hidden_units: [256, 128] + } + relation_tower_names: ["ctr"] + use_ait_module: true + ait_project_dim: 128 + weight: 1.0 + } + l2_regularization: 1e-6 + } + embedding_regularization: 5e-6 +} From f3d4b0e6d5d95a8289e9ba23ee1a2efbf5d64d50 Mon Sep 17 00:00:00 2001 From: tiankongdeguiji Date: Tue, 18 Jun 2024 17:18:07 +0800 Subject: [PATCH 2/6] [bugfix]: fix number of negative sample when len(ids) < batch_size (#464) --- easy_rec/python/core/sampler.py | 9 ++++++++- setup.cfg | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/easy_rec/python/core/sampler.py b/easy_rec/python/core/sampler.py index 6baee406f..779b30b48 100644 --- a/easy_rec/python/core/sampler.py +++ b/easy_rec/python/core/sampler.py @@ -268,6 +268,7 @@ def __init__(self, def _get_impl(self, ids): ids = np.array(ids, dtype=np.int64) + ids = np.pad(ids, (0, self._batch_size - len(ids)), 'edge') nodes = self._sampler.get(ids) features = self._parse_nodes(nodes) return features @@ -491,7 +492,9 @@ def __init__(self, def _get_impl(self, src_ids, dst_ids): src_ids = np.array(src_ids, dtype=np.int64) + src_ids = np.pad(src_ids, (0, self._batch_size - len(src_ids)), 'edge') dst_ids = np.array(dst_ids, dtype=np.int64) + dst_ids = np.pad(dst_ids, (0, self._batch_size - len(dst_ids)), 'edge') nodes = self._sampler.get(src_ids, dst_ids) features = self._parse_nodes(nodes) return features @@ -571,6 +574,7 @@ def __init__(self, def _get_impl(self, src_ids, dst_ids): src_ids = np.array(src_ids, dtype=np.int64) dst_ids = np.array(dst_ids, dtype=np.int64) + dst_ids = np.pad(dst_ids, (0, self._batch_size - len(dst_ids)), 'edge') nodes = self._neg_sampler.get(dst_ids) neg_features = self._parse_nodes(nodes) sparse_nodes = self._hard_neg_sampler.get(src_ids).layer_nodes(1) @@ -669,8 +673,11 @@ def __init__(self, def _get_impl(self, src_ids, dst_ids): src_ids = np.array(src_ids, dtype=np.int64) + src_ids_padded = np.pad(src_ids, (0, self._batch_size - len(src_ids)), + 'edge') dst_ids = np.array(dst_ids, dtype=np.int64) - nodes = self._neg_sampler.get(src_ids, dst_ids) + dst_ids = np.pad(dst_ids, (0, self._batch_size - len(dst_ids)), 'edge') + nodes = self._neg_sampler.get(src_ids_padded, dst_ids) neg_features = self._parse_nodes(nodes) sparse_nodes = self._hard_neg_sampler.get(src_ids).layer_nodes(1) hard_neg_features, hard_neg_indices = self._parse_sparse_nodes(sparse_nodes) diff --git a/setup.cfg b/setup.cfg index 337833a0f..b43211827 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,7 +10,7 @@ multi_line_output = 7 force_single_line = true known_standard_library = setuptools known_first_party = easy_rec -known_third_party = absl,common_io,docutils,eas_prediction,faiss,future,google,graphlearn,kafka,matplotlib,numpy,oss2,pai,pandas,psutil,six,sklearn,sparse_operation_kit,sphinx_markdown_tables,sphinx_rtd_theme,tensorflow,yaml +known_third_party = absl,common_io,distutils,docutils,eas_prediction,faiss,future,google,graphlearn,kafka,matplotlib,numpy,oss2,pai,pandas,psutil,six,sklearn,sparse_operation_kit,sphinx_markdown_tables,sphinx_rtd_theme,tensorflow,yaml no_lines_before = LOCALFOLDER default_section = THIRDPARTY skip = easy_rec/python/protos From 457825e7bb096fb794ac34e714446bd52a98307c Mon Sep 17 00:00:00 2001 From: zhenghong Date: Wed, 26 Jun 2024 14:28:50 +0800 Subject: [PATCH 3/6] =?UTF-8?q?fix=20id=5Ffeature's=20number=5Fbuckets=20u?= =?UTF-8?q?age=20with=20fg=EF=BC=8Cadd=20more=20info=20for=20using=20(#472?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/benchmark.md | 1 + docs/source/feature/data.md | 6 ++--- docs/source/feature/feature.rst | 11 +++++---- docs/source/feature/pai_rec_callback_conf.md | 2 ++ docs/source/feature/rtp_fg.md | 2 +- docs/source/feature/rtp_native.md | 2 +- docs/source/intro.md | 2 +- ...73\347\272\277\351\242\204\346\265\213.md" | 5 +++- docs/source/predict/processor.md | 6 ++--- ...50\347\272\277\351\242\204\346\265\213.md" | 2 +- docs/source/quick_start/designer_tutorial.md | 4 ++++ docs/source/quick_start/local_tutorial.md | 4 +++- docs/source/quick_start/mc_tutorial.md | 24 +++++++++++-------- docs/source/quick_start/mc_tutorial_inner.md | 2 +- 14 files changed, 45 insertions(+), 28 deletions(-) diff --git a/docs/source/benchmark.md b/docs/source/benchmark.md index 8e2d20c6f..8a2c5348e 100644 --- a/docs/source/benchmark.md +++ b/docs/source/benchmark.md @@ -9,6 +9,7 @@ - 该数据集是淘宝展示广告点击率预估数据集，包含用户、广告特征和行为日志。[天池比赛链接](https://tianchi.aliyun.com/dataset/dataDetail?dataId=56) - 训练数据表：pai_online_project.easyrec_demo_taobao_train_data - 测试数据表：pai_online_project.easyrec_demo_taobao_test_data +- 其中pai_online_project是一个公共读的MaxCompute project，里面写入了一些数据表做测试，不需要申请权限。 - 在PAI上面测试使用的资源包括2个parameter server，9个worker，其中一个worker做评估: ```json {"ps":{"count":2, diff --git a/docs/source/feature/data.md b/docs/source/feature/data.md index 827791ffb..169902b78 100644 --- a/docs/source/feature/data.md +++ b/docs/source/feature/data.md @@ -2,7 +2,7 @@ EasyRec作为阿里云PAI的推荐算法包，可以无缝对接MaxCompute的数据表，也可以读取OSS中的大文件，还支持E-MapReduce环境中的HDFS文件，也支持local环境中的csv文件。 -为了识别这些输入数据中的字段信息，需要设置相应的字段名称和字段类型、设置默认值，帮助EasyRec去读取相应的数据。设置label字段，作为训练的目标。为了适应多目标模型，label字段可以设置多个。 +为了识别这些输入数据中的字段信息，需要设置相应的字段名称和字段类型、设置默认值，帮助EasyRec去读取相应的数据。设置label字段，作为训练的目标。为了适配多目标模型，label字段可设置多个。 另外还有一些参数如prefetch_size，是tensorflow中读取数据需要设置的参数。 @@ -10,7 +10,7 @@ EasyRec作为阿里云PAI的推荐算法包，可以无缝对接MaxCompute的数 这个配置里面，只有三个字段，用户ID（uid）、物品ID（item_id）、label字段（click）。 -OdpsInputV2表示读取MaxCompute的表作为输入数据。 +OdpsInputV2表示读取MaxCompute的表作为输入数据。如果是本地机器上训练，注意使用CSVInput类型。 ```protobuf data_config { @@ -160,7 +160,7 @@ def remap_lbl(labels): ### prefetch_size - data prefetch，以batch为单位，默认是32 -- 设置prefetch size可以提高数据加载的速度，防止数据瓶颈 +- 设置prefetch size可以提高数据加载的速度，防止数据瓶颈。但是当batchsize较小的时候，该值可适当调小。 ### shard && file_shard diff --git a/docs/source/feature/feature.rst b/docs/source/feature/feature.rst index a41b42a53..901fe6673 100644 --- a/docs/source/feature/feature.rst +++ b/docs/source/feature/feature.rst @@ -3,7 +3,7 @@ 在上一节介绍了输入数据包括MaxCompute表、csv文件、hdfs文件、OSS文件等，表或文件的一列对应一个特征。 -在数据中可以有一个或者多个label字段，而特征比较丰富，支持的类型包括IdFeature，RawFeature，TagFeature，SequenceFeature, ComboFeature. +在数据中可以有一个或者多个label字段，在多目标模型中，需要多个label字段。而特征比较丰富，支持的类型包括IdFeature，RawFeature，TagFeature，SequenceFeature, ComboFeature。 各种特征共用字段 ---------------------------------------------------------------- @@ -71,12 +71,12 @@ IdFeature: 离散值特征/ID类特征 .. math:: - embedding\_dim=8+x^{0.25} - - 其中，x 为不同特征取值的个数 + embedding\_dim=8+n^{0.25} + - 其中，n 是特征的唯一值的个数（如gender特征的取值是男、女，则n=2） - hash\_bucket\_size: hash bucket的大小。适用于category_id, user_id等 -- 对于user\_id等规模比较大的，hash冲突影响比较小的特征， +- 对于user\_id等规模比较大的，hash冲突影响比较小的特征，用户行为日志不够丰富可通过hash压缩id数量， .. math:: @@ -91,7 +91,8 @@ IdFeature: 离散值特征/ID类特征 - num\_buckets: buckets number, - 仅仅当输入是integer类型时，可以使用num\_buckets + 仅仅当输入是integer类型时，可以使用num\_buckets。 + 但是当使用fg特征的时候，不要用integer特征用num\_buckets的方式来变换，注意要用hash\_bucket\_size的方式。 - vocab\_list: 指定词表，适合取值比较少可以枚举的特征，如星期，月份，星座等 diff --git a/docs/source/feature/pai_rec_callback_conf.md b/docs/source/feature/pai_rec_callback_conf.md index 151c07b1d..901ec4eb3 100644 --- a/docs/source/feature/pai_rec_callback_conf.md +++ b/docs/source/feature/pai_rec_callback_conf.md @@ -1,4 +1,6 @@ # PAI-REC 全埋点配置 +## PAI-Rec引擎的callback服务文档 +- [文档](http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/pairec/docs/pairec/html/intro/callback_api.html) ## 模板 diff --git a/docs/source/feature/rtp_fg.md b/docs/source/feature/rtp_fg.md index baeaf078b..40da4852e 100644 --- a/docs/source/feature/rtp_fg.md +++ b/docs/source/feature/rtp_fg.md @@ -2,7 +2,7 @@ - RTP FG: RealTime Predict Feature Generation, 解决实时预测需要的特征工程需求. 特征工程在推荐链路里面也占用了比较长的时间. -- RTP FG能够以比较高的效率生成一些复杂的交叉特征，如match feature和lookup feature, 通过使用同一套c++代码保证离线在线的一致性. +- RTP FG能够以比较高的效率生成一些复杂的交叉特征，如match feature和lookup feature.离线训练和在线预测的时候通过使用同一套c++代码保证离线在线的一致性. - 其生成的特征可以接入EasyRec进行训练，从RTP FG的配置(fg.json)可以生成EasyRec的配置文件(pipeline.config). diff --git a/docs/source/feature/rtp_native.md b/docs/source/feature/rtp_native.md index d2524079a..8774041c7 100644 --- a/docs/source/feature/rtp_native.md +++ b/docs/source/feature/rtp_native.md @@ -1,6 +1,6 @@ # RTP部署 -本文档介绍将EasyRec模型部署到RTP上的流程. +本文档介绍将EasyRec模型部署到RTP（Real Time Prediction，实时打分服务）上的流程. - RTP目前仅支持checkpoint形式的模型部署，因此需要将EasyRec模型导出为checkpoint形式 diff --git a/docs/source/intro.md b/docs/source/intro.md index f4dabcb76..4974ff24d 100644 --- a/docs/source/intro.md +++ b/docs/source/intro.md @@ -62,5 +62,5 @@ EasyRec implements state of the art machine learning models used in common recom - Run [`knn algorithm`](vector_retrieve.md) of vectors in distribute environment ### Contact - +- DingDing Group: 32260796. (EasyRec usage general discussion.) - DingDing Group: 37930014162, click [this url](https://qr.dingtalk.com/action/joingroup?code=v1,k1,oHNqtNObbu+xUClHh77gCuKdGGH8AYoQ8AjKU23zTg4=&_dt_no_comment=1&origin=11) or scan QrCode to join![new_group.jpg](../images/qrcode/new_group.jpg) diff --git "a/docs/source/predict/MaxCompute \347\246\273\347\272\277\351\242\204\346\265\213.md" "b/docs/source/predict/MaxCompute \347\246\273\347\272\277\351\242\204\346\265\213.md" index 7f0b9e675..dd867a165 100644 --- "a/docs/source/predict/MaxCompute \347\246\273\347\272\277\351\242\204\346\265\213.md" +++ "b/docs/source/predict/MaxCompute \347\246\273\347\272\277\351\242\204\346\265\213.md" @@ -11,7 +11,7 @@ drop table if exists ctr_test_output; pai -name easy_rec_ext -Dcmd=predict --Dcluster='{"worker" : {"count":5, "cpu":1600, "memory":40000, "gpu":100}}' +-Dcluster='{"worker" : {"count":5, "cpu":1000, "memory":40000, "gpu":0}}' -Darn=acs:ram::xxx:role/aliyunodpspaidefaultrole -Dbuckets=oss://easyrec/ -Dsaved_model_dir=oss://easyrec/easy_rec_test/experiment/export/1597299619 @@ -23,6 +23,7 @@ pai -name easy_rec_ext -DossHost=oss-cn-beijing-internal.aliyuncs.com; ``` +- cluster: 这里cpu:1000表示是10个cpu核；核与内存的关系设置1:4000，一般不超过40000；gpu设置为0，表示不用GPU推理。 - saved_model_dir: 导出的模型目录 - output_table: 输出表，不需要提前创建，会自动创建 - excluded_cols: 预测模型不需要的columns，比如labels @@ -55,6 +56,8 @@ pai -name easy_rec_ext - 多分类模型(num_class > 1)，导出字段: - logits: string(json), softmax之前的vector, shape\[num_class\] - probs: string(json), softmax之后的vector, shape\[num_class\] + - 如果一个分类目标是is_click, 输出概率的变量名称是probs_is_click + - 多目标模型中有一个回归目标是paytime，那么输出回归预测分的变量名称是：y_paytime - logits_y: logits\[y\], float, 类别y对应的softmax之前的概率 - probs_y: probs\[y\], float, 类别y对应的概率 - y: 类别id, = argmax(probs_y), int, 概率最大的类别 diff --git a/docs/source/predict/processor.md b/docs/source/predict/processor.md index 0ce0b4bd8..dabdb7aa1 100644 --- a/docs/source/predict/processor.md +++ b/docs/source/predict/processor.md @@ -1,17 +1,17 @@ # EasyRec Processor -EasyRec Processor, 是EasyRec对应的高性能在线打分引擎, 包含特征处理和模型推理功能. EasyRecProcessor运行在PAI-EAS之上, 可以充分利用PAI-EAS多种优化特性. +EasyRec Processor([阿里云上的EasyRec Processor详细文档，包括版本、使用方式](https://help.aliyun.com/zh/pai/user-guide/easyrec)), 是EasyRec对应的高性能在线打分引擎, 包含特征处理和模型推理功能. EasyRecProcessor运行在PAI-EAS之上, 可以充分利用PAI-EAS多种优化特性. ## 架构设计 -EasyRec Processor包含三个部分: Item特征缓存, 特征处理(Feature Generator), TFModel(tensorflow model). +EasyRec Processor包含三个部分: Item特征缓存（支持通过[FeatureStore](https://help.aliyun.com/zh/pai/user-guide/featurestore-overview)加载MaxCompute表做初始化）, 特征生成(Feature Generator), TFModel(tensorflow model). ![image.png](../../images/processor/easy_rec_processor_1.png) ## 性能优化 ### 基础实现 -将FeatureGenerator和TFModel分开, 先做特征生成，然后再Run TFModel. +将FeatureGenerator和TFModel分开, 先做特征生成（即fg），然后再Run TFModel得到预测结果. ### 优化实现 diff --git "a/docs/source/predict/\345\234\250\347\272\277\351\242\204\346\265\213.md" "b/docs/source/predict/\345\234\250\347\272\277\351\242\204\346\265\213.md" index 56f496945..8cb7db1ca 100644 --- "a/docs/source/predict/\345\234\250\347\272\277\351\242\204\346\265\213.md" +++ "b/docs/source/predict/\345\234\250\347\272\277\351\242\204\346\265\213.md" @@ -1,6 +1,6 @@ # Model Serving -推荐使用阿里云上的[模型在线服务(PAI-EAS)](https://help.aliyun.com/document_detail/113696.html)预置的EasyRecProcessor 来部署在线推理服务。EasyRecProcessor针对推荐模型做了多种优化, 相比tensorflow serving和TensorRT方式部署具有显著的[性能优势](./processor.md)。 +推荐使用阿里云上的[模型在线服务(PAI-EAS)](https://help.aliyun.com/document_detail/113696.html)预置的EasyRecProcessor 来部署在线推理服务。EasyRec Processor（[阿里云文档](https://help.aliyun.com/zh/pai/user-guide/easyrec)）针对推荐模型做了多种优化, 相比tensorflow serving和TensorRT方式部署具有显著的[性能优势](./processor.md)。 ## 命令行部署 diff --git a/docs/source/quick_start/designer_tutorial.md b/docs/source/quick_start/designer_tutorial.md index 95d9899b2..66a22d15c 100644 --- a/docs/source/quick_start/designer_tutorial.md +++ b/docs/source/quick_start/designer_tutorial.md @@ -94,3 +94,7 @@ PAI-Designer（Studio 2.0）是基于云原生架构Pipeline Service（PAIFlow `pai -name easy_rec_ext -project algo_public -Dcmd=predict` - 具体命令及详细[参数说明](../train.md#on-pai) + +### 推荐算法定制的方案 + +- 在Designer中做推荐算法特征工程、排序模型训练、向量召回等案例的阿里云官网[文档链接](https://help.aliyun.com/zh/pai/use-cases/overview-18) diff --git a/docs/source/quick_start/local_tutorial.md b/docs/source/quick_start/local_tutorial.md index 8074c1218..0d07db1c9 100644 --- a/docs/source/quick_start/local_tutorial.md +++ b/docs/source/quick_start/local_tutorial.md @@ -4,6 +4,8 @@ 我们提供了`本地Anaconda安装`和`Docker镜像启动`两种方式。 +有技术问题可加钉钉群：37930014162 + #### 本地Anaconda安装 Demo实验中使用的环境为 `python=3.6.8` + `tenserflow=1.12.0` @@ -52,7 +54,7 @@ sudo docker exec -it bash 输入一般是csv格式的文件。 -#### 示例数据 +#### 示例数据(点击下载) - train: [dwd_avazu_ctr_deepmodel_train.csv](http://easyrec.oss-cn-beijing.aliyuncs.com/data/dwd_avazu_ctr_deepmodel_train.csv) - test: [dwd_avazu_ctr_deepmodel_test.csv](http://easyrec.oss-cn-beijing.aliyuncs.com/data/dwd_avazu_ctr_deepmodel_test.csv) diff --git a/docs/source/quick_start/mc_tutorial.md b/docs/source/quick_start/mc_tutorial.md index 16761d2db..0f6065f0c 100644 --- a/docs/source/quick_start/mc_tutorial.md +++ b/docs/source/quick_start/mc_tutorial.md @@ -4,14 +4,18 @@ 针对阿里集团内部用户，请参考[mc_tutorial_inner](mc_tutorial_inner.md)。 +有技术问题可加钉钉群：37930014162 + ### 输入数据: -输入一般是odps表: +输入一般是MaxCompute表: - train: pai_online_project.dwd_avazu_ctr_deepmodel_train -- test: pai_online_project.dwd_avazu_ctr_deepmodel_test +- test: pai_online_project.dwd_avazu_ctr_deepmodel_test + +说明：原则上这两张表是自己odps的表，为了方便，以上提供case的两张表可在国内用户的MaxCompute项目空间中访问。 -说明：原则上这两张表是自己odps的表，为了方便，以上提供case的两张表在任何地方都可以访问。两个表可以带分区，也可以不带分区。 +两个表可以带分区，也可以不带分区。带分区的方式：odps://xyz_project/table1/dt=20240101 ### 训练: @@ -24,7 +28,7 @@ pai -name easy_rec_ext -project algo_public -Dconfig=oss://easyrec/config/MultiTower/dwd_avazu_ctr_deepmodel_ext.config -Dtrain_tables='odps://pai_online_project/tables/dwd_avazu_ctr_deepmodel_train' -Deval_tables='odps://pai_online_project/tables/dwd_avazu_ctr_deepmodel_test' --Dcluster='{"ps":{"count":1, "cpu":1000}, "worker" : {"count":3, "cpu":1000, "gpu":100, "memory":40000}}' +-Dcluster='{"ps":{"count":1, "cpu":1000}, "worker" : {"count":3, "cpu":1000, "gpu":0, "memory":40000}}' -Deval_method=separate -Dmodel_dir=oss://easyrec/ckpt/MultiTower -Darn=acs:ram::xxx:role/xxx @@ -32,26 +36,26 @@ pai -name easy_rec_ext -project algo_public -DossHost=oss-cn-beijing-internal.aliyuncs.com; ``` -- -Dcmd: train 模型训练 +- -Dcmd: train 表示模型训练 - -Dconfig: 训练用的配置文件 - -Dtrain_tables: 定义训练表 -- -Deval_tables: 定义测试表 +- -Deval_tables: 定义评估表 - -Dtables: 定义其他依赖表(可选)，如负采样的表 - -Dcluster: 定义PS的数目和worker的数目。具体见：[PAI-TF任务参数介绍](https://help.aliyun.com/document_detail/154186.html?spm=a2c4g.11186623.4.3.e56f1adb7AJ9T5) - -Deval_method: 评估方法 -- separate: 用worker(task_id=1)做评估 +- separate: 用worker(task_id=1)做评估。找到MaxCompute训练任务的logview，打开logview之后在worker1机器的stderr日志中查看评估指标数据。 - none: 不需要评估 - master: 在master(task_id=0)上做评估 - -Dfine_tune_checkpoint: 可选，从checkpoint restore参数，进行finetune - 可以指定directory，将使用directory里面的最新的checkpoint. - -Dmodel_dir: 如果指定了model_dir将会覆盖config里面的model_dir，一般在周期性调度的时候使用。 -- -Darn: rolearn 注意这个的arn要替换成客户自己的。可以从dataworks的设置中查看arn。 +- -Darn: rolearn 注意这个的arn要替换成客户自己的。可以从dataworks的设置中查看arn;或者阿里云控制台人工智能平台PAI，左侧菜单"开通和授权"，找到全部云产品依赖->Designer->OSS->查看授权信息。 - -Dbuckets: config所在的bucket和保存模型的bucket; 如果有多个bucket，逗号分割 - -DossHost: ossHost地址 ### 注意： -- dataworks和pai的project 一样，案例都是pai_online_project，用户需要根据自己的环境修改。如果需要使用gpu，PAI的project需要设置开通GPU。链接：[https://pai.data.aliyun.com/console?projectId=®ionId=cn-beijing#/visual](https://pai.data.aliyun.com/console?projectId=%C2%AEionId=cn-beijing#/visual) ，其中regionId可能不一致。 +- dataworks和PAI的project一样，案例都是pai_online_project，用户需要根据自己的环境修改。如果需要使用gpu，PAI的project需要设置开通GPU。链接：[https://pai.data.aliyun.com/console?projectId=®ionId=cn-beijing#/visual](https://pai.data.aliyun.com/console?projectId=%C2%AEionId=cn-beijing#/visual) ，其中regionId可能不一致。 ![mc_gpu](../../images/quick_start/mc_gpu.png) @@ -68,7 +72,7 @@ pai -name easy_rec_ext -project algo_public -Dcmd=evaluate -Dconfig=oss://easyrec/config/MultiTower/dwd_avazu_ctr_deepmodel_ext.config -Deval_tables='odps://pai_online_project/tables/dwd_avazu_ctr_deepmodel_test' --Dcluster='{"worker" : {"count":1, "cpu":1000, "gpu":100, "memory":40000}}' +-Dcluster='{"worker" : {"count":1, "cpu":1000, "gpu":0, "memory":40000}}' -Dmodel_dir=oss://easyrec/ckpt/MultiTower -Darn=acs:ram::xxx:role/xxx -Dbuckets=oss://easyrec/ diff --git a/docs/source/quick_start/mc_tutorial_inner.md b/docs/source/quick_start/mc_tutorial_inner.md index 04940ad8c..d04dc2e8d 100644 --- a/docs/source/quick_start/mc_tutorial_inner.md +++ b/docs/source/quick_start/mc_tutorial_inner.md @@ -34,7 +34,7 @@ pai -name easy_rec_ext -project algo_public -Dconfig=oss://easyrec/config/MultiTower/dwd_avazu_ctr_deepmodel_ext.config -Dtrain_tables='odps://pai_online_project/tables/dwd_avazu_ctr_deepmodel_train' -Deval_tables='odps://pai_online_project/tables/dwd_avazu_ctr_deepmodel_test' --Dcluster='{"ps":{"count":1, "cpu":1000}, "worker" : {"count":3, "cpu":1000, "gpu":100, "memory":40000}}' +-Dcluster='{"ps":{"count":1, "cpu":1000}, "worker" : {"count":3, "cpu":1000, "gpu":0, "memory":40000}}' -Deval_method=separate -Dmodel_dir=oss://easyrec/ckpt/MultiTower -Dbuckets=oss://easyrec/?role_arn=acs:ram::xxx:role/xxx&host=oss-cn-beijing-internal.aliyuncs.com; From 21528151b6595abc10f251e9be6c664b6b8c20c5 Mon Sep 17 00:00:00 2001 From: paradiseHIT Date: Wed, 3 Jul 2024 17:02:07 +0800 Subject: [PATCH 4/6] update images with public cr (#465) update images with public cr --- docs/source/quick_start/local_tutorial.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/quick_start/local_tutorial.md b/docs/source/quick_start/local_tutorial.md index 0d07db1c9..443312ce9 100644 --- a/docs/source/quick_start/local_tutorial.md +++ b/docs/source/quick_start/local_tutorial.md @@ -33,8 +33,8 @@ Docker的环境为`python=3.6.9` + `tenserflow=1.15.5` ```bash git clone https://github.com/alibaba/EasyRec.git cd EasyRec -docker pull mybigpai-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.6.3 -docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.6.3 +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.7.4 +docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.7.4 docker exec -it bash ``` @@ -44,7 +44,7 @@ docker exec -it bash git clone https://github.com/alibaba/EasyRec.git cd EasyRec bash scripts/build_docker.sh -sudo docker run -td --network host -v /local_path:/docker_path mybigpai-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15- +sudo docker run -td --network host -v /local_path:/docker_path mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15- sudo docker exec -it bash ``` From f433f4bf41c79dc8869b9d9015f40ac1e9024e44 Mon Sep 17 00:00:00 2001 From: paradiseHIT Date: Thu, 4 Jul 2024 14:01:16 +0800 Subject: [PATCH 5/6] update example readme to update images (#474) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update example readme to update images * change to public images --------- Co-authored-by: 叔宝 --- docs/source/feature/pai_rec_callback_conf.md | 2 ++ docs/source/intro.md | 1 + docs/source/quick_start/dlc_tutorial.md | 8 ++++---- docs/source/train.md | 4 ++-- easy_rec/python/compat/early_stopping.py | 2 +- easy_rec/python/test/train_eval_test.py | 2 +- examples/readme.md | 14 ++++++++------ scripts/build_docker.sh | 2 +- scripts/build_docker_tf210.sh | 2 +- 9 files changed, 21 insertions(+), 16 deletions(-) diff --git a/docs/source/feature/pai_rec_callback_conf.md b/docs/source/feature/pai_rec_callback_conf.md index 901ec4eb3..5679222d7 100644 --- a/docs/source/feature/pai_rec_callback_conf.md +++ b/docs/source/feature/pai_rec_callback_conf.md @@ -1,5 +1,7 @@ # PAI-REC 全埋点配置 + ## PAI-Rec引擎的callback服务文档 + - [文档](http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/pairec/docs/pairec/html/intro/callback_api.html) ## 模板 diff --git a/docs/source/intro.md b/docs/source/intro.md index 4974ff24d..b91c0e7cf 100644 --- a/docs/source/intro.md +++ b/docs/source/intro.md @@ -62,5 +62,6 @@ EasyRec implements state of the art machine learning models used in common recom - Run [`knn algorithm`](vector_retrieve.md) of vectors in distribute environment ### Contact + - DingDing Group: 32260796. (EasyRec usage general discussion.) - DingDing Group: 37930014162, click [this url](https://qr.dingtalk.com/action/joingroup?code=v1,k1,oHNqtNObbu+xUClHh77gCuKdGGH8AYoQ8AjKU23zTg4=&_dt_no_comment=1&origin=11) or scan QrCode to join![new_group.jpg](../images/qrcode/new_group.jpg) diff --git a/docs/source/quick_start/dlc_tutorial.md b/docs/source/quick_start/dlc_tutorial.md index 22e067daa..f766a5f93 100644 --- a/docs/source/quick_start/dlc_tutorial.md +++ b/docs/source/quick_start/dlc_tutorial.md @@ -88,16 +88,16 @@ dlc submit tfjob \ --workspace_id=67849 \ --priority=1 \ --workers=1 \ - --worker_image=mybigpai-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.4.9 \ + --worker_image=mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.7.4 \ --worker_spec=ecs.g6.2xlarge \ --ps=1 \ - --ps_image=mybigpai-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.4.9 \ + --ps_image=mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.7.4 \ --ps_spec=ecs.g6.2xlarge \ --chief=true \ - --chief_image=mybigpai-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.4.9 \ + --chief_image=mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.7.4 \ --chief_spec=ecs.g6.2xlarge \ --evaluators=1 \ - --evaluator_image=mybigpai-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.4.9 \ + --evaluator_image=mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.7.4 \ --evaluator_spec=ecs.g6.2xlarge ``` diff --git a/docs/source/train.md b/docs/source/train.md index 843955e81..85dd4af0b 100644 --- a/docs/source/train.md +++ b/docs/source/train.md @@ -194,9 +194,9 @@ pai -name easy_rec_ext -project algo_public ### 依赖 - 混合并行使用Horovod做底层的通信, 因此需要安装Horovod, 可以直接使用下面的镜像 -- mybigpai-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:sok-tf212-gpus-v5 +- mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:sok-tf212-gpus-v5 ``` - sudo docker run --gpus=all --privileged -v /home/easyrec/:/home/easyrec/ -ti mybigpai-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:sok-tf212-gpus-v5 bash + sudo docker run --gpus=all --privileged -v /home/easyrec/:/home/easyrec/ -ti mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:sok-tf212-gpus-v5 bash ``` ### 配置 diff --git a/easy_rec/python/compat/early_stopping.py b/easy_rec/python/compat/early_stopping.py index fe4c12132..fc850fb62 100644 --- a/easy_rec/python/compat/early_stopping.py +++ b/easy_rec/python/compat/early_stopping.py @@ -21,9 +21,9 @@ import os import threading import time -from distutils.version import LooseVersion import tensorflow as tf +from distutils.version import LooseVersion from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import init_ops diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index bf2052dc5..68d0b8656 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -7,11 +7,11 @@ import threading import time import unittest -from distutils.version import LooseVersion import numpy as np import six import tensorflow as tf +from distutils.version import LooseVersion from tensorflow.python.platform import gfile from easy_rec.python.main import predict diff --git a/examples/readme.md b/examples/readme.md index fd02b6825..bf936cf21 100644 --- a/examples/readme.md +++ b/examples/readme.md @@ -36,12 +36,14 @@ cd EasyRec -- Docker环境可选 (1) `python=3.6.9` + `tenserflow=1.15.5` -docker pull mybigpai-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.6.3 -docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.6.3 +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.7.4 +docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.7.4 +docker exec -it bash + (2) `python=3.8.10` + `tenserflow=2.10.0` -docker pull mybigpai-registry-vpc.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-0.6.4 -docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-0.6.4 +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-0.7.4 +docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-0.7.4 docker exec -it bash ``` @@ -55,11 +57,11 @@ cd EasyRec -- Docker环境可选 (1) `python=3.6.9` + `tenserflow=1.15.5` bash scripts/build_docker.sh -sudo docker run -td --network host -v /local_path:/docker_path mybigpai-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15- +sudo docker run -td --network host -v /local_path:/docker_path mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15- (2) `python=3.8.10` + `tenserflow=2.10.0` bash scripts/build_docker_tf210.sh -sudo docker run -td --network host -v /local_path:/docker_path mybigpai-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10- +sudo docker run -td --network host -v /local_path:/docker_path mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10- sudo docker exec -it bash ``` diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index be4113257..16a80775a 100644 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -18,4 +18,4 @@ then exit 1 fi -sudo docker build --network=host . -f docker/Dockerfile -t mybigpai-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-${version} +sudo docker build --network=host . -f docker/Dockerfile -t mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-${version} diff --git a/scripts/build_docker_tf210.sh b/scripts/build_docker_tf210.sh index 876d6dd06..33bc1a11d 100644 --- a/scripts/build_docker_tf210.sh +++ b/scripts/build_docker_tf210.sh @@ -18,4 +18,4 @@ then exit 1 fi -sudo docker build --progress=plain --network=host . -f docker/Dockerfile_tf210 -t mybigpai-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-${version} +sudo docker build --progress=plain --network=host . -f docker/Dockerfile_tf210 -t mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-${version} From dd64fd99594c5e2541679778f3b727ad0e9d88a6 Mon Sep 17 00:00:00 2001 From: chengaofei <52209156+chengaofei@users.noreply.github.com> Date: Tue, 9 Jul 2024 14:09:40 +0800 Subject: [PATCH 6/6] fix update_config_remove_invalid_subseq_feature (#475) --- .../tools/add_feature_info_to_config.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/easy_rec/python/tools/add_feature_info_to_config.py b/easy_rec/python/tools/add_feature_info_to_config.py index f1b4a4cfd..b11cfc0a7 100644 --- a/easy_rec/python/tools/add_feature_info_to_config.py +++ b/easy_rec/python/tools/add_feature_info_to_config.py @@ -111,6 +111,28 @@ def main(argv): logging.info('drop feature: %s' % feature_name) feature_group.ClearField('feature_names') feature_group.feature_names.extend(reserved_features) + for sequence_feature in feature_group.sequence_features: + seq_att_maps = sequence_feature.seq_att_map + for seq_att in seq_att_maps: + keys = seq_att.key + reserved_keys = [] + for key in keys: + if key not in drop_feature_names: + reserved_keys.append(key) + else: + logging.info('drop sequence feature key: %s' % key) + seq_att.ClearField('key') + seq_att.key.extend(reserved_keys) + + hist_seqs = seq_att.hist_seq + reserved_hist_seqs = [] + for hist_seq in hist_seqs: + if hist_seq not in drop_feature_names: + reserved_hist_seqs.append(hist_seq) + else: + logging.info('drop sequence feature hist_seq: %s' % hist_seq) + seq_att.ClearField('hist_seq') + seq_att.hist_seq.extend(reserved_hist_seqs) config_dir, config_name = os.path.split(FLAGS.output_config_path) config_util.save_pipeline_config(pipeline_config, config_dir, config_name)