From 4d8582008730eab5972c9841409913e98842d38f Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Fri, 21 Jun 2024 14:58:40 -0700 Subject: [PATCH 1/8] add dask client, use it for scheduler.create_job --- jupyter_scheduler/extension.py | 31 +++++++++++++++++++++++++---- jupyter_scheduler/scheduler.py | 36 +++++++++++++++++----------------- pyproject.toml | 1 + 3 files changed, 46 insertions(+), 22 deletions(-) diff --git a/jupyter_scheduler/extension.py b/jupyter_scheduler/extension.py index 1a4ba3736..3fc323361 100644 --- a/jupyter_scheduler/extension.py +++ b/jupyter_scheduler/extension.py @@ -1,5 +1,4 @@ -import asyncio - +from dask.distributed import Client as DaskClient from jupyter_core.paths import jupyter_data_dir from jupyter_server.extension.application import ExtensionApp from jupyter_server.transutils import _i18n @@ -73,11 +72,15 @@ def initialize_settings(self): environments_manager = self.environment_manager_class() + asyncio_loop = self.serverapp.io_loop.asyncio_loop + dask_client_future = asyncio_loop.create_task(self._get_dask_client()) + scheduler = self.scheduler_class( root_dir=self.serverapp.root_dir, environments_manager=environments_manager, db_url=self.db_url, config=self.config, + dask_client_future=dask_client_future, ) job_files_manager = self.job_files_manager_class(scheduler=scheduler) @@ -86,8 +89,28 @@ def initialize_settings(self): environments_manager=environments_manager, scheduler=scheduler, job_files_manager=job_files_manager, + dask_client_future=dask_client_future, ) if scheduler.task_runner: - loop = asyncio.get_event_loop() - loop.create_task(scheduler.task_runner.start()) + asyncio_loop.create_task(scheduler.task_runner.start()) + + async def _get_dask_client(self): + """Creates and configures a Dask client.""" + return DaskClient(processes=False, asynchronous=True) + + async def stop_extension(self): + """Called by the Jupyter Server when stopping to cleanup resources.""" + try: + await self._stop_extension() + except Exception as e: + self.log.error("Error while stopping Jupyter Scheduler:") + self.log.exception(e) + + async def _stop_extension(self): + """Closes the Dask client if it exists.""" + if "dask_client_future" in self.settings: + dask_client: DaskClient = await self.settings["dask_client_future"] + self.log.info("Closing Dask client.") + await dask_client.close() + self.log.info("Dask client closed.") diff --git a/jupyter_scheduler/scheduler.py b/jupyter_scheduler/scheduler.py index 867034c60..1f7facc15 100644 --- a/jupyter_scheduler/scheduler.py +++ b/jupyter_scheduler/scheduler.py @@ -1,11 +1,11 @@ -import multiprocessing as mp import os import random import shutil -from typing import Dict, List, Optional, Type, Union +from typing import Awaitable, Dict, List, Optional, Type, Union import fsspec import psutil +from dask.distributed import Client as DaskClient from jupyter_core.paths import jupyter_data_dir from jupyter_server.transutils import _i18n from jupyter_server.utils import to_os_path @@ -96,11 +96,17 @@ def _default_staging_path(self): ) def __init__( - self, root_dir: str, environments_manager: Type[EnvironmentManager], config=None, **kwargs + self, + root_dir: str, + environments_manager: Type[EnvironmentManager], + dask_client_future: Awaitable[DaskClient], + config=None, + **kwargs, ): super().__init__(config=config, **kwargs) self.root_dir = root_dir self.environments_manager = environments_manager + self.dask_client_future = dask_client_future def create_job(self, model: CreateJob) -> str: """Creates a new job record, may trigger execution of the job. @@ -437,7 +443,7 @@ def copy_input_folder(self, input_uri: str, nb_copy_to_path: str) -> List[str]: destination_dir=staging_dir, ) - def create_job(self, model: CreateJob) -> str: + async def create_job(self, model: CreateJob) -> str: if not model.job_definition_id and not self.file_exists(model.input_uri): raise InputUriError(model.input_uri) @@ -478,25 +484,17 @@ def create_job(self, model: CreateJob) -> str: else: self.copy_input_file(model.input_uri, staging_paths["input"]) - # The MP context forces new processes to not be forked on Linux. - # This is necessary because `asyncio.get_event_loop()` is bugged in - # forked processes in Python versions below 3.12. This method is - # called by `jupyter_core` by `nbconvert` in the default executor. - # - # See: https://github.com/python/cpython/issues/66285 - # See also: https://github.com/jupyter/jupyter_core/pull/362 - mp_ctx = mp.get_context("spawn") - p = mp_ctx.Process( - target=self.execution_manager_class( + dask_client: DaskClient = await self.dask_client_future + future = dask_client.submit( + self.execution_manager_class( job_id=job.job_id, staging_paths=staging_paths, root_dir=self.root_dir, db_url=self.db_url, ).process ) - p.start() - job.pid = p.pid + job.pid = future.key session.commit() job_id = job.job_id @@ -749,14 +747,16 @@ def list_job_definitions(self, query: ListJobDefinitionsQuery) -> ListJobDefinit return list_response - def create_job_from_definition(self, job_definition_id: str, model: CreateJobFromDefinition): + async def create_job_from_definition( + self, job_definition_id: str, model: CreateJobFromDefinition + ): job_id = None definition = self.get_job_definition(job_definition_id) if definition: input_uri = self.get_staging_paths(definition)["input"] attributes = definition.dict(exclude={"schedule", "timezone"}, exclude_none=True) attributes = {**attributes, **model.dict(exclude_none=True), "input_uri": input_uri} - job_id = self.create_job(CreateJob(**attributes)) + job_id = await self.create_job(CreateJob(**attributes)) return job_id diff --git a/pyproject.toml b/pyproject.toml index 2ae7b9476..f5def93cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "pydantic>=1.10,<3", "sqlalchemy>=2.0,<3", "croniter~=1.4", + "dask[distributed]", "pytz==2023.3", "fsspec==2023.6.0", "psutil~=5.9" From 320be26dcecd5d2355bf3783e89964df4df410fe Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Fri, 21 Jun 2024 16:56:33 -0700 Subject: [PATCH 2/8] use dask futures for files download instead of multiprocessing module --- jupyter_scheduler/job_files_manager.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/jupyter_scheduler/job_files_manager.py b/jupyter_scheduler/job_files_manager.py index 0e39c2b76..fec5caee1 100644 --- a/jupyter_scheduler/job_files_manager.py +++ b/jupyter_scheduler/job_files_manager.py @@ -1,10 +1,10 @@ import os import random import tarfile -from multiprocessing import Process -from typing import Dict, List, Optional, Type +from typing import Awaitable, Dict, List, Optional, Type import fsspec +from dask.distributed import Client as DaskClient from jupyter_server.utils import ensure_async from jupyter_scheduler.exceptions import SchedulerError @@ -14,7 +14,10 @@ class JobFilesManager: scheduler = None - def __init__(self, scheduler: Type[BaseScheduler]): + def __init__( + self, + scheduler: Type[BaseScheduler], + ): self.scheduler = scheduler async def copy_from_staging(self, job_id: str, redownload: Optional[bool] = False): @@ -23,8 +26,9 @@ async def copy_from_staging(self, job_id: str, redownload: Optional[bool] = Fals output_filenames = self.scheduler.get_job_filenames(job) output_dir = self.scheduler.get_local_output_path(model=job, root_dir_relative=True) - p = Process( - target=Downloader( + dask_client: DaskClient = await self.scheduler.dask_client_future + dask_client.submit( + Downloader( output_formats=job.output_formats, output_filenames=output_filenames, staging_paths=staging_paths, @@ -33,7 +37,6 @@ async def copy_from_staging(self, job_id: str, redownload: Optional[bool] = Fals include_staging_files=job.package_input_folder, ).download ) - p.start() class Downloader: From fb67a585b0958d7519f1d032eb85d4a2c4b3d2da Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 22 Jun 2024 00:11:39 +0000 Subject: [PATCH 3/8] Update Playwright Snapshots --- .../list-view-linux.png | Bin 32894 -> 22535 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/ui-tests/tests/jupyter_scheduler.spec.ts-snapshots/list-view-linux.png b/ui-tests/tests/jupyter_scheduler.spec.ts-snapshots/list-view-linux.png index bd7180c37d46cbba3891e939e655aefe64978155..d3b01a4a84fa991bdf01d515b0557f57f0ac48e3 100644 GIT binary patch literal 22535 zcmcG$2T)V%`!1^PZ2=Kf1O)=vKoJq?p>I)POA`b_2O&xe9YQCuP!uId?@hXdUP4Df zi1eD!i6PQU=$&vr^mpdWJ#+54|CxKvj1yt8va<4hZ+YJ5ectt6LtW_{Jv;reW5>=t z`S;PYW5@m~J$CFw_L&pl$dBfmdEmd}F3*${j^%ekW{(}y{r2S118tAtg|X8f1dHV9 z1K+K&M0X2Icgyji*dfm(%hp~xMlrv*W5*Tf&MTbb^JBNU`nN)eHDj&c#jEGNo@Qs_ zx)C$Y&=~WeV08FFB?{@9ZrGcb{35Xty*rzg%yk^x zgP`n5%HbpEDe&VNJa{=qz5-sZLjD4;esJ*e%8>!QzTo?PWLpe^Z)zH=^_Dn#g9jfU z1OLTCc1&My?=x_*SJ5%og}Q#9SdX5H2>_@3Z!dZtb94s{Lg3|}|Ia)4zkae)r|{u( z?dp@!$rg;$H<&-1lUx&ZnW>m*T{v)Gouge~W0QRG<5N~bOQBig_3PJ*4_4ovX1b?W z_%pV^z{+ao-Gv*c&at~5>>>qHX~=kCDJid?Uth;^Gaan~TdtH`X$so0|E=~KWiH$u z@l=Wm-_2JkP#;0gGcn=k-z>qdAjNh4D1=IOBit+#x15Y+ zXM()U(ehhMQpaoeM9Cie+beNTlI48&4-T?aV~ZU~b)}`EB*9py(?qO@B{s9$yh1HU zT*h_g6Prnu>ukPpbqyZuYlBl%;|l-VE{2iw51=R%*<<GEKUL zXK)frwB6MrLx-pB=E|l*sACAX(lf4ZTWTct#-S+HE@m#P(y!8rOWg^!}4|#$@CiZ?%j4F2)DFb z`|iqc8NzpOv({_l%ZOJnn@HxVsuMjbusnm8Xl#RBi)B|*9#8Upvo{u20z0#PSxO;XvR2(!1zu&Qa-2B!ljj_s zK5i_^n6qN4N*2POUMG3abv-KnNe)w_{aGIth^s8Qhf0OW`)p^^aF+wH)uEqOHOM+n zE+c|lYEYUADTM`ZE*j2)hnukZOraFwMi}aioEG2eo4Hf83Yu=}vR#&Pq1t&vX>QYb zINk)2Z(~A`>JAuLFZ5u2jSBN_BV#csQSpH@a zau}^hHS(D4<-xIW$)k*NG>Z&fG+Gnn(RH$rWO=lSes|5N07SYIwxZdVvXJ{d2iGlp z^XAQMM*QdXO3N-T+{VS1THq5l+KX7d`?EFU^coQZ`Gi%O-Gh>*O=bii}X9Ev*cneh;`Cq=ra=Jie`yvEY8DwqOvS&sjB1t;n%*b`~$Dkx}L)|)pYn9 z;*a|io#l30*19p|K|5f;wAb~hsCg5G;pE-F`Ir%}BV*i?Kj@gF%tm_s;5S0mqsN^j>Xd9 z?Nw-F?xfhzs;Zg3tPu7`WW5jR&0Co5W5<4GoW--x)J_-_Op;;H@k>4h!`mcUC#_@i zhx&&C;0hgwwG zF^*VCe$N6XXVesd9&g{4<0#|1Wry$yeeg}WnO$$y#dWb|TFf%^U*R16D#V zYLr&$N~-fkHktFrh#{ibVLd}Z{a_)0vkjPsfsiJ!-r_gz+|yY($o)DmD179O8fH$@or!f+ z$l?B4*JgK4p~u>kT8O9DsH6$b-gtBA$jnir6;9G_3(FTU+_Kfuc;zbP&tx*$em-*1 zvBq|&`0kjb8v|ZZS-G{`ZiH7g<{yc_k9}3)#BkqwFE(Hb+!<$y!3SVQSYs1*mzOh6 zgGjCBr?b9LyE;%By1A@b>mUf?{)Oo6?a+zt`1eyc?5AQWk$AZ1H;|TCOsKJzm_&QKKJi-3j7WO+k{lh&e4Cnw`sWr60uI2H@e1EwSU6G z)Rlb)tyfr>(@gcCs0dnU!MC@k4Od)uRwtG7>FDTMQ(8l(IuZZ=`z|1_KxPD_(Jd0% zL1C`ed#f`^c5kLd1RrN-V`KB^(f3c+9=2z-LM*d2pw7R3jNlw@-MS^HU;JXc#uK*Q zj}YtXNmBwza6}2}mXPN;wY|MUp+C6wG|AO_YS72f-{i`rOX*F8b$Ja=wZ)tmq-UY? zq%zfUq5Ri4SQYnBiL(R#lHY%k%B0VKW&j;F5R$WxopoPp6FDxobpqmIUrx^ z0$lMhnIY4_pKOpYGvn*fhRY)QYDs<04l~%#_GkAP&oeEJPafn`9Ve@%b{@I~|fI zt?I8?4hz;?WIj~j&b`m8o?zt<_b=y0ZJW(>ryeB_kSbKKsy}%W3rw?kDqAy+D?|QL z^~*NaCS^1l?XTd%G)S{!@ycsRh^?eBu$Z_n3<{ZlKhvQ#)W19Re17E4CCG*e7t`ax z#v+RCA_*o0a^Hu`f?fsHJ4x^!(ePVOiH3Tqq~FU$xXR3{+KniaISQCdoEWi55V1v1 zm-Y7NX!-6>-vIDXFpy$ov(nYmI#JrVmge_GBR5@L>j=v^b#5$^)1;kp04g}A*H~LS z%_Ah_)R(D7!G~n=fiuWZrE2!X1c$G)tfW9s^o~$cq-L6;TFm`#KFs%z{{Fv&pZ|Zs zAKSK8^3}OY)@t6PkBGT`gi(K$|3_Q&k=d<`V>c1!~gzk{Tck9KU{hY&Ac7-`RHGR(C;^2dVB%H_vc8Eqw0UI z=m-DLk?WBET;dWr=J&gTKL778wKb-$Q?&BuBeJ7KA*L7=4kzB6o^ihhjtN@nl#_Kz zPo1CUwI{nQ9ASU1=%wGV()lBzb%dhEOLKXi5TL_;AO2i(DapYmTqab*iI4K*D}Ny48f_jH@^?{xy7d!Li$?u# z53>~GKe#iyz#F-$F71%!kUp)AJF(P+&5Yj6QI9!u*!1$!Cu`Hg3W0ED367Q8!_UEj zncqrqQfO_Z164QLSn2@nP%&AyW8$TSpzra#gE!JqmL_ZNoPvFluLhMSLmg!#MRx>e zB`X?3{T46q5q-_i*#`f+^XvH=R0dsUf=cIH0)bi@TiCtpv_lp_AC6n%8`F~N^a&hu zP@g!hsnU+qNH4AAZM57Ry!n!~h(_%)xebT4HaWDdXs+1MYJIkwr=;()W(?P*7AK1s zgqSXWn;FtKPGlbVEH>`it&~5`3pw00>h^z!O_~#Cz;D{E+XQ}FbK~?j%U#=VaH71T zCb?UktzCP*-kNB!{j{UbQS~AlSEpRm&TQk#cck}Q`_Ya$%p)ib-dN;o>b2`=OHvV8 z$X}d1n@s$RhOj)}mQ4+g$gNM1HBDKIZ;!FxNrTzv2=jqFaVGaDc%b=qtz zsJ?lVCY81zFdaGg%QAi%(&$8cy1Nm}VTF=8@3uyg+g>fO!MUHs8{9+=eYvJ05xpoi z%3xX*{RoQUip=2r=@|%5e80i!N5MuJLQ!!qQM--_wVdxA6Ji@haDJ4#$bqf`6GyxP zG_zP+9s*6wSgC|UOoLQJo@%vMuUF^W{ffOA>D!*Z?qf8Wwaw2|yva%#Sb$Av4`eQ{ zz>-&{1eKbGcMiMUt7~XO%X#0TJ6;@8N9Z{}v2b~(=X zgM^?p)&0_N)pvA@O4`Ft4pxrG5`-XINwt|@Zyj`m$Xe#@`b@PR7IbFOA8?8AF%TV2gv^v?}6! z!Mt0y^#XzMt?V1!DcxI*jxt`8op17X4?|8RBJIU33lc3G+mIY9?o_uOJnxtxCh_uS zx_)W#;pnTGEu|@2-(W~yjGEK<%f=|k-3lS$6gp*E(eZV4 zOljr>q1DsCz3i3SdulmESfVy!zZ9MuDaR&3x)IvXEvgf5ST8W-G~++>H%?%lSUxVK z?U35IkCs?Wjh@frWU`$9BsgyN1MSmq8IvcQ$ARQ(Tc}93MCB}4JL3g)k&(I4R&4F% zo8zGii=&=%!fFLu1Vg>Wg~7k{$a?V`x&9okggjg~54S5 zLow&|vY2Sj>&T^FBrKL5t!;mOjoo+RDXmD8J}?3l~OD2Qh@WMx4( z$o`%8OL)R`M~H7oa)F)~PaBi2_JMN(qcNw<93x}E*nYgb)ZFgs>cNXi0p#CP=aFae zHm)aOea2F5HufU@d~0R-rpcI!+`f<#8p)Ecxm1u>7p6i2J8%E(gWLGTf+1){&)okG zX^~RNF%%^ay5fzK{BG5TB85T4;BPTW*P&6{64ZEYPPzY^bqjKL)N$?&FO=DnG&<>A z?UIl|s5~FOyx9B~draNrC-?e-=$Y?1Fv7Fhg_oKAvuF@X<93P=nsXFj)#;&gc7($? zY{5y#_MvfVod=aqAcCr9Pv%~-!HJ!L;N*~lM0tHFWa8@*<{IwahXLX{N*}1ME zC+%sj5M1|acBE@kUvW-kJKaA8yIo=1$nw6dss2<$>GktXyFSI zpWRG5NfH)9^?j@KUMqMFA%D?F-}_CcqCJObqa)TfOI8E%#aLT9K9b25))y=G3$_r- zk3+H(WokvjO|paY$xm+y5ZBqerS9-;Zlf|-tqGx~5s%9g7}uPpJ-R1?k*C#rA{9N1 zzH}aVMUF%vZ4Gf(=G>+SxiDp!UhZq|w-zssvo-8>7Hy%DGBS!+gNKLO_4)0kj)Q7} zMR=Xn27+W9PO+DLB@02Uth;%48q0HY7D(O}h^_nAJ0fn^!y}?ML^z0T$9?vOuOIy7 z<&xCB4ns@UuXR$x$^2*`gR?1HU#1W3vK*pgb^7u%goc?&+>-N)>!|jk(~drsO3-q~ zr(y+^bvCztsrAyH4QH_mah&Axm>Y`Q9fLfjDNc|Pd-j01AkLQm#vR(J>b}T|sST0# zmMpJP(&zjzLME=yvrWx(??-B=MG^kpYqE(F&xYgY2U{vQJb^Gc$-jFXy$3%EzSIWpw`0=hdYmt1g;5W#dvL{)d{xGf}5xEq&TZK4WVAm$7lk zb5nuQ=hRF6@iFjm0Z8r|avxDl$f@km$WnD7sf)Q!YjTp=?q$*8tFGFMn33mhShl%( z`g}4Z>wQzty6%$y?3MV|xJ*M4u}G0bN8EO=i~BsOEs?c08KWbO^iL$iS-KpN>lYI} zu(EmwSa*&%my)mq^|dy}1UGNq?<->)1_F7=tZ8m`WFMVk$+Il$u}+3 zhv;Ny7279S|HVfS-8{DGh$Y3%OGI9Li@cic&|^WkmumF+c7a>W`3k&{{LtmERYn8I z;wt6v$5=uStHiaZWwO75n!QPKx-_@>tSE}yB2Im~vP^n{3fEvfmme%v_i zHlcXE{{CFpFztfuV?WJYx9v0dxhx-f{weW?iNX@Hvr@-}S zU|CBPX?uaSEYNn((xgKikzp}@BqB~d_^0otPEDagT5c)DTft{`rf)ZioTxOb1vq% z5{gw%xv74i71}BWWdOptSAjiA;4ma)cb}Rm3(!x#m^0#!nb@K|`p1=HJ*NhTyGly6 zVRf2!?WKG&S^qKq)~|M2gaJP_BfhcCaq5j=^~Nz6baT(OyWK7@Pxl}iH?k<{K8B4@ zD7lxNX*)^+S$6TJ*Lz~HOvhTAT)I?O*3?Z-+iF5S>)cu|INPB7YRI;RawVDO&+e%RoLp1`} zjPK}BTizsRW6=mXlVtNSx3AC3f7~-g-@4^6tIza`aOHzqX%n1!b5ZC+`zD)-lBf4> zmO2=Q=xobnX;k98x=hl7O?E!?JPrTo;tG{R%P&4fOc*RYU8Y4+;4EKGd2`9@J1wC$ z8;kGasmqm$zm@nICBMBmr$FjCynlNdNI|x(uc*stu*8_wuEtdy!j*Zp)Kl4 zqj^IP=GC{h8X;7OA=M=JF#Q65xwmTNq-zZC<~FmNMMKd@B79!Q-o?!*?JqNl#n@&i z&4FZ@!>sjs)A-&1?_EdiKiHT$SM(?G1SEfx(_5QR?6f^TZ)nnE+5YAyK|x1L)x^yjG8)$sI(rlOovRCCSl}FALSfJ8ICSYKCq|!=y7$H6vq-6U zZ~&n)cL-3pyjo{HNbg%zp(r1s!_Zcwo2fp>54*Q0Aw=YfZVIA*#v@U^rm^U=%dC2% z>dc{4JA|LxitSQiDeEvt*;!Bdg-MSvVk@gHp&q0dn{hFvF51>^Vn@qUgpC^m^(l9* zTECGLE_Rx8>_3*{hN2e66C4+c+A({cQBkd}5>re{Q)#!*I@P{E^7Mf5q$M5}J=wCe zyqEi=%t3oeXgKe5VIX{wtXykjxOgf!Kn*9SRJOIhQJ*)^;sf1YScGnf3@Z{7AKN{8 z%uhYp?alXiwvb>Df3mPZJ=$7cdsd7nv*fv85aFgUZ|%Hy%evH3YV#OIaO(nwGiz&1 z6EP0}d4R5KJQt?VPuqUj z$B$k`MgA@g35(T}2AqkuZW*Mz0I`&XbpGS@o@lVrlzzoJ+hGDP%Las~rBd*KTn?RUSo!K}w_Az9Tu zthK+A{2o-crt2-vDc4827rm(}7G3^$i>c-=^?5xTsy@BET^>YR8<^!p-X?Ib=N)u)Oj+OK-_hS% ziwG_)9yissOp@F*IR=(^=H2!dlXva~SL+vYB^lq-GJ3D2?eOQs7?1r%zagL(=NrJe zf#YDZGGB}xF2mNdRW0ss7E@L1tJMN>e{y0(`iw0FA=2eFhR{u?ieQ7X#$7p7hTAzq zobv){KYZuh+}Mhgo3MQ_Hnx6^5HZ0SaNBb$p{&&pjz8I*L>=qVtXzqOuf^76?TDJF zGvPb7}Ii_b`Fz* zTU#a%3h5uqs_}X2yF&^rw?0GxjxWhfxRV07*!wdpIGs}Igq#BBz1s)%tsIbu8*)Cw zX02hMGC_TFu#ww!>~_;yo}ITbYE5z(IINtV7OAy2(5lS8_vlu{Eac}3Pu-;rnIpci z|3i&3`hDH>ww|^P`CP~$Wk!e0wtwrYN?-j2edgqM>IaPy&lZn|e<)icp;!EAPZg58 zjE;!7QrcmSr49)@wzhSyx*Q_iwHMx|z4WEY`{_Z4G$&#sIf8K55&D~LoLm+*wOLwb zi+uadbFs6?=H)h}-_VxwmO#$8TyN41JDXuSQ#NqM@ejqRy~&>yNWn`8J`AT_+=lMC zLBoEoaKC}ngxe%#cK;pT=_LR05AoPKPu44=SSV*KGF^J3Bj)~nVC10U%i=qU-|QoQ zn0ND^EB3ct!01F%dQT-N@4M^@yvU_pyt1XE@fp<=kp741H2LgkJJ@U|A**xcI7LsOL2rG?|$<655GJ4 z>jn15?wj8<^QW!<3L_0@>_5Vc|AQyX!=0R*K(l5ZblA>X@z(YfzidmIuV&zuGom_! zw(O<~_)sHOOjnX@yx>@O>O)ynM(LN8%lC|a?2pEYAz+!Ru{D=4(_P60(e1G!fk*WE z@QRY~sz(|RkD*4V$U8dg4u`Y@I{cBs7op z2ILkGk6NNsnMFt3;}5JlXO@5-!pbUpW5mY*_BPc-`>FIleYF3&#?DdI4kUbLyW(>ie`IdTrLfK8^JkhuwRqGo!I5@Zh za!Ix(B~sUcha{%Pe3&&@Z5J&cEKB^4eUOWyNsI`3&Y8O9xvI7Lq}$-)n2h zIQ`!X8I4+5>alcDf~F?5UdUj5n%|Rzp#F|HF@BAdS0-{6?N`v~!vRyY@dF<`hI7+# zH2q2VMAHY*&C!f4ei+Eeu2t=}xV`Ox@U&=;NezwU*M;peoH^62x_-3ESUN^-IqXdX ztX>iAU~fJ+0xI?ZXFoRDPz0-d^!}oOMac9MpFj%3$PMz^7zE1iT^o!93LWQ1P@*GPW)`!qWO?cpjBbHR9ybdmBCI3k zU~dbd3X~4mN=Z3b>b`D;BN<2^^x;e6wT3*2v5|CmVdR`PBJN2zuXQ_)ghHSgn3!~v z@(s$qm>`-TDJtHHxpNM5;znM$HyNq9?58mZfqy z=o*X4t0&7D_W=w0F6ScR{O!$2b~)63MS%R_zRTX<7B8aj8Ot=RfVTRFOMJcAnm70< z7QztAiYqW^lDJ)*FMEbhuo2Eav89!LsV|F3`W}!tb)Lib7QylRJ8OY{ejb68)ycMc z+U`7;phT?T{rlYl5YtL$p5*GkPBZF}D}dQ^iRh*i8*cLOSnK8J_UXBoom;#8{@*VA&w@h;Px135YVML4=_#xaf9^G=mMC}{EkPs-_ z@tR6hzpzy&P!W-nfa2;Obq4%y(Lb+GERNu|81X0+3OU!m;;{pKUbF+496(={9Aluz zY^?q{EiKQ{c8pYJ^|3kLx}DN#F?bXXf)~bXs{_FGa>#1&;-2kC<}}$FSrSNjmZ3aL zY6-t(ndvLj;1uw|%pbm|#PV3)=GV_}Z#zPOzNyb~sQ6{a;|30gnj#y*oz0^#04lP_ z*Pg=FlVoOnr}fM_DU)pDl8k51_Wcd|ps67*f9xkxz^)J7TCQ`c0n((szcKEZNO$1# z6jjuNcuFFi`@_m@7k5lBP(hU)x|4jf$_a5catp1%ASC%A{GCU7lb3B#pb!M^%qHi8 zg&G0_3fb5FqMf5_;>mOUdV@QZcs3by#DUwtpvA&#Lp=+>RZS*uCJm0hHM8HI3DdG<67iIQ4ArR zKH4uje9$ZZvzNtR6ZX6@Bp?q2a*_3gp+1T?<|PPraX$~XNQYDL4iSj11j)7A2k2ux zSJn=f&#?(VDp&+twNoImK(4Uv%j8jxup-qR_|Y-9b9nl6tsJdXD2U1QAdvX9UuGCd zPAjaNt^l&7G{xZFHnDG($-Wj2yn9;^Qnh;l>R|WeSytET)V&|G-bbd!j2S7lPV?DE zRsn+^sh=K{3FrZ<1e8`Ex#elVFS~`0d<#Dy0Gh5pna<)Ts8lLYH)N2O4mL*|5dGPy zdPYE=)|n_hKe?%E{l=^@fbSCPZH=#755-J&p5;{ORm~A{`3;X6{#=-j!%tg1O58g; zJA8anG$zmafdbu3Q4#C$njK)RGQ3dV-O>qUOBhdN7z+ki=j{4z! zG%+Y=V7qsNIaVLs`cYJ%1Co$zeMA>$p-;z!b%xkE?fBySlpCGvN(R=DqV=jrnsqKlqwz6NpSM z-caHly{E9>DCbj$BZQk94KmRvpx)^XF8sdn;>KN3lkv0oSkHyuZ$8%Ln2?ZwRb^vi ziydh!f<55>Ot^OR#b5maAYaci;c@0#_TwsaA?kII$Y0zZWB7;>R3*u|Gh@MjY&g6t znLgy!(L1gJ+!{wp==*BO+{u$6hZVjT@%#1wV9C9w#N!LybJ`bRJ6|j}v(&P@{pp=w zX#8~$@Fs;Uw~|5#1#cW~mKr+_?Yr$1U1>56?s)-ZBQc>Q+mIU9TTj+vODU|sSErH} zA`#<7wsCbPKlsKz?tU%5Y0Hd#dyAH}W%_RKIV=XK->O(i-c4fXw1Gx+?tx8ZXvkr% zO?|3mJHuXnmH6TFh*Jg~)jm^-3a<;^9F9u-s~P|MK8?;}9L808>f@dO6&JhK4|%BM z2D!6aPwQ5GGe6aUTru-g1c(L?d$$In9iYC{HEynYvCDOV}0FE zi0^w1=>Lyvb|g~V&fdF~E`x>c+5!Prj&;Z?lg#3Fdl4NTP4k83%aKPjq@SQ3a*Bz~kAH8PvlL>f$L1 zkpTk`;^5O&HdrG zZCt(V2d6O~x$Aj=+QENUA;10UALwuU|BvYa<;b7?7l&k}r{C5BVmf|YARKtXquhr_ z;PxN4Uj#s4Ab(rW*K3elH*b#C`AYve=RckjNa^qI|Ni~EwEI#72TE#-7cNlqKaBLM zG?$!DwrFfb#3fEnS$Nw}@$oEYxG>QiK|ZqhJar4MKeq&~9xsTkaHoy~5C{aVMXs~G zO+e9OP-^95U-+WsqCg?5;n@P6AN#sGzrE*YLWq`@7KK6qGU2>;Y!@$Hgg|_xj5Qc7 zqBK5yIr--kV(aaGqx4rU&~mw`%N!g)S~nIG6EkC^@cSG??xq_@p7GOqt%xO*XXkx7 zq}|}(l6GEJOF6dREA}bpUBdheG1fo7Or|8P$#zIx{GVdeA658Qm$P`LBiXpjdQr^+ zaCszG&MXkhAA1F9zDc-yu*E&ZfG1uaa~j|T2PI=S{PA5vkoj|AVPQ$DXMY`?IIR|{ z^G!^}9a#dw9DTcq39F+}{L6xZq$Xw%LF$#A!S?CJ4khyhR&h>^EvUk--p_ogW^;l|S+ygdcPZxW3_{#ffW%}9oF zocQZ6lIL_HK>IdKR8l=!RrMovHqx|&Q*X=!Y?d)7f3mZ0i_`$kat%ACESeDlEmZKQ zw^;@S8h}=ka&k0D&Z|$kF(pzi`{e64ZpiN~lnj`7=jj!h0|cmF;b=C+G?FD*YN{Y$ zP}=>{=h?Hc5bG!bLkoHCMN&O!^!xEMct6TC3#KiC&jrB1kWiom(Xa6++T8+QVZ0uF z2mo%LAuivvzK*&?^Ynvr>hDHwZOgD?;BYv|_A4#7lC3f(CMNC~Rou7gPDxV=8P3vV zQiLj&m)VV!gRgqw!UZNS8K7eSzPm9`hwpmm;o5ShzaC%bv7H$buQ)M2nNRUDWG%DVVnR4N>g zv9VFZzVZN*1&9d%0Ox8;uK-D4Su-NsTF)WXB_zbXh26e@27??ptiBtZLcAE_5J+$ zJh;E0JrfI%3h1?fVzAdF-E{VRU+k>r?jbG*l0wl$q@LT2z&$rTw&*dZCsW1P#}PO7 zOqUmwOs+=y_@wW9$D9^KT_@85gsW$OV6YYl4rx@%%tK5Dr}aR-G1m=QAbads6bgn1 zVqbT}OXSM;pG79j_3mcaO@lH~%x+lp>ea7FzDTjpTJ_eDL|Q{bL%;0Yr2<|;>cNjq zJ%4!n{jnTOHyF0i&NDy)xF~baIyLB*XFtb?t7^{}a&%;d(-7$5%mJ;6B&?WaOTJfLi!Y9XXNN z%Cz_RQk1ppT`QJS6Ab@6F?^8eJWl;#wJWGsN*xdq#3utn5b34EQOQnZe`W37QZ<0t zxcuc>Dt)z`{hla>v)HZhB9AhVtIuifQl1K~B|k&vn^ric#;Ht0`=&`Xo?5{r=1ug8 z`u)&M9@!;`A$d^9B4$Z@EH2>2_3NS%B^Sg_uide$0Ls(iaAwRQ5QuB_gK>fO9~=_) zR+c(fge?mM49jCyM#}ASb#kNJp-E7u&urtM0`a;{b?;d@fAOL@FPDUU0$UCaLBz(g z0hN1cX3s0$zM@p-R5p2saT!kC%6ky>^0(74cQ8eO%H?ri8ntfbKOGb0Gey=T>%{k> zI=c(nBZvokpf11j#G_qYMrM@WGH&p0z|19c(hCchidvQYl!KwJt}b}`rdbIA@aSdw zdGRyw=8r{r8H8<@RI4aAnYGy&}@TVV$eb1Or~v@0YBeD3v3xO;b0tLt{!s(sD< z|MdrLdk{35e^Eh@BYJ!WrX{7|Yz!?Ys?f9E1r{#Q7glv@*{a2o3-i~B6X4G;#Zrbz zMxi6goH*fxRlhgJpe7R22QiPJ6YFP|Jm?f&S*v0{JCRK5bXu8c${Uz__HFcp=M=dD z6q0dYZ7l7gfi(CQFRx9m4v5k|JF8WlPgGRe$mf=TqNXfMG9PRB)Li}_i1fKW8Z1t8 zdWsV>wux?VTFNLC^IEssP@i(Tv6nhsUI|(w7CsN1>lq2;^&AZ*1)_3zg0OvMww&y}e?+*lYA@h0mHtQc3j zZ6qpS8M5xFzW*k3F)ir_y$f3a9Cs9=FJ9~q81DkG?L+sMjis@XkM%w~#ROidcrO{4 z*;2|6z^DRf-CiTELf-CsDqkk&PE`}^=}hzG3zXPipY0?3RUzwQyJ-DC95xLkZDQzW zNuV6T9gy%nfeBAJwBvh6M~k?0M1kG~4jNH}y?3bKOnxA#m z{&M`Gf&yr0=el?FCc^QKwlfoGfPkv*h;uOdQ7&_0HVULpCaPSz%1r=hm2e-aSJ> zK8=*GfF>(ll9G)8#KS(Dgh)ZhGw67wOBNeZDF3lo@_7K9>fMe7Yf2S>Y&<0$Bp>AF zEMOTgZU;!e#T7f2P1z2omt$TR_l{2xt!ES-889Gz7XFZvCY98SzHys z_~$|+^0AG0)J&zFeMn;W5-q%pv1B<45xI^CBBPnY%Q$UA#XuVcr<}?guZj0@r2Z`y>VnXs!Rpp|K*|9 z-$Z~P{C7Qj20u>T76ygMCyLYb(KLidkvqoP3jFY2s7=8%?f>&yzh`*=U*FiNLEAz@ zUHGPWh8`c4GASu3?IG^#OV!k+;j%XPFMxr-sA9WiS;BwH9?A6M$Nm`yRgk;hn!HU{ zy|YLs``YGZ4za0z|J(T4153_62iY%=<|AW03q0dH#WqOx==qEh!Kqlsh3RScUq3F) zD^2!<$kPV#>#|t0d++xJQb0Q)S<)rI@uR%*qeqVbgH}5It1X%e0s~il z9`IB>y+P2y8Al)bay}BX>0geydjJ0NT9^F5KLE0S(aFt>#Tr)=<-P%Qeg{A}&<6pK zZnV~W;G-=Ot1lPtKJJPCD37N305AK%DqxVM&bfDd-U`%U8*>DH?dTq^Y>l|B*#M?J4@%hBI5C>l-m4BAb;4?<@ zxk`RT>#*+gx~58)#m5X1k|?@i;h)Bywr6V=+L|dVHWfNG|Yy%FuHjL87VowtDq}n z+!wn7$msV^ORaj$SAVsz$3{hE?u4+5DKmG)OB8_8iG8`~7l?$zpzvY+%7sD3pg~*( zO}+}xH4DIq0{Vp zjxflAZBP*8(<@9F2D}}?iiFzB+gWLiZ|TcU%o}--{)ueCPq~oGPjG7x6lWEBfaGO)GuCV<`EwCRx~E^9J_X_8vQMlR zGfcdo^zjOLuZaF+1Q_-OT?jRmXZ0vw7jrq}T*PWVAr)ai#TC!X0frmLv%whvXyA6C zYX2#0RjAqDQ_eJygWKN)1s!cPB!Gd<`~Z;ciXo2J!fa#pf-)spZ|G8QMq7*!yxkGB zOr2AaYl9kb{r|d+#&qVmj-5_{iL4m5)+Jufx5}UZG;^hLJ7a_%>RyP8&y+A!a`o*6 zLx7-+C`AmYQ=tIO^_7)d_f3JGfyNDlUhU`27GNgPApD~bX)7x$7Sp!Fr8!;>9o=_r zjEoXCO_1qx%VJMO^zG9Tk2$Y`bXL}Er3V+4>sokj5IwHG+6$N`*~XPQ*=W*2)zXil zk=-H?nv{!pA@lfw;ew&nwF2}~+NUO&>7}w-2aob2`X!)Gt)vpxyq`ST&eb&t8iyFXPp){IZxY_ixq_VPTct z=}CXQ4ovCn?{p~4nQrt5wpa(#a+?x?3hbibV zffAcuR60+pcI9XEUa)D_&DJmjwH7R)_ZjE_fae7~`HNUP2ZBhvj#j)?4Qd`v$7wlJ zbbn6I^;NR8yL#YJxw0sb89<0!1HC@ApP*IO)(DVB{l*zQC@X{$K^W}A%CF?OAV$jb zKIe&BJ%66EI}SKXIgjOk`f}m@f9K@nIMi-LS>(S4k5Z^xZRLkS?*Z@zTI%hydN<^E zXVL(FGG8|Q=7!?OZV!L!J|+as;-QsXq7r0RtBtz>Tp(U~t#m14zTOI9J9vdw~f zJQ{?YL9tjal|Inlcv@WvcIfj~(SYpHz?3zD8D87|?k2wi{5|Mh5NLY_m8jM23ap}_ zw$lv)mrFC1+g&7*syVhs$YfSb{f2Q7t5BH9XVq&2gGHyK_ALD5ZfUN~|m>rYn`0s~3 zOuavYbFA^%aX!h(Y+mI~&Bt{w*(94-TTHf^%3l)l-dq4VO|O0q%yM^53^C#X-NJ8L zS0njO=5{!rjDD<+N(O-m6f&=n?0RK3k;Ep7zrD%~^kK-roHAj5NNM(wzM2Dg(8E!0 zUB;tu_jdS?HAkr1f`e?s3K%(UcT`Y#Gu-<3OQb#cC?QH@m}N)Y%YdV?{Zk zWAZ!Jsh}I_ObvDf@MRn4=(A_9@S0WLl^sXjLs~A9w|5s$qrDuQY<*g>8>#2^91h0n zR&PSK_Lip6h~iR^6EcH*N=HYZ!}?JUU(3j$55dJ7@5|S%+9F```kSd)vZB zDlddv-W~suq3X&(L2&f1Uv}8%;4vM?=kZ`0%k0N?d&1?{^eK@s9XYbOze`jW;HoQr zyEC9j&E)_c(n8*Rtt<@H5%7ZEh?-{`;870k2a%v*o;yGAce2mwdn7Fe%%Zaq=?Fo0 zy-eqYn2_@Z2u6n&f)GHw*MjFud=GD($NK>B_Vk(8^Q^I2BwbfR$b!HUwF zRYB+I6yDNfTJN}a(a1`{W=OHe^x%^95(EUeu`1MWtS$)#c~^m6VA54eg8T3Gkq{k^ zEgTnlkiIT+UYw3VdmRJ_2)p6ZmbgrfRE4TzBgHR2G#lPP7J**89iPav5nV&2kp|a( z_HNq_9yL$}mD&zN0B6RYCXgKr^b$J8=E0oO#MaSDiOC}YkaWyF8Sv1CHNCCJP}Gyg z6$-!Y;o_kl+N}TJkvEj}%EJs9g5LQ zB8`9pWmxW=Fa+~w?*Wn_lv-=eUr^;H4_3Kg9UBwjFQg%8An)NK4gS1<5i4?jt5iRG z0ke3?ABi10lo`u<%ANQX%Y7sgnx3{)OZgdAYl$g)?zIR?>M@hv;+HD(@6!OD(d&X- z=7+WK?tzSaP$U0vs@xAFvhI-@httk^R*==lp`XlipS-QPYq}Zv9z4?G48FC@LNCuu zRt$76ffNBeZh;J@g6Q$Qb|5}>I!SH}10oIpr@@-E)R(BW4hiral7w7|*C&e10Z8`* zdXGA`y){8x$G_fClg+*+XU-Fc>*Es@ewXl$kP9q_+#OMQgGfF}9swaqJJart+)$@2 z?whBVs?E7Llj^3TC936Ue0>iCA zA3w^GYD-CJKD`$Z;`0v`UNOCY{~mDq!}0~-sTO0a{WRNyNd3L^W&}43(*Np7VAZg8 zRhfl(-7AFw_^`Lwb2o>@AfTjpMm?Hh=X|v({YL6ynM(Yv2f_XQ+8t=Xv+okXhviY< z6v}TPvQRn>>E{cs<{l3hV?nGd0zR&&?p)*g!Y0!%z8;`#0<_bI(9qB-`m$#JuY6NL zwXp@zGEhO>=ME}EU+uD^8?&%tyZ`*vl+XP8T-|~%XL~1eL5nv{qei5!{ug<>Y{29! z2%B6PokXvjV*tbcq{j~PlwJeONGim3^R#gkRL%utS>?cC)}a27-v8Fll?F9+Md6H$ zqky}_En#VcEHY7K4Pp$gAhLr3#)1qe63~cC1qlSL0R#kG5Cs}k6ck91f}(=Z37g2) zAcQ4?LQ+rylZFUE5z_OZI*!gbwdnLm`|tfo-n;MKd(J)Q`@Z+dIlJHZ9F}(WYj2KL z;i?tUo(;>kLeqz3uXvMok-(S25=Re%KuP5Wh(|ylP&ggbhZUp-;ry54q zMds?7%I|Z}EhCyR*UCj?ekrRvpE^ggr=OZ_WoI`I;t;6bucK>NFa8y214^{byobhZT(0w0p*$ulD*i-Q=dfid3@6 zGF)5optZkGi58s&5zbqGRoNv(JDZj+;HLANtI3A0MomRz(?E1Cn)vY8UWUu4d9r2! zoOtogt974@pIGZieLh00N(&u!2O#xv(kg0)u`H@5Zrcl!jiUCA#E8?JMYekvq@d#l zJO&Tuyak@d**HV%Hv6~!F1MmO1VkIlN*6(QcQoBpQ}a3?pnZ5C>@wNgF%{-me@lbj zX|_9{jRJ-*rS9r$f5Tg=jAIJGcLTahO0p7qMXaj1iS)I`4CZLD1vMJl(!&A&#z+GUzq?%f`*n{^dl?28f?MLuz6;?VQws;;^2vQ$lNd~HwK0BvHKA31jg0e1q3dtq444py=v)l3_knUMhs2)AuV zp-<_Rt!%5A)Ci|y;s8M&sa!ZDYH6?`%$YfJ2~?;<0xj{SYOcLf}|ICR>MX-HigQ_H=;DX zV6XC*;lOM3|+)sun~(j^(aU3NibU4ByRggj-cVTJ_N2iy7Y zyYGDoxcF&Ey+=4dk4hqc{G1u#KN>mQ^Z(cUNOt(2S2e+%hZ>-JvH-nQIFfn; zA?Ugx;gkb45CxBz>JQ+4;embySpX+$IL1fFh!F2q;K*mx3Tj3eqLr9YbRQ($d{6&Co+B9m9ZhcXu=7-lOmP z?f&tdJ-cVmerM0w`G>h-JlE{U5{5XZ;Lx7y}rtlcQ1XoOzzOXA(U zOQp!9X&YpQdHeD z#X&GUZp?jWa0;@Mq(1ONIX`F_oC7Yg`p;)k;zeJ;w?7{-#ndOb ze?EShFB$&+{DIUbmA^kE4Zm6V_jkO{OeDQ0OaK1!joR+c-Fx>axwxJ*vsy#k|Ni*3 zy%Bqyt+TW7#&F@7y|N9+8*ZBXV!QbB8eZ8ZPI#c7k30!L0S!mu_E&ja8a+0T>6IVu;-qd|rRpYWp zMg1nEj?Sp@WZM0!kIxg+IL%5+z2%-bbd7AGH=*Q4J+W*-glwN3R(kiQ-FNJ^C;4E- zfdK*Yg;mqFbI3mng1!}UuN)^xbi`K%%>P3uM5;aZ5GA#Z2~)W%_}M1y=}oHrD}|>%nKvB-dUi7L8N@)kDkBXo&AC{ zzp-I);*^}6?6^7Fe|8coa@A4mv|*pbg6gn6ly7GHj^Uy{_g?F`?X$3Y)IbQ|NEkZ}dym+tO1<7ke!gI_ahRt5np*3#BC>rY3&Ps~|TlZ5hIi;XKGnB@=xrwl7D3Sn)} zuxlsgFf*Na^fOrjaX;dQ9#Vx6?VRF1lP7 zPnRWcx@2I+Fz4)dKxpqW&r5W7`JkyeowDP(M;eQs{AL`-lf7Q^hB~7>1PXs>U(@GEZo0&IolodDyXN=pvdtFGFO#Lp*h#6qdV zDKOBLN@GcOF6=`HR+9C$XP+J?OMU1Kd7*++D480AmqQ(4=gj_O(7Ka{mX=nd(y|k5 zjaVFmg=LYrKQz@ z#i*{WWjE<1RVh$=b$+~cLsC-mDHYXco@tjafdnl3aGn{|Vs%9^m_UoBU=XO;0L`8I z#9xbxSq_%r3ppuX={7SBapUe6r*^=BiKPg8zECYnoI0vfrfr&Q_FL=EzyP_)U9{pDs#PcO;GH`I5)mA%6Od8KR_ zq6ehBO?9pZgxr>dzIv>XZLK>v9=!x#mx4R|_P(SlSW2mhe%p zI~m7V=xUu>(`d(`(<0{=_8Uc! zyA@L;9{EFgN`8@%kB_$|Vyvc%v|*+L+4srG$xo5U{sMK5tBV7KR`+XD zs;U|l=B?B9?u7?T_*hosWa#ciDMsj0}-HLV%(D3;p)Qm5gy*&?ry4! z-IOmdH7csHsdqx$ zFkIbn@y;8br-Z=Y02JU`Q(STQU-Xj*pShh{E4bTGZ;a|8<^I>W_ZXDawTC*h~n3HJhJ)5f*8>&A|mC9d(?{CA&es2=GV zwm-V{X>C>kp@V7L-eRhx`FF=opkG$tnIXoi`Y88~}gRn4b1 z8%cIAsv;urXd?U<7Y!#TCtWV~+SMz}pMV{k{XS38akcML_;YC;hyW@!F3xO)V3nh> zRKDoCq##RkY^p$0oox3h8GWzFq$Z|o`kuYj^S4+YUH9FIM_xszF1)2zO95lc+D^G5rrpZH(hGt;*MST&g4aWiv93{caAGz zy~o2ivs#B0A8->)DZy%#db@eM4;|#F3VBm447iwCzra(`o~@U2#nP^N3yT!SLyetB#&nWYH#%R?)6**@ z9VZGvLOMDmKY#uV%og+QuxG?Oo#BaS`=u^6+u4_jN=oW5XC{!?7{m9Q1|ZSFO-nN32+hMusaf0{Ec0iRQrp?-C$Kwp#Du^Nel-qbrE`l zIiyC3rYY+Bl32nc+=mm3wElZbQQz$icE9FF>Lm=sfnis8uW~XJO*KltPt}tDSZc3c zBKRg-Zp%#LgBa1tv`DfLaz{Jra@(IWFmhe@8?fXfvVkOR4c#lTg>YgZXv0Bv^3z6- zlYU2B&&DJ~OH%)QMpd%nV-<{ug@>beSzv{HJYR+euTZnf7;N>p?2#YJflJ*n5wWq7 zGBVf}BSl$qN&ILo?(TR`oJ69FiRnICgeSIA+jMc$SR*-JX!5=MvP-xarrJY(^6t^b=8a`RTK37l3nAYV&C@(^ zd33Biw{+_8qp(VsSB-pG1iQ5xm}S;9@hMIYgKYAPAOexITe`h=Gu6$&$*k&(q+A8F zi#)_kef6C{_k_6_B>Nl{x_s+V@T>C4{3I=~15(@TZ(!@XL^tdC57KXSwvcDt_$L2$ zA=t-+?@3%xLWIBXQoU=bc)!Q?r-Tf*qxAxfiY~WC+Ls{Yoi4@b67pD+%E`%rKsR#s zejf7)eF=!W62ZhiU@sKc9~0vUpFYi3DV)zQZkPwL`SkSk;iE^QS#P7zc6WDyU;5ae z@y38V!ECH7>u9Z}a{UKz+iYxXGVz?9XORomQ?*8#7!(u~$sYSs8xfz2g%@A&RIYzL zJO{;xP}eUXti7Y7ED+bZDeiKC+yDgjFEE&Zc);T{yM?y6E@&0lIABZ0aarI~O9Z8Y z^)I)cI;$NP;PKT%^%PXMew~Ktrv2%B+ZcZezkiF~nQi9_+RGB50Ci1jwX?}7ko7Bm z1Tlb6z?m`QO=#N~GUT#X66Lm9(h7Cz&w97gE_pbxztx4eAz&GK%KYxdg}g-|v8l9~ zk=vVZePaBlD79wyoNu__`#PuZFoXK*ZPz{h0|5uqlF8V6cr>%?>pv*VP=SW3>Z31d z;NLR28Tg%}BxIz)pYGw}y8pjW-a5Q`tTXHCqPn#GaUU#$xZ*A zT;pib|Ia8o3bV=oYYS^Wg4dMy%wB%=zu?3FfMNV!ga{ryF_mv4Netovb@1OV zFFu&{fRTKV4nX%h6^Pmj!++3{ZO8ZA(~XiKcX{Kr>BG!tv{T+itgyR{%kvzLa|xW_vd?n;5Z)KWxZ>AtVuG>l z-SdxJ=l#~Q7&$Q&smXl!>p8unHw~g&7s18Gg1J7~T}w`ROUD3Fs)S@dS+kpAb?RIv zaodu{%icZuyo9tCdqtOPWiRmf&<-+mlziSUhES?U1fnZK+MH^q5Bj1tD^hk^ERcnV z=aoW^9}FweRBAaN8^dZ z{HPIDJr8Z^tc*s(#+Jak()8JxpUeRFmVivfG>bhY75~iAO150qhqFpEjoGEEIxiCW z&~x8Dte9$>-HqWjc#5Z=A{mC=vi|GrB?{u>8(I7h%FeUdz+`|izi7W^AC6EJVZ5fE z@cYKo7~jf}A`9CtE2||W5-V)?y}4%7w8AXD43jC{S*!^}KmJL_Qbx`~(WBLIoxI-q zlR_jz1Ib=hKn6))5lMEOxE1P6X(NH{(sFiEf2mW7)Z-NyY0<1{C_Y@1$)ozj9aA&3 z@|l<1eMRh5ryKb)P0q<;!M^<_!ENns@njp1VTr%Nl!3Z42ry3nz+lNr6aui~%g<~I#bDw@eRe(fQ3?L4kj zg9ryBYNK{4uj-e(PpplhOZZ)DYEF+$nyIT{@BJsHu#tXMIRc@T^&*JwnGlQl;MM7L z+)rLUz{;P6k|N_(9u}PlmM^K0o!3SOP70GaQ}m2nXs(6S`f-olwAqsWsoxYEUsO&X z&8G$Djl+C7lWGC?s*c3N*Fw&IV7J+pHa5P+lPJrha;^R8s&1+ggTW&DoB288`$)H+HfYbXG8J=n6ugKzg6Nmt-ms

_Y3Oh_uXX_0!q7H)aa;iXm2~2OamO6WBwky;FWR!-EOjol|XK7zInG z1pE3YrApZAgW;9;Hx@OD3%A%!4+(}nF<-kQXgMZMeHFu-U zOkiLH$CM`Le3WpK9lObO@*Z$v3M~jfurzjT^mndBbv*Xg@g|?F;$)uOY`7Xa*%WdP zr@CsW`winDOBghm{vmr&X+)3@*)&)WAL4ARx?_;+%BeI494B@XZ}5G0oKJcRl`ij= zLy3I-VEOQc2bL62A|IXV*9R-pbE9=Eu9w|AHH2hh*C)cwtz&VjXO7wKff8}i2p8EN zm;2h4Yx4RF#jGVt(sztUy;a9FW+r=gHq=84MDNtKRTYcK!ZDY4_fOP)J%u9-(oEO{ z;>-q8xhaIMU5ZvjRlxpSy;#Q}<@5B_c=83gm~}OLX|YioAErm>QU8H&Af}L#nJtWl zE1&%=Px$oQg~Mb?BHHRIVr@qdr^)k+ZjKw0-Gk%83~k%EVhGoo!?u?Cxx71kAXz3veYIk&cC} z1kIB#g$^PaiM@W~PrjhVA}9FUfHC7Y&-##)6|`R*S$+v(PxkxXc(NGO-cY}xG;Hu| z*bDI@$D*dtLhIEy5GgHYzDT+xkF?H8G}YquJl)HdMxNg} z4jT@p(=KO41f54ys9yaXHKAB=RO9{9?$K%y>ZG@Gg16dts&f{cC@HFMpWGD`65Jkc ztT(hm9mi^z3O71X9AWXcT#3QaHEAB%fnFCYG{j$Ph8PX~AZ*F7ihl&=ZPyFOBpw2Z zgYDWw3`Y9#&=}RAKGtwzwG~(m??9fECw{?{ep03BB{%WNkLZo6vY2#(HKBRDJI>}m zuPA8tr}64t*PCtbh6c2L8rx^u7^@=C7#-F^Zr^Z8O>T> zb!xv7t9@Ryjf&J)!2y;6E{_9SXGLFg(wUiU@*Wa=7ZzQ4t52o;`K<{92Q5S8_FT7~ ze2M=2UE-l!ODhW%3b;l~Q3kC#_Z3s6$H|TUnBv`j9Fbce(5y&?^|~PxB@W~5kB?54 z;Kjdfkfl!A70SkP>^cMF789I^;wlK!i^aKKo?@Zau$q*EAq)BapF!xGmnNen^}vZt z_WF%hGmr{+hRDu%H3dO*3=hjwP)@}9OXA{+5Fc%gc1qch^!B+>e!+;n`$(qm5JQ>k zH7;5uIj?Bky!dF@b?(v5ogoI}`~`_nF3p{+6+3Ai?awi&xaR7Hz=C~k7Jfq|jePl5|zuA&1$7JW= zq_;3F=1Pw+WZ!!~!aeLg5gt>Uymt2VMuX4p$=Zy_X_ge_wN<7qzbZ6K)qabN!gcHUlvcrto>>hRK9*^3JF;a@ zFjLETbU$BNAKvSU#>>ZNH(RU#oaHXVqWF^ib1UmWP*adw7dg5Ay&L8iH2ux@LFizQ0#T+WHI2fk@9{hFD@If*rDqgU+ z1}dc6zASQD9PW*4EHnVQ$9o3dJ0`TW$(z692MCx!%6=O|K6!3+4O%zWNU~kT97#_s zd3_95FEh)jm^>SzSIAL3t15@D56=$P1h&vVGtR$`Ihv@c>Y8%2>Awpeeu3#~C-3`G z6Q;;|-y-v5lbqja*tov3Qd-WkaAA5O=UXjl`+)YutReS6xttU6NI|%TIEy^4 zkD45&#@W1&mbKn~DS{I#jv_EVE}sm4nAtmYpE|ib;+cELhV7b3UBuk`^?kl%&iO{^ zhpl13ih;4QIBB-Y3jQzuasf(7afCw0EYppB?5k^eCxo^ww-BF~Hy2x)r3bajTBPE`UEot=@WVD|^OxSnI% zPFxv28JZ%2#Qz?48)nX2$ud7TWqMNwdssM~^C<#YEG3c>kzw&qX!h^n7-0yY0IUnR*%J zW#DZd0+U)Un{VJ5*K}2rqFQ{i5#R@7;9`i(Fjp|!VDqk@c*?_bmYf(dStPGMHokK_3Zvo5 z)$xa)7u^{(5F9B?;+nOd_Kx)ByFE@KG;CQPGY+eqs7=1C^Acb=+5QR|GYTY_E5$}U zs-8xWP$9Xh9TtixO6)QM3HClIl zxAnr4?}>Am3T1^o(@sz8U1##E!1ENhZvL&e`j1Xj=t4UU_16;+Y{0rM~@;nhLmefpX!CxP*hzBj~ti6@)`F0rPi!MzGb_lHaeX$zFed7;j`zrkw4lx zn7#b^rj^yDC*~Q=^8x6PXWe*C;MAp!h4->i_4d*3u|1;PYulQ6_y~~?a_r9T_gY%C=W0S}^tiK9M(*74r&|+BG{eVQFlJt?`m+;%8M5P&q`Zu84c~AY8fqOg zj^pcek%f+A2W@I&q1whVx$NY;&#N`0|3SG;tPF7ESl&RbY&@(`YRyoe$TGRsbc zySU=$&Lk$2o;#);?XpRU)&Iv?WwC4AtZvQ|x*^#R4 zH(;td`CX?q;Q&i+ZPBJ=ql;rFDiSaX&^u@Ed!h>|?_Jue~$|7jd>cK-XSp_coLtN*VXoJH!bZt^NBpuqev+z}%wBR7|i zY-Dqsf3McMf4We`SlqI1&jQ!=TY($xtIIf#}7VSZr`fu zMW}g!E^u#DRA}3;ua#DWz(SIrQguJ<+Fd2=!f)hDs{~bF-R_>@x@rvx4v*@(N&`tp zbIta$g#BAz2O)0Z5q=Oaamk$uN9YC5#?O#0uCpLVjSIvb#`&B^K>dQ;_Ojio^rgrz zs&q+NX{eVZ=KaRkd)^^T?kh2{e5_`W3GX)GI0zmJhBgR%a^GftvzGpe`=qh((k!&I z7h~k2q^d!E$wY`fO6l20f3Z9IrGy1DP9NK6&VxXGxL7lCpmkb7ThiV}`w&=QDVi>9 zWGD!9$&Oo$5A5x5<5Go-6hC6_TowPQiQ|WEdyfz&rhcJ07+4_nU72fKy+&N7xZ3e= zh9AwmE8QEeG=AUF{Y6xVT*mc)k3PEdN+E@hfd|A6T>K!{;Fpb}AQ^pX*km!BHeI8w zY84e15!7B(&uwJ5qp`b)p?lD_pzXSP^{~enPmETsqH=1uCdGiHc)_DO*{NhjVkY9C zdD^S0b=qA~!9U%lx`xfDlb-l}@i%}{iC=ogRd@&;{ZQ#R(y9ylW7Zd16Q1=#b`Xc> zg8b2=iE`=bo~XS<2F105^ExF@w9gt&p{h@^GrKKJ5oD=XSNCs1IY%~R6JL? zy2=X8(P;V3-NuX3q{uM=p?dNcr3cl|tbm2N<+q~x>`>J!y(Gxu4s?^(F>$b*vrb3|REzLKxP?kh#X)JM zEOI_acg$h)v9BIj@22_t6E|{+-~0Y<#XPtwPXS*28C)OaUk=0PDaCJ4ow8fyJs7d1 ze9jfT+Pyb?-cVMi3TmlGt4iNmG9zU~AVwBsJ2f!UZR;Y&bCcCkkV2_dER28eg3jRv z#S4z`UkEj#Y(8SFC6~@&Eq<=L(!D5#1wJlva%-*L5Cy}w?^AkyJKucb{r!97Rqz$0 zv7m>u@!;|ZX*)~>@=#s~T{z3ibyqXD@8L?e1hQ-m2O|y|31+F1La=RK#JMla>`dYO zr7i_ng6|s}Je$HDpckyzU(hIQw6qfK?Ci^6*H&Bd?wIHJ^MlC1k@_N6JsVGG7sAc3 z+;HIm$Z0@8dojQArSo=guN+@87N7cIJNJWR?B`(Ar5yNuxe`VITpO(`G9E%+J3wrHH5 zgJu8BO>^XCGi$ZS^G{ty()!#9ck6BapG-Jw>RRkfBG%myeizoap6c#&BK0g9Cn91_ zet}Ac-Ko=@_`NiR@pOsA^K52y*=BRT(3jQy2Jm5bGq0j8GLo6@!RKoei*NN6vsn;U z6;2D&%`%#928~8c`iLBdn6k@RHn6o?vg`ejmfK|$Q<;|AvE&*esxQ`!=RNedPnyQh zfA97@!D^k7ecUy%IxWZmDv_DG&zyW5yuK?_Rmr2JyQc4B^&p?&9q-5WrEdH6NpVqoSF!3J zYN-?=L8QOjm6rF;VmR0m;DbLNwyfrye;~Xw5M3}AC7%PbNdQ^j6LxE$K$A6BH5f1B z8uW0y+c2$T8P9nK4KI75Oty@8IEO<_B%v+*zQBvcrB`6nL-inMp6@|&QMY%*cqL2` zk{`$sZ2o@PhkVO{iJ{7w8z6r+!qdSqB1vyoeePnqy4lVMMXiojlN&?*>1>4CfmK)U z`1450Nu_yo!d$ZCb2fBh^gns^5F;N^(;^n&#f|OVQ%M9AYv0#|$MZLbEG%Fy?zIP& zRh8u)bJ~gnzfX-OH@HXoLD~i5NnRR4c%@A8~FSNQB_-%PYoWW&buPC&zh0eMII-eer;V^llJSlj$!OYhL}WKP!$k@-edu>}!R zia4%#K(+}`SJbPOAEAQs`QJ5UdiFom4iHcg0wkJBpLtXdE-Moh0Q`)rL%KlD%IRWL9(8t~8AqiYb(k(BH`g5<;Q%llK(R6R#O};*Y~B05 zrIq_#EL0vqEH0Pb>L^Sz{>kT9RztK8=e6-e%{X6euK<%}np|E>pDK7Kug4E@RMCsW ztWL^;!?Ug`-N=#yFe;;fCzO4~-<_&W;#+@ZtrBhsA^BW0l?|pZ+?*>bjgQPz{7xOG z6>A?*yfL&SZM}85dhkIheyH$O@$d;$51~iyVfI-3!oyU3hkRqh@=stQaP_ROxA=*H z5NsP*Xh>80NYQ%J!%Syod}Ax=T-`{BLiER2gAIM-1X^|4$>Qe^|86}l`n`iQt6_|n zTm?E|Hu%i6&-~6!=Gy+C3dowuP{L90`YAy!A!fR{@jC&I?_im8?5O!h5jE|p=ef>C zM5Q=p*UCXOoAt8j#z08BObTK&4$BteP?NEp##IR3;+swjdeb)5e7z9)cJD(+k>S-{ zd4pF5kS-Tv`QpOiY>`FdrCZx$%VaRiw2|d!@lTxQn>8vaw7qkXcCB|*tY?}Rd7vV4 zS8}W%MG8Pa-5WR1wpLGK@wgxZ^8yUbx~f19X;Z`hnm;FH5*JxK4#GtWT)Pb*AQ1}} z0QD8S{PDzkwHweV>_}4(nr#>y_=e|e$@w_G#B8}HpT3SiB1e+2)&bIifJq12OCQlS z%41k#*)=E5$5vL>4%d#kOuV&|3gz8~3>7;0kgv&Ej{TdBKb>h&eodF>TB=wj*hb0| zq{!Y$_ZC8hF!JKWiqj-tpCY*8lLLooQ+qmNLBEjTjs9%C zBu&Mfi>d+uWWF+J$drQr7*26~1YY(3>6yXws%oK#d-RaNE%&~QX<|svPT%x9(3#5D znR}DHh8Nnhb-K+yXP+u>t&i_=PEV!CXkmId6q7jJA}#*s)%9ZIvM$deVd|??yyhR( z4mzC`p$thArTpi$dDwNh0DIrf_xo%IVz(d- zmkd5;If{G_6I|7xW4ABABoaN|{2W;J_pN(}cP|-<=?$%(T}yO=EGXRdV?{ZpD1e^{UHu59=Y7xe}-VcvHD-kfcP&F zKOCmqFW=kSyN!XtVc79~V(Kx_h{)R7vN|mH)D&(J0`}$4pFdHQ%*sl!KLmwmESQXa zi;W1y(THj0o)$ICpz)fy|&>3q$}{NIMBeB_1~rL zV6szkGX*?dc#Xq~wM73Pz6=#rlkDAcK+vTD9CU=pbzN#8{@+C^rQdz2+34j-%xT&r zFLK$kHjs007!TNp4Hx^;fOje$MX#KzS;dSHJirDIhL>xB-U~QyKBcGc;;-Y|sLug} za}VTFjNs|)7eHC`0r&JbdV4!>PsUkXVmu(7Eo~=95u*kH>lMXi280sQ80|_+B5jy+ z7V3KLyD!N??qz^`q}5F(=)ww^!P#%4=*z7pqIVk3L(0p`1+8KL!R%d?wRSwGS$Ofe zMsW@tMPZjodjaSHe`((VrqWyQ4G?a;efS~GZNbnhf=7>TW@0R~le-7)Ewn4uI-1?VCVUAf`nBS59p9% z>o|ZquUf3*;_Axgvio6uD8Kyzc>v}Y=ia@0anYB@6E?2fH7h9Ie7*ZgHdwJlLDz6V zyF>9EWxB0_mILs+N&yRNywa-sEopa7@@*vFTA|GP&9r(c^vbU zl`G1Ds|FKu1@Jj6_ceIB13kc-_wOHr)fPNhK_JQ}p$Oq_p!`5F^awdkAA)|SYt`7J z8W z*;i&DBO@aiFu`Z4udmJvomyI3Q7uX+D0~O{2VFxAulW}LKmQ?DQZ#dXe0_&s^2wWR#YGT2OEj^sz*ZQ37z;g8|Eq5HR4O%n9M)`iE-+?LgeX z4v4>cNe$*h^wck2be!(BQ`~Q64LaHwApusx1t!>nX&@P_4GC&gk#tJdxo}@S!kxzJ z2C(jioThzEV0KF68Xecd-xcc!WdPBG1gLuS2z_zM$*obN7xTsL9F7JW7?bgszk3ed zYUqb@o9y{ma+;W*eyLh4tKMh$56L%90Nyj5R&~jg69Rrz&X>{PdH(3(!w;7NtgPRGP6wBS#A0iyp|)5{ORKDYdSXIg zpyBZ!4^l&vXruCz%X~1^X=50rWFh5u{016ateylS0nj8X@Ni?xL;0$qV7eVnclGF1 z3QVC5eSHdGnozU);8kxYkOk1`Hq;&M021WF++3I$k-WS-y=qYpV9BF++JH~o3_$LQ z$f5d^X$$R`plgKL7;rU4^eh=OUgyC;7h%6sf5L9|`&nIGon;YJXk=hH2ux&>l|~Z3 zQ$aTbP^p`mUZQp(5EP(PEnwlle*KDa9{{ zNJwZ}l~ftc6(@81_U*-LIT&PzhKnl-=&+Urdd4mM3at_bxN9^d+LmZ#<6D5wQm=KS z1NyNWsfL~5*kt^X%-Xe!u*<);R5>~LQZrBS`FNELV!kzS3~C}D(hUf|4nPb+x}u%5 zy|rakRg~ZTjE3fSx!HhV6&pJ{d;1z#p;#Fji;_BxE8zThPfn`FdV{)+jR^0Ag@r{j zszvX_%APJX?UItmC%!JTWox^mHNAAyF4hE$DsO)C6V;U=lxdDE={K z*}F%p*A>ka2y`caO-#ke$cWey0M58Fhe)@^-Ulo)9w}*1e!fLE_vUCRwI-2jj<)vl z`tEvS^i;h&PiJT6m|MUNK$SlQ{xeUtm>jG+xyZ$?Sn>!pEvnt>6hPPI6}9Z-cWdc)gv5U zOW?NB2h`a2g&JG~mrx*;&}#7Dn@~`Pj>#13)C1nNVaqq%%*;%~_!KbW1)IFKwl;R7 zu1J{y5RzV7S~4aHKq}=nfEEMnH;)52&+r`lyO;bna4xnE4jDk)fre5Hq^3TNX3}f} zVxGe8?CflfFPB3{y({KFXO4h3 zZ#rBMR}qv*KKz7??EXg>?EHR}=09nK_4@aPd>xlLbgwI-c~^Fp~W|2-7ImOW{(UQ4p5# z1AVB(0~ElNfnLS38Qg&y!}UQv(UT-V@Dc3XgQiv$Ch$62;c$u_4E!l`{mC{%6mh zzpJ!V*YUn|0vX5$NCNWo6lEG`hilD!X<~LxPFcV!*D!lK5mrJSmbKl4tQyvV_*5S- zt}lSb>Z_k0*ls-&HjPPZ-5-H1CVQQkaobGy02SJ|z`zW(QhmquL4T8ES6`qF9o2sG z$K{7zu>LKO>yTU4ev1Rub_*y~AQCialgp#}#{gTx3IOYSc}9$ZM^{UWltH!V_HXa& zzuzr3&PHSNMJFtP_-YJx`Q~(eLP-g0JdcgsSecOx1p1e%YklvikG}*+JV5OJq(>-t zwwfUm%R1-GBlYL|#pri9g}3s-dbWpBG)}=E{tZV=gJSYJFINEnyx0kY$~gc9GI6defyaIk5t?q%Lu!S&u83Q zvk6Y9B};?nt7n+tobfNu-8w9vlDq2q3NAyV6mqSd1fl+qZz!8tK~lsxP0-!m=;y#r zwdBmR7LAGNIJXonzN*%Y0Ub*$Cf5*JQu{lh_#LBo3DMEVDcDdnT5849(-T`D#o3<{ zDEzl4fm-)%^78VoJ6YX_I=?*z8T5hBpvFH!_C@?^;z>q)EIPXDDOPvow{J1=1JSGZ zF!cVdv?i&D{7A1yj7aT0{(JA!u7BYLAEZC6{POQa$Y!Hi%miCf|1+^+nv3S9N=;0$ zR%!pCo$mxP`U}!}p!ngVh!CaoRt?dXNjIf5ncmU*taw#a&7W}U3gJvCx`#Nfo&8nh(Ua_WWKBrGw4{xhtSG`wspo`Jc7}Tbx8=T>avG*q}slK_{xN5FnGNzm?eh68)H=7ZrmL|qcZoM}XN)<*$x z)N~L_MF;04paq(13@G)=$&EWPiiaQWTyb$8B)k@tb+Lcdere75=rhVh*M@2W* z)JXn12T8NIv3P;hHnN%Nd|cBT;Gs?W_`=GXy}gR)ECfQ=OLot7yP{(fprRJ8<}L}C z#M9ZIpLIgNLd_!?=1MHha1z#4?MLQgq2ZYZCLD(|)O6-^D_PYPy?J7nit7{Q1(v;X z_7J;pq4RyEZ_QWpS-CM~1-?tJ8 zPlwVu^9k^ob$nLKGFrY?;;n(t>2_)xOsKveiECW5G>?uSV$Smx)M{l!^qe{5^=zhJ zuTgqcV*F{#9NrKZFz%p(fE*6BN{f;dBXR97-=c@Pn1#|h-ySXNUX~g{%Q1F zimNM1;b8oE#X<)Kz2@)i7DHHXwhp=dBut9Pp{B6$GA-n4+TOnQqE_i;-b9@fGQ;n0 zhb`A@BgXmDlCKo|WvTHmi$-dd`=eLZ=_gnZ-?&vKpU+o6ir9M49i(vqo8DAcZq|s2 zp)?a6v};Z39^5;K3lK<1YZWu!unf@NTj78kabN7D-^_P4UF#d?ckM57$a8+el>=1S zL?9$HtUIH|)ui3cicRF3R#3n=w&nbsmX={xHz%N0UEF-fMc38cy}PwloI61l2%GG? z)~=X7!V`En770;zevrMbf%^OL`QK$mi+=sr$kZPFo&FC}e}7OdW}iIoPAz{>ulb~{ z;c4GZa8r|FoHe3SYn6RM5LS3GC#`H=iyqd5O@k7!~X5a+b;Bb^l(Thkc8<=^&j9fG0rcP3H^s}oXeJz%>6e_BVA zruuU~8l(T;!IWf**83e>H~JI(dAJ4UfzF)fgK|C*v_w4PdljgQA1AethAWBQRrAp| zx1wGR8$+iQ#ZX_Mo%n8UT5?EFzs;@BayPx-{C&!LW$!oYn~ygg?Cry+tQ(hb8>&c6 zIXF1}j_%KOMPp;*esqoPxIg}8vfx)oWYwgY^k2)E%>C67QFS3u{&z%8!Sq3luHApF z%|iv{`b z3Y8`0Hn%I81IQFqzudVWQ0?fJHsJ@zF01OTUXaGc z#ks8{K?bvBNkGLbKQ}iQ&t(w}3=FH><5fr)$xk+jNr7gS8~|SwU%vbZ;EW%j05b(8 zx=%y$FJa+=GV96uO=noPT+$DfLXFOu8C@Xw?w4k%H;k^b9NYJ8XdwfV!W%!nAMVWP ze){yOtE1yZ--UDx^AO1Q$g+<%$CN;BR!auIwN1<{K82QA(%Rk*l4gyvG0X5wfVbu8 z)VqmK(mx(NK!_BB10YO6t_XCS$zI4M0@*~~cL1CMD4XIt9u`$nhpJ@&8V!Jw<#0Zb z1vLlzhlhieRuhk(JjqH;O|@9An%J_po7+d1{0*?JqoWaBi>nJHIIkjleqo^k{65`! zSZD{-wxDJXtE#GMxH_`{)z{>EA&|{7E=J#gr05Tbxj%1aN`AoJ8|eNYO(f83wJbU(9 zPfssi#M_HjMDnf#3`|zoR5r7iF*& zMa9I9x)_RMN4{;;rvPYdhUkOA5~GgLJ{5^t^mfb(TzQwV9-N8Usv=5MblWWxx%vAMecA z0OXdUjXG3NAK*f5YEa|gu;gUxI`A&R7zX@OFfb1io4P$;yS3lX7r;)H0I*CDupff1 zE4uCB0(ArM)P!O&QPH3<>NmthL_`u2623r{jI#iylpS;h1^PJE` z*j84k+7vX_23{Hj-?(*$5Ulv0i2_^80My{i_F(g~0Uf#lI2OcmwB)BL0Q%GsRWn`| z7XTJQO?@e3M<)Zo{>Js7Zs%MdF5Iv!07o>%wYP&)GlIa&6lxHnW}GLbjeBDC9CxOL zM#=zkM1+OU2#T||(6DsD<@xz}(*M@pcSl8icHNQ~V`3xG7!4|Dzye57njl57ASfWx zq{o6H2q>Ka9Q9R+f2sY)9Vh9b<+hcXOEzx&a=_gnYA>sxny@3-z< z>#mi5vcfP9^Lu{hIcJ}}_j7*1wSy)6d(Qp78|kTBa;2b>5~CU8ZK4KCHokcAVgu5H zY==+VF4O#X`4B677Z;|=$RZdR7)159{$m+0AEXjVP;tJfHhdVtBLzO?!iFR4kqb^v z*+gdO4zakpdykaVJ*YK&yee}~{)mjUZ60=sL$JO|Gs;?7#IW<69nL#~#-Ge~Vtr^T zu352EKKR;3E^aIto%9OBTdXk==GmK?8ke2+?_+PdujG5hbUcPb`hS1|f1HrOCs90l zV>f8FEKl^?EcZ8(;*0li^13~y=J(&f>n<88m0H=|u4upRkegfnSbt(PpUV8i1D!;z z%_OuWeJguusCLb6%pgNwXhWZup};W__wwnvw`*%VQ3AHT;=e_LO# zD^dC>R`yc!BMeG$jKT#PV*-gT*a0rRY@sp6m1(U+neg}X^Ai$QR5UG{8}~74D)Dd) zlCUrbFvVIu2oDc0&tqW@l^%Z5$CMgpcAHUYY3V>+ypKlU&KQ0rN~rVZy_d;JeX`9# z*XV(@S;en-3i^xi5aaAG&@1u@io>~crp-`=ey)j^ClJJ49#a^x-5J^4#{lfP4BZ|E%R<6`Tx))L>3IoRzd&^A5zTFi6 za-7R;1bEs^>7nJijh#^%d$D%;ci;K@`1p`y-2K{5R9V>^tX};zrx!I-vgMNOty{O4 z^5007alSs$j6po4L|9{SDLsFvHGmRBee5gbgzVH&#i|D;CMIIggETUj;-d1nBXaF{ z7A_TN1oVt5Db&!avXL6p>p*_4+qf|mJHg1@++0K__9g~cEY}+W*W8$Cc?>wh0AH8_ z4wuPdC7{%Wa!f})r`q!`$G1P$P1H(;ddM1tc0f84WZHn}=xBK{HE@tf6v=~4h9Jo} ziwp*1^NlOm6+iDf^$4Q}&>4vC-u?cx8z)F01HaTS^>pXtczb(;*VtxO>MbrPco5>j zo=T*ObwB)Y`C#i+Ibq!d-|0m*o2@`yD4w3FnBQsEUh7`sop;c|AyeEm{}Jv2NNV*u zW*57^zn`UOXm{|=nr%HpLut@xf_535^q3ra55g_Ix%rfO-K`btH{GvH^$P;w_qMva zIwII{*UyhPPu&HbXg*a=J#!`&SLIsoY|<6#f!#NO^hSUOqTvR{NW=q&bi9{szi3$* ztdAWQ@ZbSwWe$QwdQqi}V~e41m>ZP_c~L}4N|$_{96>n<6f@Z26VTPd>mwo~g8~CV zV+*0MzbsZU4geOs$|xMi0p45HxDEnT!FLA+1;rzS+qC6hNwFv^h2~6qs3IzQ%)V?s zmuyp98)CX21ZPD>g~Q&O6$9pq_RI^|uqeJ#G$V|TqT`Y#8g#XVw{ATE1aT6pVwc~J zsXN}auLB6p?rqYaG1cNT^}TD>nP05Dyu8*42-ufu9X(3W7mY?M&j#775rA;-I$>}E zP@Jg27%gY5{EUfNX#S@+$e9`VrSYE?hBh3=(GX~x-BTl-n}f5cYAN4+<(FNsHcgyCzh1fCFe>b_*bMgq?;(4!7W zrjcf$;=QL_(m^Sj)D%qa5;4kr zEME(W%?ZN}BUPy8L__7L^u=s6*TxP(N~q$k0YExwYTBlm(}Mm6eG(n%E+`-%7JLJ6 z)2!MFR& z&8wffmc>7QYz4H(&@d%_G5y+Gfe0l~3jCT%- zWwbFETnxgqst6P|E_6!9NG#{EkB70~TC~y`>x{}C*HZ8#2H;khY<5O=wzQa8VS}k` z-V_JEhSa%%cwO+b`Rlb3RhZ z!vF-V@zXQ6pvEO21*iu=2desUNgcC{*TTX=>WwD=-)b&|fIjFEtU>gj0~iD|gS8Yg zTey=R!3n{kn7r}K)8*eH0oq`4m2n9r2X+UtX++QoArMgsP9V~yp-o^>b_xoXuS|y6 zeAmxUPN2srhcw6719$J=4`$fWZ!BFA7#w{5YD@U~_8o22clCK~YvhIrmeGn*Oop98 zc-ywa==U0W-agpO8brIbZXo7?J4-f1NDi1Kmah76J3=p`3tM)5NwZ>~}VLdDc%U0Xfi`-Bv&u&Y`15C!-fa6%?UQbM|#8PqjJT z_4RE))gu`UJ^?GNaexC=rM$eH#MsU~drqQj3PB$YsU!oy4K*1~lVjA`){coTZ-Ep{ z-=xn^-U?z*zV^x+y+!h^7sEC$Bto4+EtmEQ>} zLpw0lv+R>cb?jjuc!p%jJw43$sZDS%3$)bTXtb3TKxDGv!wGh+ZGb87r{0}Ech0Ic z*H*!i<%=P0K85mx!Q%1RIlg}uVjlv_7KHY!ig)5ELQC@ahn)F~-n=}r(|7CZm*c{( z$?H5l-hXjwLSGomX93^+C#2iIyum@(R|7lPU_;~QPh$Uw3SPogSibEYdaf+V6vC17 zv7dCAIa5!>kX5#1+At5^d}{UnSA_0BE^QglwU2#>J_9EwCmp#Z*Iup*-@j2|PBdtscz;>6#+A^8 zPDdO46wcT4PM_i`srRCN3%e#gGui(9Ue5J0wx*f=Z&uNrC151u1;O*k zCP5CR6X96lI^EpdY@>P(!6g0LvulIF9pX^h2`lEL*0gj#NsbLUHinJqCWaDaa}F7t zG}D5UC`Lg-hmRcjOj~g{^WZI{)wMI z1D6|ai8y!=9o-?b2!!)n4F|woWbfWn$dXNHeMO2G=8tTZ-NzpjybL!B8W)UFdmA$c z3gSJi1dm~EvH$}_S)8BZq>Z3C)kSO192ud3QZaRGXMinwUS3XKcr8FR1!}FxDlN_x zEps-bDPsWZJWLd;>%nG;;s;!Z!tvUcDIyW)u{|P@$51nYZ$+nPXH$`7R@0xrM=$_? zm-zFaKU$+A7REd9+`p^&DClvSo>+W*IBz)3Z~I)#+6yO{0|8Vi3$NHaYYbPP_*n5( z*dE10)pMeuw6mo87ziDOc(t(eh>pIt~A+vmLzN$4uk}y`H6ZA#r*@NTH1L) zkFbqsO9F+ErByK=PHxc)z?2YF=#qr`vBq`4lR0Qk(#?ttFhaHgq}+<^Y7}8>3J)9w6fvze_w(X2UrDYMvnVws0fuayAxt|Nf46j+wb7e*42h2JMeVWz$2kMR zTL!)DFr&y&fI9aSWyGSP`nj`^kPyKFq})1)sW|odCmeqmi{`(*EBLD>{j{%bJ6x`= zR(cfe<#o!dm9b+-VkM2b*Uk8Ofko_k>Zt5=;swPol0$t>0ZyB;&m0JjHokI z+#VV)a#ad)UAr^fayArbkT))iHc5^4i*Vf8ktuiJy0(~MwOUAD%hW=uZ`F`$n@JNH zLkvbt9XsJ9g0dmc-hy*e>0e1r|Br1IG#5;FR^n7ifx*6I-GOw;d%J@#1niPX(kZlqS$BzITpH( zWB|8&5lq8@ieNqWX^aqt3a;$5(9qD0ZK>C|q_Kz9&=4rVL9Di>rU{#y5b)}C=x-a5 z@e2wISqh6mU3q{C1MMY6DSF8c)t;a}bpYbQ<_hxi^1@&=8Yj zQww3KdF16zYt6HdCS@12KblKTfG5`P$^R>ZS|xEfHZG3bCzN__f4v&nh=Y}bVIhuY z;mTfxl*yrF443S$*H)1Usebe39WX=#xXW8NZ}!{fd|NqyX)iQxD&iWJ#KO=){cxfKHQ-CjB;21QMzGVwnkx*75&fpfXB#DsTLk8 zS&I4Q?m?o=gz6&g(E8v@3b9u=&t)Wws?|*&8rLXq`tQ8$5E*OR z_RDSQaf?=Ii9N?($IT4R%JZg9&#%)@G!`zA6*|iMjX|Bn3d&#+)wLv{z4-yvs&mj} zVroZt>yrS%e|iD>`g*+@<21#T#w)hRCoIbE5*1M}2v1P6s`Xal)^}x5dVRcZJq_PA z9ByCw9y8SAkWx~CS}E|K38{l(2|$$O+A&f9{mnVmo>!T-Pm>@4LcCi#%ibqulz9lo z%sROaPeJvW*D4EJix?RIi&kJUXE5^DK8Zny0Z`~U6s|SoB`d(hfX5JPE}{WrdE&Ov ziPFJaj}Ao_y)&Us$cGX@dS@+_zYNwvvIXJ2N(WPG4Z!O@>BKV?i|nL>a%NcIz+5i9 zh$69Ey0BbQoGL#M$Vu;C3~60&4!+p9$R!=H+*S)_Jbw&yuA;;8M%bmg`uY_OH8=LG zJm@jS)(0I>b{d$+nDn=`wc4OUR@=(s9~oV>yqtlnMfN)zOhrq|D%ex@Ib@A; zFFk~J%YO2iMM1_UkVDe7@z^J@x25+mnfljC{sBIg01ji2z!(N@#ZM2AhLaPp`+$Jw z=ykHJSO{TpaV^kqb&%?QTDvw570!wfkAHF~bjAL0XRs%lL}NhBkb#NTv3Z)@CUW(d z`g;O&YfZEY&9&WKt&aIb76b7$AMQDfqC$u&u(pegm=e8WMQAUC7(@SB)w&9CAMeg# zy`|F)wUxMfKj!`LB>2oat8K|t@7e5g^*;_bha`>1GxQU~y5%O9D6}WWhf7R-jMCJi zCkSXIX}1n8`yi3KN&o2l@`09w1XvWT8E*{J`KGN89xoPp`;zq-X}JDY;PD`?c+53p z%jFF<>rDKF6`y1(cpE%O^~pNI9S!mL{=(srdB@-1G^>q-XkU~g`{Mg~z5yRG`lLv^ z4d^w}=cu0qZwm!4L^XNp+4UvBZT={4ge?Q=u^#RG(-^$JP(R5RVB1BJ?;4YI4*H=w zuWD3*0l46^CViL?p6(syDB*;U`%F-Q&};;>3IeMafSxyHN0ggt0DS)20Ji8|5IlCl3+_a@TEE*jzs=l^0P;Dt^W_mgqxhPEDF#(w$#(Z(W z|84YHdQ-YdB3$gkLP9Shg2TcFekcSzvrQd%ioS~~+D;i1!#C`MxFwTFn)d-Dwv6F7 z7b|ghPv`DPv^5Ow&E}AXt^8f@7YHTR>vIj{SY>0QK2rGGob^QjNi!Xb3yIh;giRO# zjoQ9%X#!L;8(7Va%dVF2yZ8J^5;yi~A$yKUZm>Ew=GvN|fDFJu8VF9=WvIF9#05JN zaqIWX(`B&u+`jkaLtTp)mOp_mEy;6woT#K=gl){d>1jrBu(lC|gSpRWa{@Jjce(F# zQGeP?rkNhRorK}~VM-pkt2Zk=263B$A=?CYJF7+Wyoo%fBiCu|mz$o>5;PFmCZ;>T zugz7{9S9;!_R_6|qggi7d9NEX4;Ak|3`KUD?6FK|fC&h_c^X$N5E;1{Uw(Qt( z@oh(Ie{}TxupW0pSft5qemx;E`ew!KBkC{VSxDL8MmQJUfPik^~5b z9`T^>n1A7Oj;ihMtZ?TQP)>t{VwQjT0MuZq zMJ-5ZYP;auRxtH8Gc%(VeA|8uE)yAuHPsRFx+MRB;E@Ic&k6pPXQ|=P*S6DjZ0nP+ z)hz7fQ{0P||;EGF)$CLe6nPA_7!AHp2wYgP~9#9`29ewvu>Oxi2 zZS1n%)&x;AcZ*@*IB}3DS!p;7L*xyZ9z>9Vpz5#7Z;$Y3c8J(}<#~)0CJHb`-p{Jsw0{2UnAoPfLXL_Os05b{jXX*0#rDNzr zt%0ftk$Uh`5wZR|L2M*SO3W0{AQ4=u$3;Cl)~s3MgbDT_8hQ)4X7-;00HrYjq#_84 z4-vr+A0|WId065pPY@8qA}d&o;h~rw=I?oaK1&!H1_ZwhRI8-U&Q6<*Xprv3EcaDl z*H=$nVmVjO;@KBYlS4+FpReHrfj)2|Ov}VE3{P+5;8XplXI|Re=0pPP)D%2{QExsS&Phf#ox<`Pw6%rP^gLqIB~N+^4dY)|i`gEV!Mlj6EB4 zvDSccJ~wyXQ{&1VVGXV7h6>v)^=?=BposZ*q`7=amp1PdXG!D3Z*T}%UjbZ~KL0#Dc&_xhfa&MeNaG$Jnq zMMkFMZU6w(c-|1XOoP^~p{;F*HZwCbGZ1q2!tpBHy$N`$_j>A(#Do;^%WXIZYvv5U z{Qy!P*B~k-RX-$FI39C|AaPs?xLO(5Ajm;1dRxE~5X}I!dnd%zL&~a*i`|~y-y|Ok zd7}EE)ax$(Rx}3k^hD9#{7fR!`6m%c$rF$m!*-v& z4P2a!&J~9Mw9uP`2;mtG`Zxj71hY^em~NOsQo;C~A8Bsny;bo!0R<7Z6z{cZFLo!4 z?6f`UF_fPc0A&(j1Z#HxAY00G5a-nDko%kwT=5AB)_{iur=hk8^^ZU4jA%XcxRUpD zE0QLFy=7i&&ToHy4P!|%L|B%hyBPOCPseA3Dm3a2fH(OOj5 z*Z^bVL_s=Kz+;7Tx~6h7%yEVgk~;`neF~)zI!ppqxH@tlz03SO&r;o8OKdeQByKg- zdVLBIa<fs2RNU6`ni@W~J+7K+F=b%~OR00IO7CTws+ zh$p3d9N3_mllK$2sT7m`)qCgw0Cj7AhWvVH7(*A*lJiCUl&K znsoqzocGFsr07xT(JkEcaejWjF_X!((ZULW3avgd3lm%)HlTG!X$f8+Ujy#{SEdUU z-l~Ld2BiyHOc%%Wh=_`!7}gtr(CY)=(F9`_(Z-Q=Byq6H z2;+srp^{YoehuixZ?fe0=;LE!cmPUP0~}vC?q`i{fJzb8Qm$a#u(M>E6;A9usM~sR zM^=&bv@e}bfDW01J_vS|+S5rk=JaB$XC;~vtXz}-)`JtkWFoLb1xSPJN4-h5D7#7s zsBc$qN-Fo3Slo0LiH8d&$ENwoY0}+ei|z3C_G26arU4REBvl%5Vu(wU#z7PEBgoK7 z?$6>p^j>tr+lfs>uK3aE4Uj!gT3Zh|Q?0vPtS1fywtMB?`XF22IAzfw@G1VhuWihI zO^$%7Pm!9HRg&`jgs%4vUvn{-k6<4y=t|tNBwrZ~Ark9FV9up8u@@luf#itTt`*_7 z%s3Kq&l%s z*`OOd+^pJJgJ+h-0$o^UwK$a6N~(m**^Bc}y%V2XxOAPhv3Y2Gp3lO|^|+Q+-I9Bv z>)cwxWu6$xTb?^_!--m}p+Hq%HDAG~QyA29(#`rX#nPQ;8<5r!$q6fZszLKBw#a&M z(|3-wN`{>}D}a0|o%?Kbp0r&0kbmVYxZ(Y~9{@t!y!nmNbkBp;rwxeMoIyeXL93^# z`dn%L-LB3fKl$trU#qtww(=%dVRY8rOLFUR42+S>9;&8HmTaCmuYF^IOFy5YS95Sl z_}DiJrjv+0>z+4%0Gb7XFEVea#`l0)g&Vs6dT-gdvI@`6`#7dY{0a(l!JHrzsJQiL zVhfo(8I~?&yg#68XwLmeG z$>709Uu@sMso$zQXaZv4c@lZJA{U-&hskx}4SHyp_VE=W&!j>>#v)vV{rKgt4QoCh z%xKeZ$}mgCEg-sygR^rZhH9B`lEoxYqTC589s0_mVA?L09HgJe)tR0i$~L6hyV@C3%SQ>gp zMxt$!(skl)pRb8X21Td~1Y%s^5D%EQgO}Iu>l}M*5qk!dndLH7&>VGex)a@w9M_}}HG;+e`B2A`Utv!mR0hP7j{dtPELx9=U7(wn+N_EOY(G>4^Vp$?@+$@%P>Q zydVDMK+fMG_xY67lfT#D?{&Zq`JV^X{y&a~xw1+2mTdvMR>t)$?h(AQD}T9MC)@1W zvA1{p<03+vk~VG(yOH7_U=x{j?RR_19XCaufN#jjLG>Fr@_9|@00Lou04bUNFoeMAloR)&sM(K6GP8<>+3Af+0SH-m@v!0#lZ!@?s($#hQEJjj@eZV*RC7SqTXV z&diKLUaM`bAY$`V!fNmGwvLY4?(Xggtwe@}mFu$MTwBiQs3crKoXJU9ytBQd!}T6M zv(v-Krn>8LX!qD&0Lm=eR{FBxs;XRDM@9Y%=U?o?7g`+bwtp3U<+h9AyQ`b_2%aZ@ zp`f-{_=4-Zt3AsF@Q#H*#uV8a&kTR~_jg6^!-u`Sy|i|Gd|23SJ$&qU>W2>K@s#x} ZTll?z>r&fs6R+{8AJ#gQq59j6{{aWuwZQ-Y From 61c832d7fbcc831a2625117d8de246ca4604bf11 Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Fri, 21 Jun 2024 18:54:32 -0700 Subject: [PATCH 4/8] fix test_copy_from_staging test case --- conftest.py | 2 + .../tests/test_job_files_manager.py | 41 ++++++++++--------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/conftest.py b/conftest.py index 5c5849235..27cbeca55 100644 --- a/conftest.py +++ b/conftest.py @@ -1,4 +1,5 @@ from pathlib import Path +from unittest.mock import AsyncMock import pytest from sqlalchemy import create_engine @@ -59,6 +60,7 @@ def jp_scheduler(jp_scheduler_db_url, jp_scheduler_root_dir, jp_scheduler_db): db_url=jp_scheduler_db_url, root_dir=str(jp_scheduler_root_dir), environments_manager=MockEnvironmentManager(), + dask_client_future=AsyncMock(), ) diff --git a/jupyter_scheduler/tests/test_job_files_manager.py b/jupyter_scheduler/tests/test_job_files_manager.py index 66a9727b7..52c0564c9 100644 --- a/jupyter_scheduler/tests/test_job_files_manager.py +++ b/jupyter_scheduler/tests/test_job_files_manager.py @@ -1,10 +1,9 @@ +import asyncio import filecmp import os import shutil import tarfile -import time -from pathlib import Path -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest @@ -41,23 +40,25 @@ async def test_copy_from_staging(): } output_dir = "jobs/1" with patch("jupyter_scheduler.job_files_manager.Downloader") as mock_downloader: - with patch("jupyter_scheduler.job_files_manager.Process") as mock_process: - with patch("jupyter_scheduler.scheduler.Scheduler") as mock_scheduler: - mock_scheduler.get_job.return_value = job - mock_scheduler.get_staging_paths.return_value = staging_paths - mock_scheduler.get_local_output_path.return_value = output_dir - mock_scheduler.get_job_filenames.return_value = job_filenames - manager = JobFilesManager(scheduler=mock_scheduler) - await manager.copy_from_staging(1) - - mock_downloader.assert_called_once_with( - output_formats=job.output_formats, - output_filenames=job_filenames, - staging_paths=staging_paths, - output_dir=output_dir, - redownload=False, - include_staging_files=None, - ) + with patch("jupyter_scheduler.scheduler.Scheduler") as mock_scheduler: + mock_future = asyncio.Future() + mock_future.set_result(MagicMock()) + mock_scheduler.dask_client_future = mock_future + mock_scheduler.get_job.return_value = job + mock_scheduler.get_staging_paths.return_value = staging_paths + mock_scheduler.get_local_output_path.return_value = output_dir + mock_scheduler.get_job_filenames.return_value = job_filenames + manager = JobFilesManager(scheduler=mock_scheduler) + await manager.copy_from_staging(1) + + mock_downloader.assert_called_once_with( + output_formats=job.output_formats, + output_filenames=job_filenames, + staging_paths=staging_paths, + output_dir=output_dir, + redownload=False, + include_staging_files=None, + ) @pytest.fixture From a80f6889605fc406c95bfe9ddf344496da18df5f Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Tue, 25 Jun 2024 17:50:42 -0700 Subject: [PATCH 5/8] instantiate dask client inside BaseScheduler, remove dask client injection into scheduler class at extension launch --- conftest.py | 2 -- jupyter_scheduler/extension.py | 30 ++++-------------------------- jupyter_scheduler/scheduler.py | 10 ++++++++-- 3 files changed, 12 insertions(+), 30 deletions(-) diff --git a/conftest.py b/conftest.py index 27cbeca55..5c5849235 100644 --- a/conftest.py +++ b/conftest.py @@ -1,5 +1,4 @@ from pathlib import Path -from unittest.mock import AsyncMock import pytest from sqlalchemy import create_engine @@ -60,7 +59,6 @@ def jp_scheduler(jp_scheduler_db_url, jp_scheduler_root_dir, jp_scheduler_db): db_url=jp_scheduler_db_url, root_dir=str(jp_scheduler_root_dir), environments_manager=MockEnvironmentManager(), - dask_client_future=AsyncMock(), ) diff --git a/jupyter_scheduler/extension.py b/jupyter_scheduler/extension.py index 3fc323361..001a9adf1 100644 --- a/jupyter_scheduler/extension.py +++ b/jupyter_scheduler/extension.py @@ -1,3 +1,5 @@ +import asyncio + from dask.distributed import Client as DaskClient from jupyter_core.paths import jupyter_data_dir from jupyter_server.extension.application import ExtensionApp @@ -72,15 +74,11 @@ def initialize_settings(self): environments_manager = self.environment_manager_class() - asyncio_loop = self.serverapp.io_loop.asyncio_loop - dask_client_future = asyncio_loop.create_task(self._get_dask_client()) - scheduler = self.scheduler_class( root_dir=self.serverapp.root_dir, environments_manager=environments_manager, db_url=self.db_url, config=self.config, - dask_client_future=dask_client_future, ) job_files_manager = self.job_files_manager_class(scheduler=scheduler) @@ -89,28 +87,8 @@ def initialize_settings(self): environments_manager=environments_manager, scheduler=scheduler, job_files_manager=job_files_manager, - dask_client_future=dask_client_future, ) if scheduler.task_runner: - asyncio_loop.create_task(scheduler.task_runner.start()) - - async def _get_dask_client(self): - """Creates and configures a Dask client.""" - return DaskClient(processes=False, asynchronous=True) - - async def stop_extension(self): - """Called by the Jupyter Server when stopping to cleanup resources.""" - try: - await self._stop_extension() - except Exception as e: - self.log.error("Error while stopping Jupyter Scheduler:") - self.log.exception(e) - - async def _stop_extension(self): - """Closes the Dask client if it exists.""" - if "dask_client_future" in self.settings: - dask_client: DaskClient = await self.settings["dask_client_future"] - self.log.info("Closing Dask client.") - await dask_client.close() - self.log.info("Dask client closed.") + loop = asyncio.get_event_loop() + loop.create_task(scheduler.task_runner.start()) diff --git a/jupyter_scheduler/scheduler.py b/jupyter_scheduler/scheduler.py index 1f7facc15..f204df73e 100644 --- a/jupyter_scheduler/scheduler.py +++ b/jupyter_scheduler/scheduler.py @@ -1,3 +1,4 @@ +import asyncio import os import random import shutil @@ -99,14 +100,19 @@ def __init__( self, root_dir: str, environments_manager: Type[EnvironmentManager], - dask_client_future: Awaitable[DaskClient], config=None, **kwargs, ): super().__init__(config=config, **kwargs) self.root_dir = root_dir self.environments_manager = environments_manager - self.dask_client_future = dask_client_future + + loop = asyncio.get_event_loop() + self.dask_client_future: Awaitable[DaskClient] = loop.create_task(self._get_dask_client()) + + async def _get_dask_client(self): + """Creates and configures a Dask client.""" + return DaskClient(processes=False, asynchronous=True) def create_job(self, model: CreateJob) -> str: """Creates a new job record, may trigger execution of the job. From b02f850819a123ad6b25f6a91b84738b011abeef Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Mon, 24 Jun 2024 17:50:16 -0700 Subject: [PATCH 6/8] add stop_extension logic, use it for stopping dask --- jupyter_scheduler/extension.py | 22 ++++++++++++++++++++++ jupyter_scheduler/scheduler.py | 34 ++++++++++++++++++++++------------ 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/jupyter_scheduler/extension.py b/jupyter_scheduler/extension.py index 001a9adf1..b18192798 100644 --- a/jupyter_scheduler/extension.py +++ b/jupyter_scheduler/extension.py @@ -92,3 +92,25 @@ def initialize_settings(self): if scheduler.task_runner: loop = asyncio.get_event_loop() loop.create_task(scheduler.task_runner.start()) + + async def stop_extension(self): + """ + Public method called by Jupyter Server when the server is stopping. + This calls the cleanup code defined in `self._stop_exception()` inside + an exception handler, as the server halts if this method raises an + exception. + """ + try: + await self._stop_extension() + except Exception as e: + self.log.error("Jupyter Scheduler raised an exception while stopping:") + self.log.exception(e) + + async def _stop_extension(self): + """ + Private method that defines the cleanup code to run when the server is + stopping. + """ + if "scheduler" in self.settings: + scheduler: SchedulerApp = self.settings["scheduler"] + await scheduler.stop_extension() diff --git a/jupyter_scheduler/scheduler.py b/jupyter_scheduler/scheduler.py index f204df73e..2ae53a13c 100644 --- a/jupyter_scheduler/scheduler.py +++ b/jupyter_scheduler/scheduler.py @@ -97,23 +97,12 @@ def _default_staging_path(self): ) def __init__( - self, - root_dir: str, - environments_manager: Type[EnvironmentManager], - config=None, - **kwargs, + self, root_dir: str, environments_manager: Type[EnvironmentManager], config=None, **kwargs ): super().__init__(config=config, **kwargs) self.root_dir = root_dir self.environments_manager = environments_manager - loop = asyncio.get_event_loop() - self.dask_client_future: Awaitable[DaskClient] = loop.create_task(self._get_dask_client()) - - async def _get_dask_client(self): - """Creates and configures a Dask client.""" - return DaskClient(processes=False, asynchronous=True) - def create_job(self, model: CreateJob) -> str: """Creates a new job record, may trigger execution of the job. In case a task runner is actually handling execution of the jobs, @@ -393,6 +382,12 @@ def get_local_output_path( else: return os.path.join(self.root_dir, self.output_directory, output_dir_name) + async def stop_extension(self): + """ + Placeholder method for a cleanup code to run when the server is stopping. + """ + pass + class Scheduler(BaseScheduler): _db_session = None @@ -426,6 +421,13 @@ def __init__( if self.task_runner_class: self.task_runner = self.task_runner_class(scheduler=self, config=config) + loop = asyncio.get_event_loop() + self.dask_client_future: Awaitable[DaskClient] = loop.create_task(self._get_dask_client()) + + async def _get_dask_client(self): + """Creates and configures a Dask client.""" + return DaskClient(processes=False, asynchronous=True) + @property def db_session(self): if not self._db_session: @@ -783,6 +785,14 @@ def get_staging_paths(self, model: Union[DescribeJob, DescribeJobDefinition]) -> return staging_paths + async def stop_extension(self): + """ + Cleanup code to run when the server is stopping. + """ + if self.dask_client_future: + dask_client: DaskClient = await self.dask_client_future + await dask_client.close() + class ArchivingScheduler(Scheduler): """Scheduler that captures all files in output directory in an archive.""" From f4d8f8af5c70c8d89d7b8e43d0946a9f8e0e0fee Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Fri, 28 Jun 2024 13:24:41 -0700 Subject: [PATCH 7/8] use default process-based Dask distributed cluster --- jupyter_scheduler/job_files_manager.py | 33 +++++++++++++------------- jupyter_scheduler/scheduler.py | 25 ++++++++----------- 2 files changed, 27 insertions(+), 31 deletions(-) diff --git a/jupyter_scheduler/job_files_manager.py b/jupyter_scheduler/job_files_manager.py index fec5caee1..384bcbbd6 100644 --- a/jupyter_scheduler/job_files_manager.py +++ b/jupyter_scheduler/job_files_manager.py @@ -1,7 +1,8 @@ import os import random import tarfile -from typing import Awaitable, Dict, List, Optional, Type +from multiprocessing import Process +from typing import Dict, List, Optional, Type import fsspec from dask.distributed import Client as DaskClient @@ -14,10 +15,7 @@ class JobFilesManager: scheduler = None - def __init__( - self, - scheduler: Type[BaseScheduler], - ): + def __init__(self, scheduler: Type[BaseScheduler]): self.scheduler = scheduler async def copy_from_staging(self, job_id: str, redownload: Optional[bool] = False): @@ -26,17 +24,20 @@ async def copy_from_staging(self, job_id: str, redownload: Optional[bool] = Fals output_filenames = self.scheduler.get_job_filenames(job) output_dir = self.scheduler.get_local_output_path(model=job, root_dir_relative=True) - dask_client: DaskClient = await self.scheduler.dask_client_future - dask_client.submit( - Downloader( - output_formats=job.output_formats, - output_filenames=output_filenames, - staging_paths=staging_paths, - output_dir=output_dir, - redownload=redownload, - include_staging_files=job.package_input_folder, - ).download - ) + download = Downloader( + output_formats=job.output_formats, + output_filenames=output_filenames, + staging_paths=staging_paths, + output_dir=output_dir, + redownload=redownload, + include_staging_files=job.package_input_folder, + ).download + if self.scheduler.dask_client: + dask_client: DaskClient = self.scheduler.dask_client + dask_client.submit(download) + else: + p = Process(target=download) + p.start() class Downloader: diff --git a/jupyter_scheduler/scheduler.py b/jupyter_scheduler/scheduler.py index 2ae53a13c..1360c70cc 100644 --- a/jupyter_scheduler/scheduler.py +++ b/jupyter_scheduler/scheduler.py @@ -2,7 +2,7 @@ import os import random import shutil -from typing import Awaitable, Dict, List, Optional, Type, Union +from typing import Dict, List, Optional, Type, Union import fsspec import psutil @@ -421,12 +421,11 @@ def __init__( if self.task_runner_class: self.task_runner = self.task_runner_class(scheduler=self, config=config) - loop = asyncio.get_event_loop() - self.dask_client_future: Awaitable[DaskClient] = loop.create_task(self._get_dask_client()) + self.dask_client: DaskClient = self._get_dask_client() - async def _get_dask_client(self): + def _get_dask_client(self): """Creates and configures a Dask client.""" - return DaskClient(processes=False, asynchronous=True) + return DaskClient() @property def db_session(self): @@ -451,7 +450,7 @@ def copy_input_folder(self, input_uri: str, nb_copy_to_path: str) -> List[str]: destination_dir=staging_dir, ) - async def create_job(self, model: CreateJob) -> str: + def create_job(self, model: CreateJob) -> str: if not model.job_definition_id and not self.file_exists(model.input_uri): raise InputUriError(model.input_uri) @@ -492,8 +491,7 @@ async def create_job(self, model: CreateJob) -> str: else: self.copy_input_file(model.input_uri, staging_paths["input"]) - dask_client: DaskClient = await self.dask_client_future - future = dask_client.submit( + future = self.dask_client.submit( self.execution_manager_class( job_id=job.job_id, staging_paths=staging_paths, @@ -755,16 +753,14 @@ def list_job_definitions(self, query: ListJobDefinitionsQuery) -> ListJobDefinit return list_response - async def create_job_from_definition( - self, job_definition_id: str, model: CreateJobFromDefinition - ): + def create_job_from_definition(self, job_definition_id: str, model: CreateJobFromDefinition): job_id = None definition = self.get_job_definition(job_definition_id) if definition: input_uri = self.get_staging_paths(definition)["input"] attributes = definition.dict(exclude={"schedule", "timezone"}, exclude_none=True) attributes = {**attributes, **model.dict(exclude_none=True), "input_uri": input_uri} - job_id = await self.create_job(CreateJob(**attributes)) + job_id = self.create_job(CreateJob(**attributes)) return job_id @@ -789,9 +785,8 @@ async def stop_extension(self): """ Cleanup code to run when the server is stopping. """ - if self.dask_client_future: - dask_client: DaskClient = await self.dask_client_future - await dask_client.close() + if self.dask_client: + self.dask_client.close() class ArchivingScheduler(Scheduler): From 74aaf9ac99b5c2487e413a7082f556373a04af4d Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Fri, 28 Jun 2024 17:56:41 -0700 Subject: [PATCH 8/8] add scheduler.dask_cluster_url traitlet, create dask cluster based on it --- jupyter_scheduler/scheduler.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/jupyter_scheduler/scheduler.py b/jupyter_scheduler/scheduler.py index 1360c70cc..3ec33aab0 100644 --- a/jupyter_scheduler/scheduler.py +++ b/jupyter_scheduler/scheduler.py @@ -7,6 +7,7 @@ import fsspec import psutil from dask.distributed import Client as DaskClient +from distributed import LocalCluster from jupyter_core.paths import jupyter_data_dir from jupyter_server.transutils import _i18n from jupyter_server.utils import to_os_path @@ -402,6 +403,12 @@ class Scheduler(BaseScheduler): ), ) + dask_cluster_url = Unicode( + allow_none=True, + config=True, + help="URL of the Dask cluster to connect to.", + ) + db_url = Unicode(help=_i18n("Scheduler database url")) task_runner = Instance(allow_none=True, klass="jupyter_scheduler.task_runner.BaseTaskRunner") @@ -425,7 +432,10 @@ def __init__( def _get_dask_client(self): """Creates and configures a Dask client.""" - return DaskClient() + if self.dask_cluster_url: + return DaskClient(self.dask_cluster_url) + cluster = LocalCluster(processes=True) + return DaskClient(cluster) @property def db_session(self): @@ -786,7 +796,7 @@ async def stop_extension(self): Cleanup code to run when the server is stopping. """ if self.dask_client: - self.dask_client.close() + await self.dask_client.close() class ArchivingScheduler(Scheduler):