From b9ca547e4d7098001e829f760d4f6cbaa63d2a93 Mon Sep 17 00:00:00 2001 From: Shobhit Gupta <43795024+gushob21@users.noreply.github.com> Date: Sat, 30 Mar 2024 00:40:02 +0000 Subject: [PATCH] Adding ML platform reference architecture (#266) Co-authored-by: Aaron Rueth Co-authored-by: Kent Hua <8052337+kenthua@users.noreply.github.com> Co-authored-by: Kavitha Rajendran <103603287+karajendran@users.noreply.github.com> --- best-practices/ml-platform/README.md | 74 +++ .../ml-platform/docs/images/configsync.png | Bin 0 -> 36630 bytes .../images/ray-dataprocessing-workflow.png | Bin 0 -> 92010 bytes .../examples/platform/sandbox/README.md | 340 ++++++++++++ .../examples/platform/sandbox/backend.tf | 20 + .../examples/platform/sandbox/main.tf | 520 ++++++++++++++++++ .../examples/platform/sandbox/mlp.auto.tfvars | 5 + .../examples/platform/sandbox/outputs.tf | 13 + .../sandbox/scripts/create_cluster_yamls.sh | 60 ++ .../sandbox/scripts/create_git_cred.sh | 33 ++ .../sandbox/scripts/create_namespace.sh | 64 +++ .../scripts/install_kuberay_operator.sh | 51 ++ .../sandbox/scripts/install_ray_cluster.sh | 59 ++ .../platform/sandbox/scripts/manage_ray_ns.sh | 46 ++ .../acm-template/manifests/apps/.gitkeep | 0 .../acm-template/manifests/clusters/.gitkeep | 0 .../templates/_cluster_template/cluster.yaml | 21 + .../_cluster_template/config-selector.yaml | 23 + .../kuberay/kustomization.yaml | 30 + .../kuberay/rayclusters.yaml | 22 + .../_cluster_template/kuberay/rayjobs.yaml | 22 + .../kuberay/rayservices.yaml | 22 + .../_cluster_template/kuberay/rbac.yaml | 44 ++ .../_cluster_template/kuberay/values.yaml | 112 ++++ .../_cluster_template/kustomization.yaml | 19 + .../templates/_cluster_template/selector.yaml | 22 + .../_cluster_template/team/kustomization.yaml | 22 + .../_cluster_template/team/namespace.yaml | 20 + .../team/network-policy.yaml | 32 ++ .../_cluster_template/team/rbac.yaml | 59 ++ .../_cluster_template/team/reposync.yaml | 172 ++++++ .../app/fluentd_config.yaml | 44 ++ .../app/kustomization.yaml | 29 + .../app/serviceaccount_ray_head.yaml | 21 + .../app/serviceaccount_ray_worker.yaml | 21 + .../_namespace_template/app/values.yaml | 319 +++++++++++ .../examples/platform/sandbox/variables.tf | 182 ++++++ .../examples/platform/sandbox/versions.tf | 39 ++ .../use-case/ray/dataprocessing/CONVERSION.md | 31 ++ .../use-case/ray/dataprocessing/README.md | 133 +++++ .../use-case/ray/dataprocessing/job.yaml | 21 + .../ray/dataprocessing/src/Dockerfile | 14 + .../ray/dataprocessing/src/preprocessing.py | 158 ++++++ .../ray/dataprocessing/src/requirements.txt | 8 + .../terraform/features/initialize/backend.tf | 19 + .../features/initialize/backend.tf.bucket | 20 + .../initialize/initialize.auto.tfvars | 7 + .../terraform/features/initialize/main.tf | 131 +++++ .../terraform/features/initialize/output.tf | 17 + .../features/initialize/state/default.tfstate | 21 + .../initialize/state/default.tfstate.backup | 197 +++++++ .../features/initialize/variables.tf | 84 +++ .../terraform/features/initialize/versions.tf | 26 + .../terraform/modules/cloud-nat/README.md | 108 ++++ .../terraform/modules/cloud-nat/main.tf | 80 +++ .../terraform/modules/cloud-nat/outputs.tf | 33 ++ .../terraform/modules/cloud-nat/variables.tf | 146 +++++ .../terraform/modules/cloud-nat/versions.tf | 34 ++ .../terraform/modules/cluster/gke.tf | 197 +++++++ .../terraform/modules/cluster/outputs.tf | 33 ++ .../terraform/modules/cluster/variables.tf | 75 +++ .../terraform/modules/cluster/versions.tf | 26 + .../terraform/modules/network/README.md | 110 ++++ .../terraform/modules/network/outputs.tf | 28 + .../terraform/modules/network/variables.tf | 59 ++ .../terraform/modules/network/versions.tf | 22 + .../terraform/modules/network/vpc.tf | 38 ++ .../terraform/modules/node-pools/nodepools.tf | 85 +++ .../terraform/modules/node-pools/variables.tf | 90 +++ .../terraform/modules/node-pools/versions.tf | 26 + .../modules/vm-reservations/outputs.tf | 17 + .../modules/vm-reservations/reservations.tf | 33 ++ .../modules/vm-reservations/variables.tf | 55 ++ .../modules/vm-reservations/versions.tf | 26 + 74 files changed, 4790 insertions(+) create mode 100644 best-practices/ml-platform/README.md create mode 100644 best-practices/ml-platform/docs/images/configsync.png create mode 100644 best-practices/ml-platform/docs/images/ray-dataprocessing-workflow.png create mode 100644 best-practices/ml-platform/examples/platform/sandbox/README.md create mode 100644 best-practices/ml-platform/examples/platform/sandbox/backend.tf create mode 100644 best-practices/ml-platform/examples/platform/sandbox/main.tf create mode 100644 best-practices/ml-platform/examples/platform/sandbox/mlp.auto.tfvars create mode 100644 best-practices/ml-platform/examples/platform/sandbox/outputs.tf create mode 100755 best-practices/ml-platform/examples/platform/sandbox/scripts/create_cluster_yamls.sh create mode 100755 best-practices/ml-platform/examples/platform/sandbox/scripts/create_git_cred.sh create mode 100755 best-practices/ml-platform/examples/platform/sandbox/scripts/create_namespace.sh create mode 100755 best-practices/ml-platform/examples/platform/sandbox/scripts/install_kuberay_operator.sh create mode 100755 best-practices/ml-platform/examples/platform/sandbox/scripts/install_ray_cluster.sh create mode 100755 best-practices/ml-platform/examples/platform/sandbox/scripts/manage_ray_ns.sh create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/manifests/apps/.gitkeep create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/manifests/clusters/.gitkeep create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/cluster.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/config-selector.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/kustomization.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rayclusters.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rayjobs.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rayservices.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rbac.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/values.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kustomization.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/selector.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/kustomization.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/namespace.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/network-policy.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/rbac.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/reposync.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/fluentd_config.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/kustomization.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/serviceaccount_ray_head.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/serviceaccount_ray_worker.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/values.yaml create mode 100644 best-practices/ml-platform/examples/platform/sandbox/variables.tf create mode 100644 best-practices/ml-platform/examples/platform/sandbox/versions.tf create mode 100644 best-practices/ml-platform/examples/use-case/ray/dataprocessing/CONVERSION.md create mode 100644 best-practices/ml-platform/examples/use-case/ray/dataprocessing/README.md create mode 100644 best-practices/ml-platform/examples/use-case/ray/dataprocessing/job.yaml create mode 100644 best-practices/ml-platform/examples/use-case/ray/dataprocessing/src/Dockerfile create mode 100644 best-practices/ml-platform/examples/use-case/ray/dataprocessing/src/preprocessing.py create mode 100644 best-practices/ml-platform/examples/use-case/ray/dataprocessing/src/requirements.txt create mode 100644 best-practices/ml-platform/terraform/features/initialize/backend.tf create mode 100644 best-practices/ml-platform/terraform/features/initialize/backend.tf.bucket create mode 100644 best-practices/ml-platform/terraform/features/initialize/initialize.auto.tfvars create mode 100644 best-practices/ml-platform/terraform/features/initialize/main.tf create mode 100644 best-practices/ml-platform/terraform/features/initialize/output.tf create mode 100644 best-practices/ml-platform/terraform/features/initialize/state/default.tfstate create mode 100644 best-practices/ml-platform/terraform/features/initialize/state/default.tfstate.backup create mode 100644 best-practices/ml-platform/terraform/features/initialize/variables.tf create mode 100644 best-practices/ml-platform/terraform/features/initialize/versions.tf create mode 100644 best-practices/ml-platform/terraform/modules/cloud-nat/README.md create mode 100644 best-practices/ml-platform/terraform/modules/cloud-nat/main.tf create mode 100644 best-practices/ml-platform/terraform/modules/cloud-nat/outputs.tf create mode 100644 best-practices/ml-platform/terraform/modules/cloud-nat/variables.tf create mode 100644 best-practices/ml-platform/terraform/modules/cloud-nat/versions.tf create mode 100644 best-practices/ml-platform/terraform/modules/cluster/gke.tf create mode 100644 best-practices/ml-platform/terraform/modules/cluster/outputs.tf create mode 100644 best-practices/ml-platform/terraform/modules/cluster/variables.tf create mode 100644 best-practices/ml-platform/terraform/modules/cluster/versions.tf create mode 100644 best-practices/ml-platform/terraform/modules/network/README.md create mode 100644 best-practices/ml-platform/terraform/modules/network/outputs.tf create mode 100644 best-practices/ml-platform/terraform/modules/network/variables.tf create mode 100644 best-practices/ml-platform/terraform/modules/network/versions.tf create mode 100644 best-practices/ml-platform/terraform/modules/network/vpc.tf create mode 100644 best-practices/ml-platform/terraform/modules/node-pools/nodepools.tf create mode 100644 best-practices/ml-platform/terraform/modules/node-pools/variables.tf create mode 100644 best-practices/ml-platform/terraform/modules/node-pools/versions.tf create mode 100644 best-practices/ml-platform/terraform/modules/vm-reservations/outputs.tf create mode 100644 best-practices/ml-platform/terraform/modules/vm-reservations/reservations.tf create mode 100644 best-practices/ml-platform/terraform/modules/vm-reservations/variables.tf create mode 100644 best-practices/ml-platform/terraform/modules/vm-reservations/versions.tf diff --git a/best-practices/ml-platform/README.md b/best-practices/ml-platform/README.md new file mode 100644 index 000000000..10c43eb52 --- /dev/null +++ b/best-practices/ml-platform/README.md @@ -0,0 +1,74 @@ +# Machine learning platform (MLP) on GKE reference architecture for enabling Machine Learning Operations (MLOps) + +## Platform Principles + +This reference architecture demonstrates how to build a GKE platform that facilitates Machine Learning. The reference architecture is based on the following principles: + +- The platform admin will create the GKE platform using IaC tool like [Terraform][terraform]. The IaC will come with re-usable modules that can be referred to create more resources as the demand grows. +- The platform will be based on [GitOps][gitops]. +- After the GKE platform has been created, cluster scoped resources on it will be created through [Config Sync][config-sync] by the admins. +- Platform admins will create a namespace per application and provide the application team member full access to it. +- The namespace scoped resources will be created by the Application/ML teams either via [Config Sync][config-sync] or through a deployment tool like [Cloud Deploy][cloud-deploy] + +## Critical User Journeys (CUJs) + +### Persona : Platform Admin + +- Offer a platform that incorporates established best practices. +- Grant end users the essential resources, guided by the principle of least privilege, empowering them to manage and maintain their workloads. +- Establish secure channels for end users to interact seamlessly with the platform. +- Empower the enforcement of robust security policies across the platform. + +### Persona : Machine Learning Engineer + +- Deploy the model with ease and make the endpoints available only to the intended audience +- Continuously monitor the model performance and resource utilization +- Troubleshoot any performance or integration issues +- Ability to version, store and access the models and model artifacts: + - To debug & troubleshoot in production and track back to the specific model version & associated training data + - To quick & controlled rollback to a previous, more stable version +- Implement the feedback loop to adapt to changing data & business needs: + - Ability to retrain / fine-tune the model. + - Ability to split the traffic between models (A/B testing) + - Switching between the models without breaking inference system for the end-users +- Ability to scaling up/down the infra to accommodate changing needs +- Ability to share the insights and findings with stakeholders to take data-driven decisions + +### Persona : Machine Learning Operator + +- Provide and maintain software required by the end users of the platform. +- Operationalize experimental workload by providing guidance and best practices for running the workload on the platform. +- Deploy the workloads on the platform. +- Assist with enabling observability and monitoring for the workloads to ensure smooth operations. + +## Prerequisites + +- This guide is meant to be run on [Cloud Shell](https://shell.cloud.google.com) which comes preinstalled with the [Google Cloud SDK](https://cloud.google.com/sdk) and other tools that are required to complete this tutorial. +- Familiarity with following + - [Google Kubernetes Engine][gke] + - [Terraform][terraform] + - [git][git] + - [Google Configuration Management root-sync][root-sync] + - [Google Configuration Management repo-sync][repo-sync] + - [GitHub][github] + +## Deploy the platform + +[Sandbox Reference Architecture Guide](examples/platform/sandbox/README.md): Set up an environment to familiarize yourself with the architecture and get an understanding of the concepts. + +## Use cases + +- [Distributed Data Processing with Ray](examples/use-case/ray/dataprocessing/README.md): Run a distributed data processing job using Ray. + +[gitops]: https://about.gitlab.com/topics/gitops/ +[repo-sync]: https://cloud.google.com/anthos-config-management/docs/reference/rootsync-reposync-fields +[root-sync]: https://cloud.google.com/anthos-config-management/docs/reference/rootsync-reposync-fields +[config-sync]: https://cloud.google.com/anthos-config-management/docs/config-sync-overview +[cloud-deploy]: https://cloud.google.com/deploy?hl=en +[terraform]: https://www.terraform.io/ +[gke]: https://cloud.google.com/kubernetes-engine?hl=en +[git]: https://git-scm.com/ +[github]: https://github.com/ +[gcp-project]: https://cloud.google.com/resource-manager/docs/creating-managing-projects +[personal-access-token]: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens +[machine-user-account]: https://docs.github.com/en/get-started/learning-about-github/types-of-github-accounts diff --git a/best-practices/ml-platform/docs/images/configsync.png b/best-practices/ml-platform/docs/images/configsync.png new file mode 100644 index 0000000000000000000000000000000000000000..75ed75ca4823226adb9767ffe476abb75bd87813 GIT binary patch literal 36630 zcmd3tWmFqoyZ5mc_u@|Q7Aq79?(VdO;;umh6nBT9#oZ|`1&UiqffBU1yB2qQ)B8E+ ztn=YM-`}++tXT;&vuE#XF8lu`Qe9OJ8-pAJ0RaJ9L0(!D0RbfmzTb)V68<+3N7Wwz z0i)DLN=jWpN{Uw9#nIBn-U0zZKJrTns(R8KG4%M_cP|tnUkSYfeFB}9@d5#3LPA@T zfHoNqIe+fU4VT z+nbBeB0De-D;xxE3^@kHKnp~9BFl(i-$k+njd838JaGgvb;K7xom=+kKd7rCh=T^6 zmj=8Lc=wLeRBRvbpPzqketrORAgJRFIk$ry*oK(LFJoR4SRn*x=Q%G`VvRExiR((l z5J#LXDX6Ax=PN)mw>h;0^LwZfjK`|cUg5}PAt1<5#qzxUj$VdiAv#1XnIj%?X5PV3 zUT!0t9G7PwvK^tGGi5d4TbgvjKO#`h6rM!8wP#oov&t$@QtagNGh&j7x0kwRzYNo1Te=^&ixgWZMx|qhA=p6*eM$nzY6Exp_^)cwVewS_y~<_WSce zrN~K?f{HKCLq-2I?06}8tEoI!8zJho@&mZ1KF$kSHTiYMzpF^6X03{aUSDL4peJod zzg;Z0iqs4$x_x*>V)KB*R7qCU6RJPe|{WnnnlK zqY$lsp^9(y*eg47A~Xskkbi0+aNY)E@!ST#U=T;f5GS%mGBD{ zQP8c~En}zp!ZiNHhafjvnpZ&}V`diQ`))*aY)H^X9)1=gNzZ#HLU4e?3EoAZ#0f!6 z05=zo1RiyPBv_tkBItrn>nDb(Bu6~SD})zP8tQM75zy(-8wutSh$Lmy$r@?C2Kt4n zN|EFrs566U_QS*So5!q316ga4DiJBV&yAUW1wF2*e@4$j;0$+N6aI{;kMXfrZuR92 zy$|7^u=~~26BZ=Aw*?{A082E&Q*opOYX)k3Kt%#l5lXrAe!OK7(f8mBx|(=3Qpzxh ziSaKkRZ=5m(y(Y_xb!GyTs_0e`4ba>50w;wQqLz^skruDT7y8T&2>emdjrXS|pRIgobAm^^ zcQ!mW;#NmD+D?LR;!xg7L>tSTFi8`Jgj|LY^qTYvN|LMznq;vkO)&!rkRydnahzYD z($=Ma&J>WRWFAbwQh*hyYZBN{l_zV9EHa>;5Pi%{QNpOjNS%_sYpdZHGm!u2O;+UkJ zWIWH1E!vyZX)JEk|Ac;oa>RG^<&DO)__S0CpzTwW{#B!W&0!g(0Zfxao#VYj!=lOe z0~e{Y&z{JhoSr#8IS+`pN@vL6kZPstJl8#nUDUmgyTKR&FQi^HNhV6>_iBW?hvLh4 z%251p`eFaW6!8k3gZ636F#6yh@+Qe| zIe26ne}D7+#yEffZQ&c#4a=<;Jbf^AAUw^u_ttI0E!s^FEO*g(LA+ZvRq@+9Pi@L- zpLzdy!twXBv9UFtN$q&vxbaW*AMv0xI~}{5pAoj3wvCHr6T8*iur!-2`*vtc9|~XH zI|W`={_h}+%=vT-b_&>hIzrv-WZCcyT@!;LG%Cw`sGU$fS8}w&x z0z@+F&%-y1&ex+M=PI!8e#yartI+0Emc=hc zQ!0}V_hu)Owp^Fz)wSKap^YHs^}P+<)ww^6$3)xjw=etkph7)u)875iX{*rD7e1D#xSM&RDGkbO}rL+ za~6#jSr*Kub#?f4;bvVC>BBd3Y`k~iJ}asOdlE5ilmFCN(nU0wA3)SLI-qz?$<5m>L0`jRyc_aDh&+ny$BQ4w9vlmv9sMQd zD;yIXgVmiA0TU%lUTw`sPnsz@(*}PG4UO^*>&n;rsF!o9I~=?Zu51opFJ9kT^^9I$ z-5)xZ@s_<23=oRn_)9~Z{+55xkY}TkDDs)o2@i#-dE3^ggvatPz1e#x-Xd=adPR``53(deoLco{FF3nn;_S zoq6?p%5(l0tkj@&UcU0nzXDk3Tk=r3K3BCPzjFpt$r=?A^O69s{r(XWjm$p$iP|u# z9ejOOFnsj!p+>)L-}h4Ub;aewyB#~LY9T{0-(b(e(~_l;FusK*t!8`2Y};l##}<%( zTX|o8;dfWRo$nko)zKcp$NC+^o!$-JCLUI5I-?!SY0I77?cP$?T7SAc<8O^0*5>7A zD(4-1$_-o#_ogQEMy8Bw)i`*~dgbDi!bW?r4mAj)cdU>9sk^4n>#>yb!-T}4u7ZdV zrdSYmsSx6c)kY_@UaE{fYoR<5s*NsaS)z6a8X`!3M8I>x3|O+*Wr^OOG(|uw$aiK& z(9=_z`L!tYu23yQh6*8TO~K1ty@mLpuVm-?c>k#sHR&tgYVMN}PQ#1SHz3Moc;+f# zp`&1_qJqE%-$z4047Wi*f$t&0KjiQa0s>MV(tm%0l9Y%1-}?xie}4q>zI%^=Ada9Q zEusAx@h}@TO90ur~UU$ z7lmmUD1I*^jw1c9TQ}OzbaRFa)_*^mK3+i_R-`Nq{MTdQUv{H`*N0o2Hx8Rc|IbU% zCIomejfmbKMxt7WqeTS#`$meW;>tsC_X-go@87nc47667O!QR$$8FLZqiia_{r~sy zG1ham{~0?z60X=oEro~|*v2lE!(h?tpz7mY6(_y=F-pY{V^*82PMOX4o*z%@NA=zB_hYhKZ z#E##$h{Jp43*~V6A1@|TC!XPBTrATlT+~!z-}<2M{A0r17eBCx_GCNLv9&bU4|q8t zbXXgFFk5G~&pkiagu()~%JrT0Js;tB*iN@{$K-6U)$`o?VnO?*Tt`*pQk&nnJHWC$ zy}^3PZN@E5`0OXcxZ)v=sL$sH*Zose(A=ol^I-#|XJmC)-*sf`!zUZYnQ|>(lEj2AT=xsJeQQvQ+8nyaA->)ox_~iEc{cEo(FV%L_l#_~@ z&V9d|HB_;)@8>X~R8rsTC67PFsfK4%ofER~`QoELnN$<|J}P?u%P3nhMk9l@B&|1Y zN+)GELcul_0k-FT+%c27;B!8*W9a|rcKvWZDnzL)^ryvaL@@c!_Z@1v$CjNOZ@$R} z-rlKn_{nn@tP5io>xVfq-K}M=WT+GrwB{o8>sg(ACM0UESsjyq$B8}d+K*X0+?@V_ z@17jU9CbZEK_DI?Xr9|O}ub@y}diASxUFc*(26>l9*}pI416ioE+uKuY6pqyUS5Q$N_(LLZEh^ zHF3zWDLMV}wAvF)^^!Az1incA?vg$DTa{|-69t7~lZowep<&To*1odlSUWl)$`|dG zE6%q+fzYcBJdDr?fGggdsGiS#Ss{c?Fi9XBr}YSPvaI*81w^mO$P0y{DZ2_`(g$*>zN;xh9j^|G}9iA*k@F#{aP*|1{Tmi-bD< zJ+baixDBIkoD$&c^|ERs6ugP2@g?9+BLRuQuEzqt8yZ*cH9)L74xVq+1%L&hR!VC7j39p&{~R8z`LsOh{LT38)0pVqZ< z3#axfd$hkw|ALAGE5sZjsG6i9a%1A|eRsQ=)VAQ*wM4NFzF809QtGu9yWJ!dJFB~x zP-?3-XuE@~y{>6rNAm@=$H*f?==xLCePtu$Ae4K4kCz|CK6R~yQQZ0J2bYz>ZN0DK za#{}(?ojuG*j2>)3OuR8Z=g@ysDfgO{&2P6_~)=;^s3NhI{o44{&=bD>5`NGxc$8K zsCi{M8y=1gyJnyMv>o5QlixM2{xJactVj~~t=B*UH0aZjgG5uq>N4UBqnne7dWAeiP2=0uA_U9dO;ttNQ&QDL2us%IWU*0hi+1D5JZdbfOc0;H=>1rN zwP1IXQI}t5Y*CH4uUvL__e3Bn^=^$D{1)#HCO)QF<#_rmfX3Z*EwqaK6xyJ0PY-+_ zSKYFXv2r!}QR3q`;P9nAS;cVuVSM8bVZ`&}vA=J6Pwv5quA}Dg;}ASg7~}Cf`ZnX! z{}x1XBrqC1m({qJj0~b3#r8$pX9(G08pJX8|3Hy@iIphZq4bHOx5GjNfR)&tyJt70 zGG&&GzhmHcyH#uzgvc~cV*ia!Dy-$*v`*iGmOir3BYdnUT7Bbk6UN^WAdm2eWXW?U zd*{Rq@3(@l-0dVyQKy}DB<>FD zUQ;LK1^zPG6!2>h#Ta1lN5dW;%a`(%o?@%0T)zPTF@oRIBgvqx=H*Sw%1B8C<|*wm zv?^|mcx2&M{+241E=Awifu2J(cB_86eo4ghF@hT06A7cSS0wId@@qC}wca;844n;X9$u zoguc9dl4D^N}jug5G=q-IGPa2fZ z9g*MTI~G7^b9;iic|#IAyZQ<^2fpdW5%5~c>CXX0dkuRH?8Rv2G^!+XyJ3tz|15!# zN?MTs2n@`{yj0{e_;Bjn<2RgpjpO}JdQdyhw{flShpHmzTZYQQhKH!BMtZ+CLr>}oouXQKakWZcf&W(;Q4a$29@N!ArIKNe~2s&EtS zsJXV&q1=I0wG?#=lO%J9st^@E5O)(@q+emNN{?Txwg(t(r?QcN=FQ>$@feAk+(R&l6%pV@}@8<>fk@wl>WrT9g1cL zrJKn^ah##A2lY8U6QTPTN*^c^FJshkjl);-1`P9_ECb@eHcGXbq;JfHRs>;(wnMo2 zkNILyT4X-qBfw?Hm%>Z*xv=U9CiT&i!_X_QY+?!1rq|@@{qOfapPs0*4#c7$=G-Rm z{vr*M#u-Z>Uoig}A{jWl%O}f1bjY`0zd@xyN>`fY66F|S{U*Y=cWl`>7~5wj7EJ0( zYk-9~{USa>+z=@SCtv7u+PD_Eg=}n!p2K%d0#Y{VVY-+yjXCx7vC}@*z_s32NCnzs zx9|jSY0X5ghOr7DUf9jT05hi)2KPpO&MPR@$TRFIm(G?57N-gz%rxu$$wmCBG)Zlv zr!JYN5W7g}c8!!-1*I>=D{O%7j_*wjFyulZ&aP01SCcfkD0V#J$(UMO%Q;ThF+BLx`$3=I9TEuO9g^PFE*~6FuE^5e@P9tngUD z59OYeE|cOC9u>LPcl@k*CGd7!Ms|YxY@BDF7g&^E2)T-dAV}U86oTiuLRwzlEtjg% zi1o%+*X{Co*?MiM#oT^u98sQR2aBp2MK(#LRxNMp_IFFp*j&T<J4DmCfL{J*-Nt z`p8xYfWWQftGta9$(1sta=_k{X7gH$bp79uARtx)akbAd&ZBYaM(pe?I^63n-V?d* z=aJ!MhHCf|^eEg98%weXa2H(iT+wIqW+@0qV=>Y3iR2s$_@|SP^29XIpfVnbLDmXJ ziT8dNjCmH|^e>n6DZhF#N{MgQ7`%&miFNRbUK*27C6Xr&^3GidL<@LGMhz}^!6!1RHF}VFNyCmxaz3?5G8A*v1h8!#y7O!QfXdf;)D4f zzW$gNm~!DX2$SQ)P5AOt3RU8zPd&{XJi%A;QoG~&%3uD&kI^D5Tiz(ZD?}7YRV4>| zpOx_N{k;6VbHSvuZK2hyzl}RB=+h^PQ6-7`9EoA64nNCpbh@KxZh{HMQ>cxgStn_+ zjVYawb)`Jr&%C3yMSS%)SuVZ6F4q3}8H5J)DYQz zHpLh~$T&bSNf_gCSRp`_>@Ji$@*K6J7fl8i-A-2Wv8XTZDFbHGctSabr&EL&izyzYX3@e8EUBueGO>{5? z4msMPj+5*Y&a1{S*a>lS=(32!y#n#*73c{ha=Gi;y?7irk=<2p(k{z%CTP5Wj7d$zv;I`d{FDa=_i%_27j*WmoSW?(Z<3YX-#V8+T81e6j<&wETT{o&G&@>O}cHwg8jt%^1{DL0RMtV~U=7irG zGKcm#Jo{eJKSKbc?;co{E)aT_dAN;UTv`?mN#m-#IL0%>4UWN~rs_}+@ZT#=y~FpC zJ@d>%Qy7RJB%sa`kl4cmI>yY>`>Z1vH%k9a57eE<6ih@2WhuDT3n3BKUb`iGwR-@d zJRl;C&K4>?I&2Dkqy;+oZW$#xR!8Ldunbwg_NUIjMV1 z_MWOE^~A^9{d<%1@V$NL-qeH#fldj=u8P)ov3- zG%U{#d*DKzm=irK4oJD+$4`Cq3Sw_o6fCqF;mN367Mjc@%b%Q$c|1siw}O|bU|ds? zyf=ZzzRyz7|E8xUjT(yzK*%@Ne>>3=6j2!| zov+?qwP#M~9(^*{E=MT4V9m=wubzy6jRIqk?H*U7e3c(^H^f5WWsmO^qWPAs{}Y0? zX~g@OH5Qq)oX#CUcNsH;6nNR;b$P}-9&g;Zu3V4z9*Bd<7mLU3hGj)dHDC0S*~@}r zkAZRy_(57{6%8sMLEg=)vx{u<>%l~O%=lD6goFtAJ0=BhNG**=l9}VxaK*b%hke%O zwg_PYv80#QEHdZ#Ihof&RL&)J#WDfm0h|WCl1fvaYbsfcf%M}9P^fzdmpro z#x-)}D2b%;>G{~KuZcv$kKn1CAl@g~xv~!M5>YEnr;E9UXRtc=VvgF4(uzi-F_}k} zADC9Wp$kkX-nepkburzag>@x#mo9hgj5antK)flWIYn%qfj>$Tj3it*z7uH$%MqF< z0Vf730o`SCt9C`E=N;$COA5K=gU(ZS$d<%wW78syO|v|wJ(&a{|R;~t$}g7HjZ3p-%$EXrD+jOT&q2e z_)CS(`3{CTcywac+9#8j;vazwdUY}lO2>lAiv1Y%$4Q3L_Ec&u$myC)%k)f)0_&xB zUz`8+Gud-Q3ZurcLbm(bV?rf=ZvQM1JQ8*S*E(~IBNip8Y%CI(bh1rub2-Q-Az3Z+X)S*~ zPXL~<=g)VJ_TUFf#5xcRkj6RzkX}k99r4Y>c9|uuP&zC#m;oP2M`CowTJ+LB-srWKe*0TZq+=<4hSUkfc+Ta7;=MF5(KqC*8PN1JUnN zWK#i|?MDHm@qePl&D3?#BZTIjfLGhr#EvN^n7SlsdA`bTv z;GkdU6+ei2##}t^V6vLvi!v&Yf_y3&`eyWHW|>*-sFVEKSOH<~^hXxWuK2h{Z-zKs zuVgTd{b>E)$VS;etB{OqF_28Vzu}egY5$|<(`&KhZ{nz*QnQ*VYyz@uzi zgo+8QfI*0>qI&oo>l3x@Eg>`Ir8rBi#B;-Ze5F;$9Gbw1jw({yf<1nNL<^G}(d7#_ z61Pv;-cGFTV)p@CAxe9ewalhWOrJiO|h+bD*NJ*#ixDH<^@W@(bkvQZM34 zbB3}8g2NE|5Q}@vemaMQoN=wA736*1jdyD7K7)AygzaB(;Br8vPBGW>y&H{#2+!_xEU@q`LvKH^Q=wVksj{9Vjm?-R z@t7P1YwU1blLcX(KG!t2CfV9eU?WbZm4BrN>Z2 zc{%7rDfc=*rq_UAN%@QuaDuhhL>wBVilx=t!UUrQ-jZTV=No@+z+PIX#4?zG&H|o! zy$+#ds?=p-k^QTFEjTz#6ZG|j8}TfpSr)ptC;Yh^G!V8D_aEpQa^R`YFYS}ry(sSw zbl2zK=6FP_7QWfNC=%I7CW5YNm$Z}46##}8ipS7N&j@P5p z!}xHPB*mCOua69%bqQsT%BiceP>O147re-5ztaD*avT(T=r}rX ztFbvt%Gco*KTe~kL%%sNbOP>7wQWAa)cE=iwng*xj4*zTovBu-+ytXVdb*y z&~~)Fpw{M~X_RmMnCKq2`C=`@*-IKqjEeFL?`5xzcZ@f@ps=t?!~tPlmCjL1yQr%Q zV+9*QujO1;LEI_spnan6U!5Xwk$iXRP~g` z*kopS=L2_CWL)(F&G=~djPdxmf}2)Vs$9Z@n>EI0hB)KobkLeoU`m^-N~p4w(I4%Dkyd^hhBq!Z2+W}WAxL@fLQHGr$8EVFDCbUTJ*QZy|bu< zNt}_!6;VURt(WIzOGiR%z7+j!`+HHO9PN5!z4B~Yht-6GbhiRTAd?J{_>Txf z%!6cEZhiy+Sg^mQ5p9Ki)D)+Ur}9K$kq;D93Q=y5n#fU`?GqT}h-@?LaImZJv(WdJ zktRT+U1Ng>@c? z7&dYtRt)WilSy)0v1SpZpic4iaSO~(|EaHiz(7oeMv(tCjme$DiZjL!wzIcfBJZ$WF0QB%u$XL9WVd1>18O9d(dJye>;4f>1*KC^ zNHE>_{jh18n;);peYTt7hq+hdgDrMMtoRGnqRgx zEWy0KiVJe-<|1ag_Xi#N4fA+^IFHB_T(l`th4)vDz7}a9#P|22jR@D44{Cr85W@m;dDo;sE@3p7I1mt?U1Tc_3nf zg94$5uRZ_$^wtbWE==z&6ea0u}p z7x|Y_{xfv#w1ATD?@4PDN&f9V;U}Soj}-8w}{#lO_|8frs}5T_4B1cbLD7KZVy zh}o`>J{fp_e_HZVS8#_|`o=Ln2O_d>^>W6@LrwJ7e9s-|w!di!S9zKb>xWyv@8q;Kc90wSJwJNz zyKKr|!>=aP$qARk9KO8&aC%!laCz`ZUN=rp#{Xlgd94H0g7Tg7b;GFW!CxWJ>6f|( z2b{2QSPLb|`7Dm@-*(iruvlx-w+Qd;f9(5Sxm5XHs_Un85 zcE@9>&BFuU8TLI2(t!OXKr*_u)I_!pijF()VIw@F!VVj;{O&v>yjl(3e`TU}_8$Q+ zE~jNFK-wu@2ane)9&nnX=^oB*_-1;;Wx985)sNQuqb+jOp8dY?^Z&zG7__;&VX`;I zA6@0<`d+0s*U8&m%~^TC6~<~kdgRE0GEaRs;N;KPOHWAyQtbkBQ0g2HEbm&AFP zy*Vq>G>R3a2nT(YvIVZYEGBEI?#Xp<@%`;b@|*Uf&D5F6HE$*=r94*u>3a6#_qwod zUXOa6GV!z)PO~^JM|1Razw+E_Q<6TQB6_>|=dUc~Nrwv$`F49czLCtebMbzsLIh3g za1Y4TX*Kv)q^joOc%@U6scT)^oG7q}W~khxP# z&7*Gfcn23+7ZsSZH*>wSe_%q#7mxz@xMy83?puO` z8_NTLQG$P5R^U1c??{zEgQbYkEhf=I<1M)?o1KMHpiXUS*cK#HJ0XV^f zva)smt3&7Q_Ui**JkJmSK?iq_aqq0)RCbo*x>7F2H5ngoNC)6iTUiO_!|7t_|v@P45jkR`?Hy(b`{)3>AD`t+)Clt zKE9wk4!-;ck&>p(6z`K`;?72fBCItxFSmvQo^g|aUm}A4F6giL?AMB!@CJ@(bkH2hN2$^#uKg9~0gPb9NK;d`_u)8HK4wbP zy`HPejxAU7b{)Y{W|4jwaCU$ye*nHLZGgkngjW`*|xZi2pRg3!qd0lSAWZ zj!BOS?Z@kA{2#_CK)WyA&Y36O3TrICuB;8Zhflm#hM|Rr)F)B}@~N#TzOpsrN+=?V6(0>fw}||yJ;iZtywkQ#LB(p)&v4JU4?#G35v(teRFYxK zmg|40N+~?cD-ds85piOY7wUyfN4~e@v2w{b2xoDQn}0kOF89F-mh-a#py=N1)>+|M zfkuN_i8Vi^W4OD$Opu2YBo|sIv#LTr!TkFI(U$bRw%KmetTG|4SbL@$j^BQNa05M) zV){nGL%sMnbZwEhkucJ=GWG-xgVvi%g{UkgW(?$X7*95cUgQR#e^nSKdWMZTz^TCk z-p3JG4bVJ$*cJeu3S``jTogS$-rsZiW$}{Up$)Y8ax&ui2yVpiJOBlF+m)@>`Kg`& zal#Wjv0-VqO9DiAz&l?P`%qN70l;b z0&DSkN?6@UaY1rEucOldVYg_iELrmYj|WB_f#KC--+6YBZ%xd(^{oQr$x9{^nq#5OXwJ?S4|0t#D7Ck}%{-$l z-{pX9Mb)-k7=M*N;w; zT5;;kbJt$(%HAw@@N|6@gmM_7UK{G9tVaPAM>kca6-rsbnr1|YFw8fSS!IpGCEg+T zUUcS`j_c(GP;Jl4;?@wa7ny_hrPf}RYyBCD8u?V;BtG3CRJxLm0NmK{6{iA;vB_2B zy8Z*r!YQ|(#8*GH<7I*R~E$t>3W)G8dR^yT=s0e^Q>37_k zlF5Wr%CaBu=;KM<2FQBKFzb!ArVvf~(6CqFpwqEXy1-n7%lDi0=*Npp1NXTfG*X{X zsXyn%iPF_1`k?G?@({SgnJQ1YY%&0Nx#1$rI#AEC>f1qp>7sPw`_wSM1u9Bq}>$Q%O(6JCjM{Ncw*r}9`F5AQET z6}aIliR|Vwk#NZ37aq2><_U_%S7u%cfm{V3A}hLSPR_e6<6FZR_L;EEDHTa4geT$7 zuqBSKXI9o@Os?dWx<9QzQG~q&$xlKz49wu9mtg`gdF(!|)+WCK{I#T0ogtuu0mB7! z`jI8^+X(HWj;FW=57iF*0J7AwRqCbi_CYd*Ch0Tkx)WGaO94w>H`*pxjX6 z>;UI`g_ao{a!=<=*Y4RMZ=C(~J7cvXU|{&0gQ>@eXa2J3VGVI*sc9TY=zrH@Na~;# zf}4Q6Ex%3*^pYkSAD7*&Y#?RJwkISP^5dBZ;$c=rEB}GeEC#VaNt69DZp^d2cSuMj z$v~j>G0j#S^%WgKB#aGGr9Q1=7CWIPt7U!DLM`b>#$JHPx7G?42ayMZW~_t}3tZqy zAx%wY%+Ja4UK?v!RqBqO&ui_qlul>_2Bxh<{%$)pQeX@mB}(*NJRGU4Eae(0C-8&> zuK}Ifq0%G~&@*BW1A>P2b<8HjkWkPp0rx2c_&8I1{yZ((>-4cQ4@(S02x zFP0-i|Av)wx{;o(ABCBA%T$(k3nFUFn|Y!f=0;#)zSVcGCLm@J>)I1?29Jv>4{8@N ztnvat?#* z?+fKPg<~d4`5_?nMnm+tNe?*fJis9CExY;dC_$xdKp}}i;lAv+R?miFWPl!fpU$3q zek=texDlSF=hI|iA7!PK{CAC$h@L!ZtI{cZnA46^JPk}NASpF5V~di>ob?0FDD%*R zNqY9Ac7}o0?pfLJ2eWBO6upqq_r8w{s_^3#(zHXbMt4C#Wq7|RA-!q~)85;Hppchc zR3v^Ad$eGZH80+uF&N=UJH;uV*cOY9(NWyzZJ22cADg=#M^edTBUFUm6z~hMIp2f? zEgP@mT_!Usi_mF?vc^=7_FeE~7KCx+lfV@@LJdyo-XeB92N=pEk>p@{JrTLnE)W<& zSV7IGFOXSoR~g#R=gMLsnAozz70dfl`VpSxQi~GcSrLqKYmqvy(h4N8Sj9MG!IX{a z;gbtyEv)rnS=e<|no5dANn(Prl`O0wY{kiB2*#oDmf+In_X+RK&*!MnPQ@zjk>D%r zitncVg6ST_7py>Qi@JG(0VXi5U=db}#0tj&&Z5v~^@ zuj}WF{HKQ?&KLMq%@wo39H+R5)p5(6!5IV`kw%dQMDK=}^d~W+SRpKMmFR*O`ll%s zqE=~p0Wp^K=T6$=-Q$C1RpirE#uD#Kvd1Fu$z%$Z`UhuA_HYE=+D}0zvMwR$955D}Zj})gP@f z&q#Bo_YxCbOi})R6JvWDOI^Bhi>@uz^~-rA!e^>n$+!PPx1_XK5OTgU-VdlE{4v$k z()79e`dvN1Tc(IxfW zw05lc+tJ2j=XaXRL;d8+x(Qh02kFQ`522D^Ovq~mZlw0P;Bmg`mt2Dt`~tLl59zDP z=IjO5Q)x##giw^pd7U-}Hd0x%1W-g(@ehjLvANw>{Ql}gVw4{QN^W>0__<%D|Iw@= zm|1I{`EDhcc(%xM&|_To2_-SxY|Bgwe-6?Us}&$q<5>;To`Nad7dhmMMu-jz@aqzy z(Lp)W{7MwoOKkxGA^o#i*RZmH*W4dnogZ>C#sP zIK_&<#1`skjRqzZd<_9`+Ew434C$`2hJ^LcwBZrB?ukrAC0`(6ebf7+a&L-G*`|*F z(+zfy7|>msZXCaVup13G8<`RfS#gi_B&LKoV-X*a{IdUy-riU;prZC{+*tu+u zOPsly%jwI-0F%JUxE~elQOt46cnbYsB5-tlQJ+|gjReS0%r3ej7_)=hsz4^;VT;rM zbdt!KW!#ye7hR4P`18~tz}n3gP_sS5XbgurC`uevQ=Su47nVc(+RjVKq772oSy4P8p^ z$NOtXK41?;3s<#g2VZpZ+q>RY;F_KjSB~?86D2lbjbJQ>R+0`+WV5!D0l5Y-r$sCw zr3#ZWs1Lt~{=0J_o4fu5^B#U={~deG#H#u3Ow9_zfhc2k&Cv)tmv4+JWwzkZySU{s zI^net!bKSpgH4?=iZ$V(Fn+uAJRX1?g^?{A${2EYlj8 z5R1R0X_0QQKNXu+oVvl2*G&|u;vEo&(K`TceJ{xuy} zXP2*_c^{>-)(#6qy9MN+5l)HlLc->eUhYkrs%|E7BuD@oK4Tuol$-oP90Vef$O)eTjZBgDcyoQe(e9fgoI5gu>k3BCNkRCd2a*ercujwee;8Ia zc2x%h90(B*^<%PGJC;H3F7`G+OS^(s6g#hayP2;9l!Ulgq=Y^&3?=X-N)VZk68z);%`KTyy z*X*{~ab-lEV=T6s<-LIJnn<2Q6yYXWlU&N|gRAsj>^6vsS)rAwA$@hYVM6CiqJawje5Fn>nkL&N*j=+|9ufGbXCSBe*TZD*V6d-U6t~uF)SwL_$Jf zE7F}J-JQ}Qf^-Th2_oTRwj4f4D-=ZnsM2@9N_x6T5n zfhHx99TcxE?o2f!YgqoELA+hb0ONofjcr%$@=4>$^cuG-w)FKUI91J*D$WuZE9r^x zFU=aJ`7bfpI)o<+vka_!%=7Wa`u``vAqW8!O*Z^L6U+bp`2$HN_C|t$`VZ{>PuJ$q z0U(IR4&DmF{K*Xb*YCsv;OZx3BN6}m%FuWIKmYIcADe*Tb%qb|!C(0+@Mag~KkU!| z{~PiDS$6*-waVRK6E`OQ|62+Ot1cHDgX#RIdJg0OgyOeY^(1>s*|U=k)u@Ck(fM(y0Y*PlmbZ-5WrLZ14kL zs%Ga7F-ATctmtQR_Uy&%&}<1Y8`foXWBC(cLaDJ??WoAppuee?YU~MBLi&yHQF=Qq zxOCRvp7bVkZo@$PRK5``-F{=$IG+-tKnhRQ_&KfZya*as8y%GG0pN0>#I;Ev zq4ScrWloH;t=oQu%8cIIX{T0Hj_TTjnqE{K=Io(!cr^SiPk?tj&DnH5WwrnPpFLy>c^d=$6l9#6HU)R;yoWzY>>MKj<-49 z)4tzcHI7jk7Pi{WNo}`&K48o{09K`w{{t!sB{meFYb1{ixH)S9&oxO`+yvVnI{XA$ zcE-}MCU;IdRe(>gP33c|0XFtZLw4y&%dRf+12mC3IVY=h*81 z#Odz(+>Z0T3?TLuoJJ(6T&Pe(lNGYWeq@Dm_dKbYVxqfqhhpGb&VWU$B^Sp``3gW> zo@8y7KE_XOfXcYDO4gtx^1RC60e<3n7FaTsu770#4DznNPjOJb1W5D^nEUI&MpO_^ zUF3u7TC&}Mjlk`Zh*@MH-}Bicz!W(GFuas!EBzDxKrSF&ST3e2r>2Sr$aQ-%#;{)G zn0*F1fUaHi(m9NfHX<~NWVp)k()}<`%n+aEx?cKP&dke;c_$rzn_k5I7!~d5zT(i- zuZ!pM=_A*VNRlNM-A_X%8So|Sr|9d?-W*noPMT_^0>Y@q+I5^;d|e*S)EDL<6=#6k z`ObREE060l8(`V4xdGs;#~RPiWJbM@hjWF18aoIe3q9&g1vcLX#kZl}Sok8Mx2rIp zA*9{O^s0 z0U$_*xz#X%1A;|5DdKf9s{1pPzR_3(d+IU*!gejx z5AuXu5ySOt3q#{W@Z21r+AI?cV{Oa|OZ)`<*kTCX_L#&a7(?bMlgw8(KsisHPIpB| zQ^cya&{C7V@VY(rhuX~Nxpxa!7hHGg7Mm%~ocq}(?E!m{xjR;%S~4O;sdZoiIM&d9 z@i(bI=uw%L+k#$}N^EP6=v&@^jcu)LXXz{XDl`zljRN-jH>EvhfDjVd&UT#t_x z?h{+IEXVwqfPbOvwjtb$A{}hfXo{2yu8sJt=mBA;?S~bVGr` z*K|DAEz{`^8uWZ9&%qYY2W*ORF45;QJp1LpoG~T8tWNtA+7Oz6&BMGW9sgO##Ya{s z7h@$lOvpGR-x*rh4G{uei)KUY{yxsWpNiD zf2q$r{fWF*pw|b_Uj|X_oma2phUL@KLQ$2?Fw=I`dktw%_0e*aDOaIPv{Un==hbOm zzou#FQSo*$hL?jN!FIwUuNjkh5ohDEK9LMkBR^@25GUN`L9#I^Mm4MhZJb~?Wk@o8 zv?srpB2IHfJiyvmuZ<-y*}o)!(Nsj;XXuzw+ot_l!zy}iM640v$2e`*Za`sqEYloJ zXjq*;slk`*FWwUOMT{4rmOse=@F{vQAqVBTykjP)XBx}H)qrCq)=ye;-lj?y> znM;b^{!WpCFGgoJV0KD(p)wE z^u${@+(JoB_FUZ3_5%S=1sSaQ6YQ9ZLaFF@4yW|P&O4r1Jc5Vp<9)Oo70PO;I?O%% z1#<@h4M!T5Q>U2DWL73$8kmh_!()eMyNttJdtWvX%Jdq==QaQ#k&++Gs2Wc2W8ZW* zSVEa%y@Y|!b^+cm&9^~d0jmSI1=rs8%i5HhZdfkst0|}J&NKf(?%kI<&)+=J3)4G= z7=MJ|<9RlHcyIvPO6hF!TGVx%z1iE(zQtoZT5DZ#T+uBNQTTZ=Xc-w9A^{5W)eJ4h z$aXEOK4L*=lmkTE-i6;kE(MeJU4RgTBUB2L5@U{GzL)r^bnwlNC-Dzk~xH?>?3=lK!!p)+YjUPD?HuHuTFf*Fc@2jmz{t z`k@h{!_y#kmNzXeOM3M}pRl-#UJ6{xKxwy@0tq!1ZlY&rMhh;RA(jKTAtlhcH1-Uo zlT?^*Fh4<3nZT+O!?y}**-sOF!E_#4k>bpLrEl#M@hUf50?L&{!94srCRy%{m_pLF zr+J^XO(bWO`7X-G5e*MVW13rsvuT?c)22!Au>%sp45e<{Tq?%rdh+bWMz4Nv9(-ZzFBNQW!PwH3>*m1MAQ={2Q@2gnwyt}M3 zo2tMn>b)cWLwA&=>0X#E0o`4_(OF~=&2Z>-6F{cD5cACNnNyfWeUX+ZB@W<`ZYxBO zVwrJUl^~Ez+ssk9DgVicSFp!|D?;t*;;*SmkrBUE7?biN4eLk84-;^Ix#(2lgmC+^ zlYYWvo&jc0J8~QRMVmFo8pyK~tL>AYKudH}7oJzF*v#jf5?jb?N6-?QAP|OtMe^d4 zoUJ`~_+o(<>1#b$E$;72d0&?~A5-*3Jmp4dIc+%v{98pBJUMxc-)eCy+9?@|sBioc z@6pS6jQ-OPdcniyrE|8uG2BTqwAc_}S5LVQH*iw4Z16>dE8%V6B}G4ZI=8Zo`+{>r^&k z#JPVkoFJM5aS_!rQAoAf9y_UYgd~WPpvOGr@FkXiz;yjtRK#F)ASbH(#jYrVkKZlW zq1BzRRJ_!D8qT`KT6%!g8K3P!W#ZlnA8%JUiaqHJp?Q|uzr`fg=P-IeLpwM}wad>m z8D$*&Oj)&=@3;-(d6%n>*3Hs!ZH@x#S(K%4U;8|%W};Dpp|R=Wy~wq!rg5!F2>C2g z{b~7mWQ)Gbx63L0CqxgrsNpaK>%4~7ppWdeGRkLWJ!UMo3ItgHVWJ8J((OKi5K9yS&0Ie9F4QMw7HeO$idS;>~DV^hj~p6mU`OOZ?Ofw%N*_Dqj1nJJu%PPdWj;2kZ2+( zs^c5ttY#$|7$}J1oln+cRCjuS6`u*)lI3rz%kq@Is zi+RFK2QEr5?~*1(VnhJB1`8dX$E=Y<2$zq!+f_GPpp3y|4JB*c+46@QF&ds(o!d^H zP-w;t=^@$4@O3m$1<7O@0*M1_f#RxUJfbalj&BKiK&5d1soq}C*TWB;FIVHx4#Oni!ey?#E{_eTKB7mPKTNy-5#EN-+7K{V1eXw#4Bbk z@vt-EZWz7ll}N0;yr@LNT0hGuk*@b?@J$H~dTc2FtMn1L74F*$?#q{8sajV;h~oD` zu}ipQtF@TEX=#JYXxMDSX*S>3S$o7O`7tZKj3Kh`s~aeJAspC&kry#e=FTkpjEq1W zlEpmC+G4~bqp?_q1jIau86B@%OmLn}py43XH)mR`(5DrOWjm zBNv0^@~^j^03!I|C1`A}X6JkA@{Go0Xo`8JHF?k!`LdPh>uOnS^gJY^kJ9B-Mx<}f zDU->$;w$$vvDaziq)nN#3V<~?u{6PuOm6)l%9JB{c`#q!>^{J?dDVWH5Zb0eX?--r zIJGjJ?QIvMpfKUHW(p9dy4{Xpvhb?* zr5tSmc~-Z4kbcw0S2h}?F1uH}CM*>`>>Jx@FI_l}|agvO^$ zC!KjkBEp*?w;|X}EjnQoIK~=LC-L@l7C`7sE%0CucVXdT@&4E3_n$5xS?eZ%pca#z zft~HnaOd%!_6`VJSypd!}th!d|%&Vs?+yB)=a;@+g?2v>&} z4xej>#io_%#UqcG_6_XGO;l}Siv=wwwva~WRTq6u;~%%q(AYC4lnQ=;WH=tklbyXoT>T-85> zebiU{3p#4CGwJi4DPbc<@hw+KS8gBmEnORl)30F>+kR z5EDgpXx$W+4>Q<9qhquMvUWWUQ@-DU8mf5p;n9Ul!o!x`!Ytw0&TGkfx62C!p+W)) zHxa&xJo&USYKLS+L}_xytsYDakV6pq3@1E0#NhR#z@!(#vw=~8=P27Bw4LT1JO$$t zU}aeJ8lmeT3#1{Zwg(|vy)6CLCSwT8;D=D1N30rb;g61%iMkMc1pCP7)>Jtup1A>C zT$U(AtPBQs!20{uN-vf~<@;M3EP@Tig5#!zb=`1?ceSg#f}B6)_bIs3+?a#%fySjX znkq6H(Uz8~;4e$Zb0B=DjwRq#^XJDIt@ck_T;^YmQcoT`$_Iz$Nz$!z(WT8ydLlhd z8HuHOK_dm(L>+`3?J++d-eRiz_7nllK|$Jgm$9XJrBAYhEri}GoSea?g+F&`I>~7z zbhz>sIytWSb&&<<{H73CB46>mp*OK0kWaOK7nw(%LZ_k~6AGvU3k=;9!5#9MLQ-KK z@ihy!9TYQ_AE`w*YUJO!l|iA{!WFByh~Y%bVqz6;RI}btyME)lEGo&%GU>R>{gtJo z8?k27SsZd3UleUGQ*VCwJ5JAhN3`C6n>MYl)_hxIc4P!9eXTzp$Y+FS$!hW`WBTGP zUYdWDJ|-pNx3Wsy8>!x7r>tGk=z&>WVIdZlWWS#fj@YR+a%x>Q`#H%c$>$-Fp7KS3fxYx!OaRoZ!vd zks0oVljz(suCmNl=y)3p!%pWQq0nu*`8hYvk=H?y?R#A8^L?TAxUm*>{oc#- z!R9=wPMVMt?!ypdjr3qJ`WAS^vl2jdn}{ky05s7lJqdLqUzlLWC%)s>dKilTVP;4t zfB>_G?((YOJ5+M+5Sb?^lae&?(P&_tFlL1HStuQ1+ofiT)A(Ojjt~i8eGq&3c0THF z9uTmf3urQUCZ*C1#b*EZVkHI~0&ekZ2Wil&{_+A4jVPj~fnb z5~!;AH57laH-Et0`A@y&$az7Cqg2j!`+t|ai5mc7HysY+cV0EC_Y$=w|H1f;Rq_ zSNa=J&|jXa<?>9k?_ag$hlgUQ7M@f2fdV%`i+Js{%VzVbAyDuE?8tZ6V*C6*YFMxp z!#Hjr8;pPxKu@mn4P>*jsif<8-Bplzfebal{&iU#OmA1>f4NtB%D56|qC^T}cN?;8 zzf|mxd{?K%SO;AweF3QsL_r<^yw$e%WtmL<;^z|v=ritr7W6OC@081bl_y0Zy$p>R zZ1qF6o78gBqF@-!EVrs&hIayg`vH=^%D3E8l#-ivzc(JFx}S891g3C1*p~IPCusn@ zdw$MtP-{kep8@1L*5Yy5X1jsaKEWdJ6)05oy!d)m_m+p@+_rwoI3l=@1q@GzLO9~G zb9W#`uZBs#!#R(^Ve_Py;SfvPc{vba?5Ilp+l?u<294SH(0A~>sAm(^L*OY?yb|2pvz@5F*N*ycMKm+fzbQ~yj3PP`3 z8=&w#kh7cNe2*3=05E(rflV6*KM@d|``&=7&H+~FFqC_T`mzJhYE3>rg}`K3%*6^% zJTi>cBoQ-C92houoj?HTH8V84l3;L!sS|(o04!yB^LFF8q4I#=hLS+4)U5#!io^kk zSAYF&@YRmM4(<>5zd9#A%c&*>jF;SI%i zqBL5Ie+H*6>2xhQyI$lxBX z$v9h_$YTr&G+74GRZ?r29xXK=^w56)jejTUhjK~Hp68Sxg4IpBFJ${a%pfnfN1$=5 zXnbd{J6ZlTgQWha83enoMQH9XW{_oI1{qYIUjt(MeGt-`6S@CPws2%JOk4gxB+;KS z@@rTEhwh+?T~+$~`z;XeI4Dm;GzKpu}wSCNb5zH~MBonql7|Wczg&)b(R&|jEMOS8!aWcijORDMnFrmKpCvc=DXtg56kx?OXS;MFT z!^q8v`(gcM-LI_KQ8^xqZT!qsL=tfZXED{X4+FZLp#;*MW36$d3S=7+Eke3xw8w!W zG?M&ROAi~7TREPlK1nsX@;vFLYS5&P+Rjb-Wnr6BjjP=5X+TIRzi{8#B-%Fv>IlR6v)IKrFFlj9OF;2mZdv3s?GzT(=F)ismHo2kkm5o?_d<3$&T_8A^;^1sl-bqHERBPk24j_kfPDdJs zB(on%ZL-`BaM{DfTZ$lfZ{gxU18%+SLwD6g6-^u8K?nxDJ*jbB0=w@gxC|Skn%`E5 z(Ihaw*SDGjh&oh&`$gfbkhouj?a5--yGtX(F@a>pC*8AZrh>dA{U`nh zU_Mmlb_68XiQz{(ECk{~bU9)OJ`7;I{gcc~@84E!bpb|V9zeP~*68d-8k=MT*5vWl zXoDg$h~Caq%obxS=^qe;X+k zjRrit@J|psU3ZXRnQ62I2awdT@9=In6%= z^C~cTKS1A8nqi0RV8Y9dVD)x*%X$di_>l!Jmg^}Gs__Y>c*~kq7HPHeYq)yw=%oF# zfQq3ACqPs5-qn8z!R@_%?Bb)pin7PMsSS0L*=XXEv#Meba)!2q;_a?5mi9b-$TD$% z{@v^6s~Vh;?U9@y9XwyLA*`A92+__|)+nE@lp`pi-Kg)$8T#or5g;a>D%T7KjD?q( z#PqUQJaZOR^L1pO2PhV0t#y|Wd4Nr*j!9nP1WYtr&}dkvE)xGM3=@1)nGg-HyYo8} zQPySFrG-{wv1>v2)WwCoLo?1cu^0UT6kM2c6|vf&({MtfR0?tlJ3*( z8G(#G>Pgu)=6pQFfp24ftM}?DwyWPK4F8;0*~8nd&4OpRKuUm#MvR}#d5B%on*SJ; z8)%lqyj(B#wu4}(;4y6UL}4OW@GPq{7Uk_9j2fzjJ<{5A7M`-<#mw=V>42d^1ZsOY7-iSsQFOW$*;%5AlR4yN<4x ze#`@u5EPTMk49K}viPC;-VSgC&BGFR-X-;b^sa_-7d#IO1e|{j7c_!M)f^1{)VKH@ zoP=G9sWpQ-GWX2TR@p>d{?>h_VtkD<@DyNV|&ReJ}@v0KZhzNUy2DUUhbj z57g2z4JJj^vdZp(@jNi1=%W*>r*Q|egEM(vwm)bmW{ikMbU^zQSFSK7i4v*Xx^hOD z$Wv~#I)YMVs3pLwy!g)&{e^V;PK2Z*&&KQ0Mai@x8*-*~|08wocfQsJ+k?A^k$fVt67({y>qGYbV8=bB8q_&5bS^Bv2nx#3z zn*?1UB*eCaIRn2y-`R7R2-4%TEs0pu#mc?&^lKSo z7e?yOI)^j?q5H=e`O%&V9IrQzROmBOP|y2D3|%WEQ!iRWWxq=A{U>En22A0jvxrl~ zHXlPoJpAbNKFYRQ?M>zPn{=-I?6bS$-QL3fy({rPK{t--BJZu?ijE|WR$rA^!IhyB zUYTG$;rVq>lvTm%&WV``Y*wSHTg0WQ=oqk~6eS@A5}Yy4t-eT#fssBPtoFp#$y`b1 zHA2qxW?DWb{&J~DoS}L>5XIxn(c`UPOcN0V-}AQto_c7!qv<0jrrR2x`=ganht<5Z zzg%zUz$W{am0!3zAv53;h34}YVX1BNm(WCgjx1@$EZ&zh-JzX&NQ!4P(NOy0xHFNH zlfe%aZ)4F}*#7ays3!;z#d*f1@Zzj34!C=n#i8eL2+v2WT25XW32*9IyY_pYn-PRu z>s*-cr~{Q1J{mdJ)POCy&HEWw$>k8pc^UBET^S1tUk1XgplWg^1iZ-lmY&S$O2b2n zTLto?83QF76zTO6(y4Me92phlZ%MKP%RVDE=cgOF>^5xuxF(QK$GrymCEOqMF%fJl z$mMS^;^K_8Fr9)cp#}rA2NUcq3sZoG0Fih0MpIH#80<}YoqCs3W9}OKO&9Aw``qq$ zO}785IuwmCtHRe_n*yA?+~R>|GSx6x420 zbbsZ4uYKSZyy8Q~Be|{NgP}OP651JIV0O7=oW9x{fH}+zRcE~^-M8e4AZG86uzjCG zhQ(J&qwjJ?j+R)YC+zcq&YgfDp*2ucJaYF`vK(VH$Ncd#1-i9%tf`HdI?-N`0-Q(O z8D)g8E1BhM<3c9DeokTORc{`|{ZZI|N z>`62Y^06Yna+uF1g@?E}^`vpVuU*tRB|2%eHTyA&{~)gemY@4V4Fk=S8Gafln65~a zARIi({GyPyB~d+^O;}5q^&K{Wse|y@T;Zb2OjseSxxjfc6hJLOWV~umOhC!r13SP3 zj!q>na?{PB++neVRbG~idx7+=^V8g~tN*MT)#cMElO^Z;xT1U3B}ip65mgzZIY>(a zn1+d7e(3)?E}|FhOHF{a@qSn@7l+Xy$Ls!AP5~9ZJVJ!=!`!527AUQ|6&E}Nk`t=$ z)FCO7++PaUL0U7-r#ht14vN-@#Z4T-n-4|Owq6YAe<2*TMdym)^?QH66@F@JhE-VE zl~U*W@uA*YSH!}0lkg!U)kj}6es0MCPBAQGN{ES%w*%NV-c*FX+*tyVjzMWF0V08N zsA=j2DtTY&4CbFdgh|(kXN4Q`)&1TILvM0CFl>Dljc<$5yTbwkb*45 zeI>|qe}Gs-8?;SkI(pd}r^AD^3SheEoFp5Aa2{J9p?eqv8AZXH^)5YKQ={EkT6F>W zIsRcE3{cs{oQ7!@dJG5jCgfU!F%EO1je#N=f#a$mmF5egpU7){e z8nc0}{XHPMr~>$sHag|D`Oa+npo-`*igr4Si_*32>FL?i`Y-Nl)b(B;)vB~NTP+$RC#m)Qhm1;iw$;# z!x^yH-}h%!hDx>QeJJ~=;&10QYgt>Q6Xjcn$<@BQwr$TiVqw+Q^s`BWZ7bTEFhyqq zapmnXmaC@a#d#{Bg62^sODC=6=?>ByQ?jZP5(1&(;9dwNV>7Xcs)|3yIujkLIGiGy zg#i(PVGkaL6eolAQ-uaj8tPH&2<;K7FQLBcG?Sz))Fj9fI`+e_eieCXrm;mkqrF_o zm2kCic^N@wf0Y%Q#fxkP+)ULBSYw;0=A6a_2#lG_zIF*!V~qPlZOPK2In(8wvvHeI zMv3_l`v5sE#YJa*qEHDq1Y*TG>)YkmE2`hZ&@fCst`*$edSAx{3Fe#!$sCm;_RQc* zn75jA&GO@5GYLHpOwCSRF7o93;$nyC;3e%-J~9(lXY$4>qYR)jr%-bQtra)79Ozf< zh>XQ;$Rz0IzEE=nyRH%%_ZG30;%SF&=88#NF&D>-an@EwL8FH2-I!>F;&1FH-5ida zHH2A)cqC(ZeNxBk+^V@pUEYtHAeUnr(VEVVa+GkKvv)kVw}H9UJ7q@3861nsz&4fSjH$RPVKeBae_$&rn_pp z3X3CvLW7Y1y%1)qDxnQ!}XeWJ02zCt`1bu7?^o!{BCr6pTY=%TvF~<+-~h z4O?7fWDWWY0FabN5%nW%*MM)3vtcxO5!`)W-xdsXvTpZ186y&sWy8j=>5Ldftv|oDQNio>c=y0bJ)TN2e6~OQD_921*Joz zgVP0*ml*a}KWXWj=F1&1--VK8M(nw99Y$kJfvDY<5F&vEd66$w4tYJ@nGvI=w0EWk zxiuVrz`@?vq(hduKgLx`sH@JUKD3wP#5{T4^=+)ac#GM}TEiqK=1rEt3kQ!J9-0Re ze!3A}6AGmqGcvBxmH!iY2g3nCqZ@kVfBc(7rZ1TBg_kRnLPxv?AzXL_fYw8@>{&@U9VSh7#hL1ZO)13aZ z`TUjgVF|*?RkBFBIo~A-hvL$5HPKUn#8{s)EFBSW3Rc2{x=C*`SjrPFTENnfWP`L zTIN50_pX74d}%#T`*%+?D*n*j-WF_&a%$UIWMd_R{vByH|>9 z+A7h%ee`p$bm{T0?=@UQ)4!k!|QGUOmyvH3(z~d;qIgm2DbN%ec0f|5U zO015jOBI-BmC5hQMW65<3093r(f$C5BDDL(U-oLD2~W_tCE&$;v*ilS1Ole(Bya%F z>f@BWPp{I8klMt83=BROShc!6>ESe8IRLhci4Yu?j^+4_gY0i=&da`3gs)o{f&81D z11O{bT|GH&O>f5WKe>Y=_;!$7bO>EqU|59R)?-v++lxcB6&ua`q0>B1d*B%mpF7{Rc#Hb) z19GG~69#2z6*ldcXxm0b=Tf6RMN_^XR7tQsu(YC)D!ufzTf+v$}l(a;gH zb&^Pdrs4u3U8^pX(3)2_D_Z9Z@3YtgAhG#0ldT;%cBjTLR;{C`qyzUC_Joov06FKt zaVi_{(>xrjx+#;Y0Zz|y;m*!XVHijed-V7kkO0*Jzph$~>P~trLBfv>5ZuQeE@8UX z0lz`bGZoEYhaIIt)otP^`pa^V`qa5WMl+}hrH#&d>Y?EK00rMhBS;fF_$K0yma`A! zbupx`$xXA@L`~IMx|1c`S5&1iCqzWzJ8*lT2S%ldOi|E3EoEUIbE$A69w+s6Lhn15o%Qmjmu=oFt4WJI3%d+r8Y5^LYCl?2%gE5*1 zGZ~oLFjdWiu)4KAlQU>68U~rz6>Gi@cDNcC z6ZB`BW3>g?6`^h9brjsU`U~}{sdqs?#+5aSq6IQ-ibH z&qE`tZYFOuy${j=E68*ZJLy48UqeGHI1mP11Etmi8rbIuk?v+J^XBK3#$F;yPZO9n zIA-DpXQ6#Lq`K8{b2=rJjOehbbDWGqcytegVD%!v?S!qDFm%_q>oysq} zmx4}j4GqP_JZYH$F_5|`d1}u+RocC#&Y(e8rIyA_ZBZ6K42CcPSKu=dgVIc9)$js8 z17xl!k=B!?6X83$M9d*rA`u+(( z${X>(=S~Ojlf|ohXo8Sal8yntWeV}P#PM)>!CNH4AzcpIa9|%K1$>Tu~%9b-Vou zNMFNekhD9YN@_i9R>8a9;j0VIF&aoV;c7QEqJ0E(=X(-HUSPvk8D3?X)&z3|j4bA8 znDCvTBJ*658EN;}{Ol>^DqaQ*Q|Qj&?j37RK0&w_2|6sT zP}2*}`H?B*OokHccM63K?hkVceVvs14~1hnp`!91~8P($l7cT zWhe`*msVbA4VLfrU7$}r_C7F3a~dvboSRMEV%MHv6&bnmjN-2ybatv%ad~$nQwIoC zF--C})uMVZD9w6o@w_vf5Nw@N5{_VZ-a-xtr@T?CAZ|$^=W-ZStn{5A05SsErTNof z>0n9d*_<1nOWvY&=z2tRtIX$+#I*m9(Ul)il<5mAFestT3wZKzB4=q4z&W26I@VFortQc z<5V~gsRAF#!BZ1f@zP0d$97PAM83kj^O2kZYbbt|G^X9#sGVpF&=kJPe^_mKtD?mQ zofc-=8QC?)R&oUBZp|>R%cIV>_cJk^{a6wgf$Vv6%LeP}<&qX4>52w6p4jtod+@>R z1NV&up*Z0JTW{}|XQc-B6D=$S@hsFoXFnIms)s}n<@l%!I8Oy0suECO8h9HB8;!4#(l8NY@+R(bbE^(#AW#xoKUI75ljv`Z8&&HIj5cV9KXgY-d< zuR+`nRL{N@@R%gLHQHQGeCe6`DE6p_QIhZpf~n4i4&ol8x7mf(bjvt)Xbp33_!t|u zeb{b8Bk)uuauCNUJ?sP-q2tQf&c4yrM03c(y#Y7gwDhgE-C#B=;cBkxuh*buKA_Nh za6okF+6^U3TB6*40ULq_c8X`y7e9f=T*GmQ^=~_XNWx35k))2kln<6?yusQ3jq)v> zlrvXMbE!sd_*Nu|hx@)p#ti@Dl;VEnMYkEEPtyX0E^k?;C2tMLwxra~Zx_8&^L_fVL~}T+30gg(M@_)l_I|}+w#opp zp~MNF*8Dr>8*g;}g+$&k&d+_12~w4QF`H}P4U!kTP{=+e!9EdD^`Ln6e#j|UGWX?f z-NTyCw;df~X1BtY)3Ybyr-;3G`U{hOyQzU`N!1)=vsA@+`$6K7>`jc_v=&tc5<~(8 z^lS@CwJGKDmA-8p1b&8u(_(1&Mhv-3q_2i9(lFMrGdxj+!3s?>=^b%uHyN6zI#| zLf2r|)k7+qcmjFjBvHzftQIHCUk55a52Q@PsbixbegzbZUBFAo0lRZ;Wga|WbSVf)l_1~lF65B3`;W5}0`oGvN2!gd>ots% z>e}nF^44LseE79Cb;fA@k%Rh1Hpbp*)4?$8;MZymq;gQwwS=#UAIA&)6syirxKQKq ztulT(x;A6>-2w>_cwhS+KgGZ9`LS{Z%~FjaX$%}EWJ;^0`_`GZcV#O%&SpbFkE3n` zQFYOC)`zCGZmzi&l!9r1M^bQP-)1?Gysv1GteYuxy?u-d8ZF53xE=)S(fzEqCYMjp zU1DNXIx#n7{C1)l$TH;#yV_BZctSEnUu-bHVQh?+1DxPyE=y7KI& zPrB!Rh(p&lbT0=!`ev7FL}Yl}hL=*LapWG8F4+ijmUr>7Df(R-aj_n@o5TiXon#q;m4)Ppi8u8HN-o&hL{fk8r^ z>{9ouNhzX-tNQf(KGvuxr41I-bAGV$yxi@-JR|pG3~GZelxuj%YN=|}0yNqY)^C>! z-|h(*%Ffn}Jo9SZDl-;&^E|MvN@(IWIt>9x(@I3XX}4uV=r@3*ZVkVqBj&Cq(g& zcK0FoM7fXfR~E=`OO0Nr#q4{z0`Z-^@>bgyC0Av>b}h^E0t*krMxg(cb*XwGqgVO3 zFD0ZCCTOj0MXUgB;*_v<>yFJGzErJL#v%3Xq5Gy6}-lNzEO&IjiU&m!3RH2 zb!w9k>NzB_%J}-xUp9txQW0_68*50FpH^>PQ?4J(HV8=*+%yo`H&2^HHCdK3P~l|; z8$U0@f?=oSKk9e9x%#50lSH5zvxfy5!wTG1bs;wl z3~I0Kby9YC&T*O3=(P$$HP$EjwiC(~1p;m?H<#Yk>>7f~;*TQxhmK#@QT23Ev>y;Rv?nJD{z z1~CzLdmD^U^A;yj84hHYx`E;+Bx^cX&NOq0-U(!P+rzZu0u$DMdog}yq6r~}6aMQ< zm(g1QZGnXcj1xY8dlm9|V@FLiAi@3ZL-0RPsL=ICtd*x({(k+5RKZ|Lvqd=AzkWF} zoF62B$%OOk9?pNi<$pbanLKevc7+w}gMYp8-yRGdwiT%@o;CUH1BUFyN2mD>>0`!nz;-FE-$ri~s-t literal 0 HcmV?d00001 diff --git a/best-practices/ml-platform/docs/images/ray-dataprocessing-workflow.png b/best-practices/ml-platform/docs/images/ray-dataprocessing-workflow.png new file mode 100644 index 0000000000000000000000000000000000000000..3d99daae1e15f42be12ee32a8d7b5b4c94ebc04b GIT binary patch literal 92010 zcmeFZXH-*7_dblGf_ej@0!kM|r1uU1r3na#^o}4UROy|lC}TJfc%e7x%GaY+uvI6m-o|?wURTOoSE6P_nvECvu8rY11(jG3-lLANJuEu z?kPPaAvtG2LPAPHeh#=}Fm3XQgycdgSW)qTnxZ1>0}nSlu(K@*$-M|@5}BSZ_VQC= zpquo0z8hD$-b>t(K9%!g==AM-WDGaXUs8YdjMYlDoyJT@NvS-mi0wk@*|-|E$d}4G zTw`aAuO@g`uyS+bPKTE_;hOOSX`V}cNuQ5IwuzV%eBarpS=}^Ix>m|050$UKxn!xW zqUZGosXWOgh11+0PTltK$66aadBUn-J9%Ob@;UABakehGdlyF}8tTkn9PuYnXuI9D ziS#eHoq1=MMw|CD3CTq4;ck-sE6)1swy9@aE}=KhYbDRWKOd9!_>bh*=Py3WzEAp8 zVabEzJW1U3{LJ{z8VcKSat~@RZ@y$p=(ydKo0qq#1B$3TPak^NedQp<9rF3@_qufV z%5*c==da;=!dzzr9yLn7Ogn}!cM%i6T*Z1(uwJhu532nUmA-WUQTq{COPi7+IpR_8(Pi+fV#u@ZanHQcSZ7=T7BW#5 z$8MSye{G3#O2#)2^6Zt^TM;?+F$T+TXt^s__@n6T{dycDbf4R9KEG<2A}=SRS1V2$ z%xXV&u6IB%VwFX<|0%_Y!iB=iA4r}p3kZ-1pE16Eh2arAqKf0=i}pXxU2J)D`kZQe zReFDUMP&u4)qJa#93>9>7Jn%VmAZ)zA^8B62Z2IDI_VrpKBQ2P#98$1zZN*nT6W<> z^j(Jvxd=*+&dBn1ttg5V5`%-Pbm%!o}6O-z_1_6dg)wN+vN|W5v(-pK~hHN z$DRc}Ag_JS^p0_jl&nqX$?5ddyWeE%PnU*HedAvyiM>O$Ojgz=xKG~6O8!yt>Z=O{ ziqdg69#c*z-0!B?3OoJ&(rx<4mzP-c@pKt^*mWGynsGFm#Ckvs7 zWhIGMTzaC6h z=dLl^UAGgn6Uk%rys8@x?ws!0u?`bLRa`B;HO9F58f}$be*P=^YmhV7p5Pwu9(^t8 zRP6DqPjkzXHtFmGRCLiAA4@*Cbgs9J&hsx!x(C-&%QBb6NOU#LhkiE;ItOAwT!b(} z&O@kBu|%af_T@{?cji$Pa`MZXC$HyepLD_rUu0^#r~I?jHRfkseO}!sc9ARhR%mrS)#B)(Py0 z;cV9P_op=;Mcj?KTksM5Qs#nUq>?hbaxI&odyVzWd8rQm4r-3VH#HCA9u|(+<=UuA zK+h#?_NesC_5`72(V;ztP(nVrer8YJR{m;1c>Z|)u_$AIa_4)F;R|3+2bbzcG`2J{ zP;w|Q^aAuEx184tS0&7l?y~ZZ?vB14SOll}= z1Xlj=cH4ct>$0-DqAey!Z%2PoG|_2ov_4NrG)lbJme$Rntys5;8T>KTF7-w#k0_O6 ztz(P>*y+GAqQ(boK1x55=M?C`2~Kjb{KEX@{YpKS+*{p?AKPr5FKG1os>=236rtjx zf>k)rsNB1$JefTEV!G0#BEurQUS`hVm)LQEVYd+$@gbpP`|^=@li%WR499_!9qN(g z9Z$Y=f4x28kgd}){Inufs7tb|esVwJd`J7t;}o^r&!oEPTu9LguUM?P$k)DE-r?ep zYcW#J8&>?HD4SUjgUl4T*1-q8#(%N$Y%d!!Ycgy6jq7!X6enZ11G|{E^kk{2rRu%| zS6erzwRiVQMNMT5!n@Qvdsl`*gaM}cvY*s6vLtq$e(Ky*ozIfb>82K+nLRy%{~p77 z+8RPiQu-dFBjY(H3Pu|yUl|`EaG%wZ-4X?7@mn~O9=@%WnZcAUlCCX9#9Gh2S@Y<# zwJO~-YnxW|B$(}*HI$N>1x{act#BpG1WbF+G;BaO2IY3-_L;-wEGq9Cs}(O4OTY|Z zmj|n0+0OQR+_WXR?{%O}}Sksq1E(_UzNN@KgqMysr+{8}YInO1dA zsXxMm##_dO-Q?QLrO?=jS5k~3l|zAZF)xp!*67dzD`w8kl1`ib9P}Cw6U~eY%M+S* zym-aMh4+ls3|%kPy^*=x6*0vm(Py*mvt_X@x6g5O=ULG^qwm7AVy%)PtZibC0zX=1 z1zLmWZMIy$0*N44xyg#po^w;$84IGz)Jc4at z2D6jRtrV>ch1=+DjstTBbMXzrsqN73;kBBzX+AE879Rxa^&Rw$O!mD++_p)b4x^XVUPU=B?qGse4v@MU9cx&@41Wv6pyb0-_5tz;|Q(cBtT z^OSbXLGHKbAM|SXWfoF8G4jhEt-o5nH#i)Z-LD^`uiqsaOQZj3bY!J%Au0wXH|L%8 zzB6k{fv=_Z`%OtMxq}tV*Gw7hv+0h~{j%nso!|R5s=id6^?qRPGxH?aI#^rLOOYT& z^wf;zkG6+Y9HG1YXSQkR^!Tru-!$)Qk__R(jC9*>qTlnJH#|&D6qjqI_ujp;{d}p` z>D0KZ8NMN;=H;jdHf%L|TOz{^*3{9g51E?tF)s>Uov>Vo$W1K`8TfW>YJD^@bS*l|!Bh|8~Vzj`NS zsj3HcA5ygLzT7ahIy!YIyK}GrvOuk(kPzZzCjR+W)=us~Z{ex5XWOBmP+v>Fpi!d9 zQGoMpB2})u4?@V_b4fc^-ZVTrjDS*~jX_KuSsWk7Q~9u@sytVP1>ZlG6gT;>K2>WJ5GFnx4=QYu3`Rb)C3VsLa>!nAoU>%NvWgpT4`$)kXS!gPbJ6fh6B|_~6mp z7g;&3p)V32ZayQn>x$rWBuO5Grbbk}3cmjIZ9aX8vS%QPpwQ)){v=<=4DygwxW#nP z4auD-Tg3Pox#?CvzWwm~{of&2LMH9~$2FwHYU1hC?sMz0q-Am{&^*R6GlP? z{B;?4`@K8;@2Afhyd(X0`A1_UBzJTb)zpBiu8oJSt*fVln-|a7G7;d$`6u^`JxNGt zIexrPsXe^;9cX_XtY_q9q%^RvfaY8w5lDI_i|{A<-;zWUFq`kuBP zif%4IpI)G!_WD=lU%&iUMOlF#L;pn;|J3t8?gEMiU62*{(`cXz62ulFU`Ix(o==^Um>)nIM` zGLJUOpn4XmEtg?8kZL^PwcRhFJ=5e{7Gz=WwltWT*%`xRI&plC!8z`5Z{74@I#J>x z9sm77d16q&Mgs`mfk4MWu&v%nc8U_Zz(-88g)9OnPlMsKHQ8Q)Dp4lGnx|* z#I^WJuQ#EL=cHSIMJ5F&K&ml4UiLrH>N_{JBzjSI1ZrrZFEzzedK4U02<3$Nu`z{w zrG|RX_xocq(rQtU2zI@&ki!(O?zV8fL?*v)r?9K2{Q&g#N)VE{t!Hq6XiQzG=LrMG zoKURz=a`@VZHM)9VGKw*#P^P#iI4*xORaF05F@l_^8r?SP#cpTtW{-;3&1yx&>{9; z+Vz(1AJbsJ=q8=xqApabAECC<4+UiP=UA`9g@8Avb9Esu&7V8L%W*e4umTq8BQ5w@ z(}o0J%qsDt@VMo{2*a&u!ZY0iCOrr1^g)5YD2L>kBYc+8u#s&pC z8p-)rzIk0;AdB=FdZ*%e*At{BRD=mXQqNes26*C5Kj1)bvI?^qD=dmxY%;vZ)FAas zD_cc9tLajnBMY(e@tx6uT)S8#XY_TETgur7wXXqhqQeljcmy!uUh#XYC3UXDoOb#F z#b#c^9oQy#aGhjyTPPWP$`v8zjhk(m0P8apS9ght%al|7LK2R&&yyRRz+K@{e>!@; z>N8x1yIteA=NGw?Li)uZ_ngYAd^Qi}qaaCIgaL2|VK`eAzK=U0-~>P{#Nh>GMHjPf zvt_OIV)-XZ4aZdEJ9pz&};J|{)Q-XO6nZFkI4fC+Ns4lZlibF7sG zG4AYzBBKOajdoj9?Q|UaA;4u zgEM4Sdr-5B+0!gwF^S1kd$ZNF&J|l&RK~lZQw0tB z)16oq-e~|@YW%YPMEIxbCi%q;Jft%f-2 z5QrK~b89tjT%M1L#0#A@x_{On1_ui<;kX^~^jo_&XWsKLTzM&YQHPs3&foNdOIA{^!nR2NaU5$Eb{o&~;tek3#gxt~hPf%Q9 za20@r@E;Ss!cVzPoo`(7(Z?6ncngBwM_Bf(r{3W-24pJrvI*BJ!vvXD<;e`(TT5|8 zEYx1bn8&ck%5uvcc4v2VrTC1> z-+kTqP{)$oda|n12T1fh-NE}yHJy|3UIk@m{LdE77OZdtPYu*WM&e=Yk@e+x47w?-Dmfk^KTVjWB@Pt$!;vkTBU-|P0 zhYP#oxxk>sgLio>OV(Dy7)rGj!znanaMhlvGQJiSvL6%KVy#%qVn>jIHOtphXL5(ozzi;IF; zxPHIcxqK^6qJ=AXAe{qE zLKWg?urgKWW@~(~gpnsI+s-7P*nv&@t$RA@98JBrWo}AwX4aPY=q)=#kxom2Hx_57>aCYQ1(U$1}u3%hAEXq0tz9TzH13$RnC~jDtjh zgwYW}tEQ;?KVm;XF}gxj7#! z`dt2aMHF}0fqqQ>g$OJ5&b_JixFXXfT3niVlK`#g0ek8f)$U6RLyA;t zmNHnU4nkgA4-k`5VhJa+HBe!Bs4a&!Ir64S^tvQ{}c{F-M+B~{RO(gWt8LOX0~=uCayVqIruUmVAa&8xY&2h7^U5`QR{^EL`q~5rZLEVi5WBA z$OpGRrd-nug~o=gWn+_VKo5B>_hJErcekKB+LOR~mj%vLrqO0PQ5Z}v&_j+Es*pc; zi#v0+O0>ETpI19I?tW}+aI=w5f6aIf0QIq0PRvox{?UMMi%9S{Qk*tnqsnqq>g1Kv zlt(Y$UkTs4RbVOExY6Qn_Y)B(Z$9}5*8&J#YxU=#2KODs2)i9>t%PjwHy!?{alYEG zY{z>aFLFoKqFf2;@`IW=C}_DuD24bvMAPCtt!{kBi6e!9^%P)8fVA)e>?Il*(s43y@OgqO&ooL$ zyc#ke!8tM)E~b@4JJ&1J#r#%F4*kVCLu}kk(Z=JyZj*6LZ3eq|8sx z)xY?IGc=T`_ZHMrN?H-81y30I_mI^HRir=+=r9!fHuLbY?Qpy*Gn6@aad_VI1wZ1X zzwBWwvm7_7D-E6*ARW#XZjczf;hCu^r2?!**h75x^~-;T>3nt8!yqL@tZkRCwmjc@&A3HBEh=Pc zz5lQ#l2~w$}_opNp3$~OF|{WFdAz^sZw zvBH0uReYxdTV2XT-RIkTZc!^uZ(^pRU|i6b2bw#y6Sf$0vw#Ge`y6 zw>tbr7Q-)3Tc#NHhgRKmfJPMeJ3F3(Ki3`#PZg3bUn=%0n3{Y1Gbj|NelTo3f_ec~ z(f8Q`f|}6r8V2)^wc8w3@XZ+r@ffNMXh!q!Kp@28z^pX|h?0>fe7>k)--i6rae?3i z4Ji_-<_X_*>f(0FK@cI`>qI-rqD#mh*w*4tvF2&Dgj#XbT0)&(uha3FBv#$#QeB@>v_0{mV67ji`^h> zW~WHht}ofTi4+xLRfXPs-|`$PdGaWnDG?U9*D!xg2>rDK(R5pRSN*jkmqT zVzZU8R*ddc)D~y`RvwVysa9 zaL^Z3bLG!D^&bTnz_Qg5y*)qc{GvhNSp~U^y@f zQ{Vq6hh5tw4OBG-e7_QZc^OvddmqAb71X7(>lAynpGJy6{ z@t5rx-}T?BJMGLF9+&w_;onL5U=YxDKr5j%<1an`ViAY8fS8(Q2Dg4E<^P=G->Q4} zKd1ONk@)|~DN4@qAoKd~QBY8T`^CcR!1lj)Q}Uy-3-^=1s{^YmS9(zg1$uF>dn>h) zv1)JO0P5%EwFVP%gtApn$}E~z*?#Bi+?%pWkB041k3z8;MO;zXd}BppO!d+MoHw|Ipux77#al>!#2C7LjWJ z9_eH5jy{{mfqQRcJ8D*#x|@cFj=cAzCdA0}w$ zpDgP?l=&qCFxQ4_!^PiXp(72@2e`2Squ&zSK)l=vsY z{?8?Ty2LlgEu)6D%H9zvOuDYG#L(|T$TP#H5B8w#6t54b$eOwN7l3L~=mg!7&zzY@ zg$llJ{))ptf&Ih7XMU}a5j9Re^P!ihpu_GSX@V>|U zmk9PtNHjMARyYxLs=&uSrRLAk{3SR#UIKc2I%bguEM+uCLVnvLInZNg!)W&Zc__lm z7>Mzlv-H5nyo~5CqWr4Z{5wDlCN&9kVKV{?s1g3$F#3;SFtG!vnQIt>Sm3eWpX3;_ z1*lDipA+!$3ODp?H@`IZ?i?T%g~NJHgjH_K&GqoVDB$Oay*yr$D1EAfXI zh1~asbzj)weAM;rM(LypFQt@I>R3-Fc7V6~USglYmvZDmf+Equk1S3~-Q^$#0^jo} z2s!3;%Z6u+&<05;F%G$%wda3L@BMVsOUpNZ%(AwwM@^39BZ&8=(tb@R*jG$s@JFNp zIbpB{XJ9G1W9=ZL3UfyMSsPB^TX2F8IzZHtQq!eowI*fSx!k!1S$q$KWZ*;uAquDY(cHhr3vWewhU zQA|C{W7+T|sBtsXv$0zFgzfl5F zOVd1Y)c)@*Dt!-_Tz7^_nDThe@(9(8FF8r%;94NjS@51p3Ma zxu{ao*BFcRbf+IdsM048qre{n5F4tAf zc3WVNn<&+r+$PFK1#SHenV>v~78}RLy^|Cy^!ml90(6fz+3tGcY|X-lop`x$+9+6{ z{~hUt6*X*WMOCl<%7XXs^PR4;R`C*{dVBYl!TZUXl=2H5hxI;&YX6Puz*&+XmdNu0 z5}B28C1Wo4P_m9n8@VI<-)kRp!FL1pCmOJ|;r6)*qP><-v@Y^OXh=&_F5eh7V{gVs zA>de=0{mqNR^ST*@!o}k584j&@mkt+$Nnaf5U^XZ*|*Wl#Y)HRak4Yx7N&S@mai^P z=;I0BMVY8O`HYfBo*&V-_|5y$((qEZdJoD4rN)Gx$Z%$ChMP_7d_2ms*AC(|GG@DJ z1U}T=g#9-VE5L!&pY$dzAieF|;TGg)93xEp+Gx{=1_$#C?ZIQglfCve#@BtfPeY+% zS+NE0#>}(r55H|FJ`}aW#b;JodZJ!?4QC)dGoeQ>X7bg)jb18lZ($b8v6uFMKLZmP z^C{M*k`vl-c1zRB*aJh_9QU^v40nPu_+cx4dwV@LGH;Hw5iYjvT*ln)H8q!72K43ydk&3M zjSOO7jD_j@y%AEcjCqZ?+RpIgzW8;U;NOCp-~C5Sd0!O@O-{wm$WZhkB`LV?#=WsN zD3#prUq_C(ZSR;58|Efb6E~osB z1`3?{jg0T!Fe*5@)%-X7V~7Tb-Wh&$(lSO;+}h}!6LT`dLdxg!4M`Zd_EFH?fE)O3 zL21=@JY+npua{F|W}NO#iWsIU-M-xNfd6Fs%g(CVoT$Iu)LrO93EVBb88!uGx+L*W zzvqH*Tl`I)#_Rk)m{Zum9|mDNtFTtTzE)s#0ZvxBL1^`o;)E^_0`Tq4Dgx-Vrn@_01>!0rGat|==U4UGl!HHEB+yk-c< z6Sdix-|5*|!8dWYsb{W<-VEHuA=&$3P34js(>^`j)&X@bs3G~_!>8||f*Z4(c66hf z#Lq~G^1|UBW6|Z}ojwHi&IG~|Cz;)GAc5pVp_DsEXnJ#mm1cirkekx8!S-m zFAeXZoW%BjXq`dwr{Oi57P;VIOCij|Yt8ntu3#PhSIKV!Tb0VU2JM$}G){zD{ETK3 zKXr!qpt7_n$d0$1_)@yBEpT zIL0`gE$R9~g_^d1z@Y5DWn3wN&}U85dGv7gnoz!~bO62$TRv^dar)VJ>Gd8%4>zcB zf9O*as`5S`7Yc)q0ogSHbS;)yr*D}Bh}Ay*jDNRTw;fa(d2l>~M1G~i-NoE8nkfLU8@ATslI{WNLPOcgG2nq?a0C26KYIj z{~-mooatQNT9Q{mZB(8a(e|znTBw&dy@}!GSMA&>Ro--fHRRGH3pGD6#d$ ziu|{sPMu}~_72L*$t*T z{7@F8>nUjFA^xbkVRLhD1R3$fYM@yeJYW6tU~aqG?2!oj_|4)V=eM77Z1(6=0bu6= z;FujJQ`3lR*rv-puS=IBKP|ydl&H#oV>~=-qh8!^iIiy(&@QU=1`+Q_U0aD~sg{t# zkvn8vm`jP+9{GW!X|;Hbv@#^Ta>{aASgZpgGTI(bDl0Y5a9^V3)Pd|gvr@O1f9J=Yamb*W{NchP+db30tL*O%SLQzH;uy8illeO> zMVxfUmWg!j?R(VoYjx{S84W1;d?aj_K6;JSy!E^-;xsXXInLXGep1bma2evH_}n(P z6~ZH+ZJEs9gDf^rQ>=&r$w^0Fzhwk|20r!QhIyt1EZV$I2l#gbZ~?0Bdbcxceh$Nx z?a^5N_()RTA;#i*hEUeW`EQJSjEKQFQC4l6nl;5ubcU{;=F!2hF*ZTNl8gJciKzgr zjAe7|nKB7AaRs7wDizmk3^iFGO9)&s)W99)@7Q_(nVYVX#FZv)-eIEey2ht^3GP8*PRD zAVG02Q{K<0qKMo;3lZ?Jk;`yfl+K+hAdRcus_kL_zNWV)O;m_M?}EK>O>&paf-G^cN-%P3K5 zpR&-2r&dzVRWs!_%(DO7I<=#1DPygKegd}?<1&vga$_quX37R^muD%1|EdGtHq+FN zk<6^GO}1EnFI!@&;2aOfP6}xw#T^|6W#x=ijD@Wy>lOwXEMJ(TD@SRHm6XTu^5u4u zHw;G;*ay6izP!W^Hkj}Z)C9IRn6`>%I#E`rpdBms4l-%7*gaBFs`#V~a@nhA` zh@a2TxL(|jD3+JS_c%y{+)wJg2CrVw$PyYXWK81J{*)0VHwtumC1WNdFKL=VHDu960)mzRPK-J#W>tQ znUcxNz36dP`HY0n{FKC!;~VXDc~KW(J8USPT;>)g@;Sg2v?OE&*)_WfF_EDX#VF1n zYm=GlF}{eD15RDnErH*` zGOOP$Z&$d$_ONm5WI4l%fw}nu#y-^?MiFEiUF&j^>zQyEx9S-w-7WV|E1lCUOC6uL zqe$i`8e+QweVEh7n1}b{Kx45WVLFslNxpbQWTvAEBFq)bB=zp_aqVj%9{i<)J({;3 z0;wsZ&K5{%o{I)-j|q=UzxrchLL)!$Z8*KKgw(qQ9L`wUUB%a9pM@{l?QN#XOQOQX zJS^V|D>Qu{L`H(!=4U&MnA+6OI0@NKRRmHP3wIc`TKcP3I&EYcAGt2zjbbFr=12n; z3()6i9*uo7yP*ejDrPt0%5DC6facG{%Q7RtX)yJ4s_Ve)sUj0Q$FupNjxqB9y}yI$ zT1sFs+;0S^$lAUqA|;OJ!qQc$cF2>RRIcaY9!dglk}Frn>ohfqhqz{18fqtWVd#Qq z`K9QOX<Ep7(>qxM%6<#BHw_D%fc%g=U$>h*|dmkiE0viXiih#OJHZV zP<5cDLQw_T5T}XO;FrA#DN!Ttm}m{+=l*sZ(M8s!p>?vnD;~fB#cNPD?Zsf?N-bEK z^(s1JNtmJy3RPpY~VZFuTs=0TFn3_M1Q$5=u?|>!iN;@xaFPkY-%(b+hnT*S&A7>Wly25=`A%oIY{}OY z7Bdl->6^uVKJ!Ojvzoe*t-RYww5$nfbY&H|Wa(lsR3k{orTp@Zk6j)tsmmChn50_* zw+B|o7(JA3>&KA4Rl$%Qt}e|o>bP8vp6R82Kj`Kxm$vR14n_%qMS{hnWQuU+-k_SbU~fF$`+Yzjo-n z5wX4JeC>|@eGXq+*a&Z)cvOY-b7(#&0=wJ;Q_(TrK}T7*Qa|SlTpFG}VF1z~JY|Qp zHy#H8XJQRL|6K-xi3fl@Bb=S;Dlko6nL*vX-a=|vIDJM@d6>x9&hXxTt!gM$sK=-8 zREuOCWuK9cN}QEmemFq4Au=$JPM=-ZUW_Z0%eQ_} zEU#=5q|LOqZg%G(%rHp*hjl*VgWVR~ZC0wUxw~1ne2)<1IIIvYX^rq^ zGm2nkD|G6_gxus|5E7ewFO@De_p~z-I?;gZs@rx$A8MsuD7FM;W-`iDCP(sfM_W5q z#-@w#%bkpDkyho6UyI%tJn8Nh71eH_K}2zO^=q(z0?mfITU$b*%i5kQI|dW=+QT9o z+-W@p-&0}8tyxvKG=aqeLeB%0@=95*-oK0a|IPxd+n?hZW+Yr`NVPeR*GMasCu|*g zE#fUru!yYKs0eJCbzw=e30Rq(Ie_JR=PBh^SAy?bD_O?cgb0@=2Vm-z{xhx8>LW)Q>~b)%u|^9V@zt%r7$Y(8?Ku%1vz$uRREK9vpoQ4JPQYY1Cs|_CoVk?=SSq3j~iD z$LG*Aq8O)3jjN(Gr{R7A=IZgofdSnBb8lSVv_wvz(9HPiwzQj3nf9ch;Yz}idph~0 zJWL^soSHen$D$D*F@cRgk`@WS%)>xUxwaj4f-MSo~qFB_SjeKI3S@QB`?d zjV93O7b`u$x8Tc+cN!vsWBT#O!SVqG6JK0;qz%rPVOZpaB_iF@pYz`>&^`_ds!c&{ z(D1Nt1s~{8&drCkCrf(Yk(MOJ6naN-!!Lpw3?j#`J%>JTWBHrhqO>NEr~vP9dE1o> zOB==?p=82nMz7e4UOraQIvRbEC4Vol;*ZxoiHvufeUsXt=cWg>(vv{jt4YYHw@dKV z)}vH)F=g3e$am}nc0oQZc}XH=#8DaJ(FY4fs_rp9G)5!*rAw1bc>*$mc=3i4HoYfM z+>b=d;rv+vIJr^b^dd?w%0fL^Yn+#LHOoC#KY1eg%X&xoamLw|IKO+4-QeN;V4X>B!#wB1)LAa>7pVqh zFIRu$o#kOEZ<&J}U8x;j>h2tLTk6ZrHiA(*HL}qYdC?Ypq}22QQlt1QX!t$R&|lV0?eGC@YLhb2OOQF(YyqN{ZyCZb`Tx!74a&-ZXF zo_$2RVO;YykoEJ~7#DfJ;u-y9D~nMfvF){0uA?;1-^6btJ|K}b`q5-YOGd;9o01u= znh9NQ(a{pRYJZHM7Cm`kdH;4^WZ?$4L9>DAeWD+wWa{44hTZom*g!~fI^4}9z4Txt zxkt}JhuQ=G6-b*Rg8VyVT(;Jmd>E5|EWMx{0{2ZG>=<1qe4GxNe%~-m7-8o1l@%5V zUZ_+9_gu(ZDO|ga@b{(E2gL>2{9!u6DAy{pnY7@U+cmutee^D9oDVQT>+~5Aw^1iCquhDZ0YH18Gk^4h&Dbs}_P$a`P|r+Q{=?CJ1g z%-ykr(CedN2I!EU@qP~n&-rJH$gu|cTPiyrBU)u+L3KwKlIYr+=F;0n6W}TkmPpA2 z8@o1rt=J3}xSF0A*jjBHbW2+iU&_P)e;-x8&XA;e-xoA?(x!gDgZB7o4|qEZ(PIyh ztKfsDhU^W;`>WT1lqI=0e0qEnNI8};@10@AYI14(0J@J0a zC!gfCw33c&1JVkwn>4M=HmLdOdABFnP~S>eC}vbJ^LMk!e7lNh^clW#v(>5CDCKR8 zyxWmFp(ue6S@>qbLqRVld0;Jl)w82Vqruu1^&X7pOA2G5rawl#6o7x$Hz*DgG@N9F z5~}~$7&m%P-L?N!y9?bD>)&8vZ3Lrt0+SvBJdcEnX{kS3K-_#QxB@!lVBf>FJhH3v3ri zdpSry4(Di7r-W;LZt>rsmx0gl7_h}fvbttaN7L{Kot^3$C~LLm5asWq;bB0>2^aRB z;fY32LH@?cmw5q_YX2Ft;Nw$15*HeKh4R{)y+OGLhkF+EK}sUqZe}OyjhIa-vDNk+ zZ=mxeX_bbFc6U&a`>$thQhAT z87XPcan=5)9{-qr-KQBTMUP<2sB6WH)iWdC*$Xd&_7~j!FH}NtG6Bg!C*R&CQ>^5{n5ae-LnVNyo4)E>TaXb(G z*CZ$#mAnkx!I}jsz3!XSDPo~hUio?10$YtoGxL0+=P_VD#HD-fOe{=!6=0Z~>?VCe zI|BuU47b`v{f4vQ%G-x#P|s16U0!~^$?P&}h>xl9)6<>L#|%aAyv0EG!_l`Rn#5#i z7JPupBNn=R@e2D%2-AKEUEyBjgRM7LR52@zat4VQH>tl#bHo6<;a)j=!d)wCRvcQK zspAk4qSIl%Nrd1{71{w$$OORp|$;7%+sOVw6rsVV6jBUbHU&c&AMRE!UPpJ8My@Fwxg6k zK9C(X_NJ5YxEH%KrCp+r#a)4F;f~wtW#wkFvU;#zfQ-yRQzX|cKO(U60?z4$ElbGM zH^x%YJ!dd|@r#3rgAli>kj#(rrMO^uKcVPg_YkM6e2Kue2FjYdqH;x}L;WJ##eeTC z{1U^x@&PncY)E4%<~hUTu0n7@BFL$;R8^7G(ZhpHVD-iu#3F(t4@*{8SEtjidZwI% zQNVFybcMPExQIvaDcsnL)^*kbvBU!UPK%=tqt9N}+; zo(ph>s8PPmqQAxbx+`?YGQ_$hi~jvIenDVB(;frv-WY&_sOPK2&+TAg3@9p%K-Zd` zW}6nrP^beVaX9!KJhav+=qbxor+1g=(|7tUJ(KXXVv@%o3{-Cs1rZfM&95O{JC$gY)0O7BGc)+I{_1zx$ z?d95PFYa0@DeRf@6WUXQFCnnhke#m1Kt!_*v{$}yFwX1CKxm{3-Qy|-=JXW!>A>85 z>F2M@n1y61%(w7!ALX?*5Ch{9{+(N8E)&?+)N?!)$bC7c8svvBZfeOs!KK{N*f&W@ z1R1VDkq`_jWbhT^gS`?xt@8%zOuzna^WWQB(y9OnliKLeX;LqLL%b6#?Sq?^VVBtA zXE{pbrOsEP>F)5=639zRO<_qF=pIa>Q}&1>(owAiE`}pE%LuBAw{T5p*e%>31uAJq zTTlN;G+fjWCBfbg9*Jc?5rgPawkz&>w1;lUu%u-uMt=66EPZ+47ZpV|WZR*9)O(Cy zlND_79lrtPs$kRF*WM_)-2a88?^w%PiF~twG;Aphrz3X7a6ho+j_7Vv z*b$HDfm%hf`}Pu;lNF)tv(kc0RhSW&Ao`?;!ho$lp!ZfVa@-TDL%c;JfYy6FoDbSC zHr}PS)2XO(-;3_tHZ4XzxUcN}8Q2YTB7#JeNy?iq<;&LDS}s_q-litLG?XfZSgBB| zV14v6)Iz<3fv{HLndM)@R?Sdu|+qHWrD zp4cy9Yc_huuPsC~h~~I4*QlUDni(~>gV*OHeq^`vBtx9C?LjjVu3zKe)~4C}EuGTH zZMW@)YIlckt0ubzA3m>iy&VS2`qN7E-o|=gg9vy5xcsrxp2w&1dN(qcae4g0_0)5C%N1Uu0>QuhZv9^{ z{!aHte&OS+-%#o0;`Rxp{O<+F41tjex|CwM5$dnto&6e{-%cW?$!t5<{QMC0%l*RK z+C|mJoxvk%;%lZUK1I{wFh%W*hTF#{Ff4ZDNKpPToma-^YrUHgld-6`fa)1i8_VRh z(jKI#91MOxH4R?30wutdG9!GKp0MTVJJKgA<+h`JHMKF*lDQL~aLb6T;pypCte(C5 z>*Pq@LU&l-qrP-mcHJ|yLQJDD)7@U(hr4bD1((N}^)KqE4U1kiu~!<*;yTE)q31^C zceJvTyy{5-HnSy^S+8KEM8jnP6j9h9Lkbvxe`Kw#KS>T!Azbcl5JwOXJ1+Hd+D~az zwUKJ~bA5bnM0?6mPge# zKS5VuFf^U+$I9JraP}{_T0D#d^W9=axwW}!$H$J5paVqSR9A5Gt#G#=D-X&T1IA*d zOP(I3yb}b9lhyg$ERGCZ{hp|7 z5VqAh$W3HW;|m@y)zU3rlj{NjAYc=9v5YtW%#E3CP&oPlM`3K%>AZ9g{#s*kqX^r~ zZrz0zp711ez^3o}{3ksScpk53DfL*p8^`NxRSo0~BXHr|-);KE&;hZ$*&qGNoXaTU zZ6sMgaB$$ka9;?JFt|kZ(a=^&@-o#VF^(m{abqLDHzl&9m0JStqTokg-*guH@PBMsi3cNPVlgKRPV$Mg%Xt#Mi4tXcXrj4SZ zT1boWTV|o!4qUPyKV~o?g7WqbwKn>NZ%9bGNweMb>+Pgy;I=bD-w8(IY*0vh0bb@D zJPvs@=ZD;#4>o-ct$)$~W1px2HB{|gTKdY#S(?^%A6(_Zp>82r4;iGz3l(}X_tJ>T z_aj|r-jGm94s2a&0927X6nOs<8c+1H1$>6kLt2XRb z>3ic2&2+^x6gsRm)3bU8woS-!CFWD7TeEj_Z75;MNvR;x%! z%))42UsST%awxrfF#l2jqhNPut@IkjL2Ij=VI(60KV-M6A~wQD)!&0j3H+**pA3nT zahc^@uNi%swZ(Ne$K$Yk>^by(!^+Kp-ROhEUe9RoG+XiO3nf^o;@4;CPYbqo;QJ*T6xeQK<7bNql42*|c6ihDvFBj5b2EJB9E z#-;qF3%j|J#v~o5i*K+wg3anxVRHVF z@$jqggUxrc|Bt=Dj;gX-`#@ntkWxYR5}*j9TL(F z($d}C=f-&Ue&6TW-*^5v=Z|yF9)q!lLl*Zv=T*O}=DdhMW9(G2m>EkIJ-!S>O<-Tj zO-i#B`5Ikl{>@9t-jOB~#g*k$HBd6jDYh}&8NEDJ$?~`s`5t|3QE!*uQ4E znsJxa5N?(w$2fRjyBJv&{-ThqtFuyXR=BpHGHDJzl2ZiwOy7}4ryf-xeZw|@4>-kn9DXNFzKUGZ=zuqa!F zX3sgr12g9p2C?czGM7w3#12`M!j^7RAAiXwr+t;`*y-@>SjH4%H;!cJls1&8Blue8 z+kDd^V^6 zcdGZo$EyL)vFVM1=XBbi`q^_SQxyP+a={Z2JMs(kpu}PWQh{sN`}(}6!?U%b=pqED z$=L$|#?`6*QX${jT3`*;)vla7PEMjOCW41K-@10+FVSSCuJ(vlPq-jFd=|2ycJwOH zHYhiN%GNTGs)L@Icx3L-=`~-N&D4q8hdo*kP(nlIbGg7MPY@@6fCt63E%jji)Iy7s z`b|DK!ISDXiKB5Z)D=HU=GF0d33=vH&<~@{V}kqHMoc8Dw^uf_pB6Z{mw&)sena0} zy_AvcC_>prD$$rV=V2?6^|@}RMt1>=sg+dp!C;Q*v4**%IfHs3R~6P z)tOlF$|ATt+5(Qpv)D*bs44?DJM@PvaNPV=t@^ird|m{aiis?>IrUXBAhcBRbHW%A zMmIzlj{eI*A+xVER6T4+^u)kr#C!&NgF+M+No5pkMlKd=QvX+MNcatw}|Ylr_el61l6V>bcXWfp@<< zvIK|uFYkx*w7fHNYamTITmutLgz=S#q7%8``3LYmTb*HH)v{YHTd+;k0gqU69b^<* zwO-WQo;~@{H{Rtgc0s?6$84zGkU7owW_rnwCev^;0Be;5huE6IbV--NJI@Mb2sxK> zMVl4dW;Y2L3*w^XTw7IP(`sodqKefKstq=MwEj`lV6}vaCNtz&jAt67L9mrkw8b<9 zeO=+B&CNc!E!+U~<6|YI>ldfi)jPY)s%1I5{}6NnG0SJPyLR<`O5g*lkqDhgrgttp ztAXpshxnGkJB#x3Geg#Dtvwe7eK; zfA3Gb`=6Rj0QazWgMp8I)%}52qodY9>zS+hXkly#U1o3)O=4ef6czi;>lK?Unkgr{ z-J`D<`D)VGkmQyR7=6-njSh;h5Y?)CK!3OQcmHE2^RpEU@2-iyaN$cD&Gk;T!uj|d zPpX>Ehy7~DZ;ZCs7n>xBQI~-%mhjk*R^*(=_gG9Mun#!=B&t<=7!NLDW?Acp* z^{)I|H2x=;187!607!TqmZL!}E(|7}ZR*KK#!h&BhXzfbbl(PA3Z4u_ngIwI@X^Kn?ZlF(9efftRn<{OpEg z2U;RqE&YYW$S;tkWI1$=cFKpSj=OkNbBWOK=z+H2SfE7^NC*IBR1@}6;{QOIk~_j% z-hJ#(tq&;Vlo;tm$a;?DEwrDd$}?<)TNUVTJE6hLN{=Nr1N!QW z-7Ktb_eu|kSWo-Kfre%(W#f)ZUDbc2^cph2OfhQ4RiM7H+*ButCl+3E5|C_sIDW*? z66N7ulSRy_rWW}rDqAtI{p7Z9*pp{}=!(VBLssMAKXof3V>*rg?6i(X0c3mb%Rdyb z?%2%#2%tK_z@#a5v!tuUB*s6}6LSq8SX~E4rOT6M$visTe@KtsSw$~~y9#Bi203(H z*vSr5uiCWVWJv?EUbbwkhpTs@a4kS%%Dmro2F_Mr{-`%k3k{waN9chdJ? zM*d&UNT6l4?qF|ueSMQ}D$wvho(=jjN?$bFV3iT4Ac&#!yT(oi7mrcw3-0f!RlbkG7lAU5g#WpNT$I03)p2t;| zfX&S6KO7}@7^^N{uL7Fg0L zpklO;+_N~S-+#~?Ou#Ay>9)-w{0BMb|15d;g^aG0pwWLuSU})^J^}u}Pxvn#ep%lC z*U%y3>kV#CA}8|72`Wwe>;09j|cCK(=6$KJs<}hMYTKVq{>Qin9TVa zy3T)x8%1tj+`Dzm%eT)oZ%`ZlepbQgIVlvtwKD}+0ir?CK?k39io=rv7{l933cw3c zfxq0Xw0sOx7BQgs--^o5{8PmzF!b#`fFl>OT9*K_^G^Y`!~C#>?SFzJunnLThYabO zYccV}fx#T~k?x#~Q2;V?b6|f1aU;sRMX_88!1^$N4uTxhIbbVB2>^DQYmfcSe~A1^ zfO8G#b|LU(*)2HW%h|xuRYle)oWHp@y}+PQW z-$3gH+|W%1rSU||yH+VGnG7VlgJ~7ufKhH{3EAIv823&W!0RofVlF;p2qde(9k8#T z&4DdeIRE{miadbb2%u$4z4As>2!y-%{vE4L0B>qk8S?M9>i#sqEWrAJ6vH(sWEH8o zYh$F`5@1HzU0(|uqv_wQ_%G9YMGresZSFFma)1NAtV|EMI`LgWf7{{PpTsLAY2COY zg{mdhoB-#1y($0?adO4L{BJC$x#LS@qu=NaEmUp075z@IbAd=eMM>@B|AvYI={w>X zpZGwNNPt*DX}keo#sC|z#i=^j-%b-#15fdeI;bp~)IkL0C4IjK$gX@jz;P4Wzqgje z1M5Qo^?G3D9s1}R*l$d-1I(BNgvK}_ru=U#&$#2u!7XqFzAjGNZ2LT)FF(BGE0*U0 z*>=W5ZSdav|Knj=Hp4s(sD;gL!2 z0Qp}ZzHxBPsLer!T0>#Yl8qkj89H+f$xru55is{En^HxBGh99pG@Ly~05v$tusr@7 zBDcuj;dfi9#rmplhU4yRGUv}fO?h|2sC3=GHKFs3bzFNhn32C|A$VMv9P4j7`VjFB z(ZQS19_SEY76jv?XKY73K}gTvy=z1uU^NjgS}0T%HT~P)!FfGI!-QeN-c%iO)PK}% z%fVm9;%{PvM*w*^E0tmYZJn6k>hAU_x#IeC^%3?&x{%F0L!e{$Uh+>)1Ana`P zO!k*0cw>Ce8^;@`pNj*q1S4SNdkA3Mf#$AunFI=2+Q~(#Lymtr?R;p{Zo?m#+|K0r zJP{{De;O11iYKJd=g@uFA~wuPwATldzv=Zo5Hv?UBlih4v}rNRaLamgEdNtg6^HTn zBE5Fn9}Isz&`TPHPNzW-m=XcP<|9bSajAOxTYKy9em!&?>t7EYgYM%ec0B*H8(uto z@R_-@=^i%%h}ScWh2-u}NiR|FOoP+dB?<(bwjzK``lA@GIjG|Y82yi3*sBKqqhE)# zSL`mT_;v!=4Fpi~H4yqsWXcY4ACd>~`UhM%ez%awnaWd-Cr^AvSP{WOmj^uf%_w{K zsoy^wN&d8kc@2~ZAhJN(&u7ktb*)6Q60~GCE8$npw&D%o`8efP zs(`ICkU%K0+{2?NHwDggeY5l@G~vuvAMPWRl)^PjI(S^VIsPF~)7?^SnRk9!PSV7m z5fFGY40yvniUiX}9(jrLY3DmvWs8qYBIHX8f>>Ur^LAEe&g24wxFN~gPXjKH<_rNW}3l4fQS)n^OAqvG*<53~DlA}_+|&bKS)3{H1u6pQr{ z0|Ns&RaKPaD1nzUd3EIf!3};kfZv_(8IuY_?)V>`@q1AN2p}~oX!XJZa&@Dgc8=oY zrS39tMK#mj2LUF-J}@vDMNa&dUbO;NwmTrZQ=jEbsyk8PUdv+U=~4jKT5*nU=`2ewMO2K7Ni8eRJ^*z`)5kDq z)pqtJ)%HzVTGk%pK4mfbEG!}-VO2C#Vt}?@C>UTh0n^Q}TOTUXy-_tZhpaMa85}I- z7)J*N1cW=zdPo7&8^)SeQ)J`mY*+i3SL?MmXks)KY4@HYt!ls1U7k)*B>#iD;ZXqO zl^jeh*ZvgaH%Py9?}D>g5~y4pWYcA6c4`i#dQ~=eWy&Nh_0`5py1CV@mjYwo!ny1> z#I=s)A}+>rqw?PD)G><4MX3N+pr0=-hDpbU<9J7zPgO{K6=jd#bsrSL>r|5CeNG{% zNf#VL_Ynn~GJcanDq30Md^|$RkIrRtshTo94XrqDqr1g_ktQWpML5*f7q zEN2`#h&CR>`1HBVrZXNaQCd8lQYOALjPK^nWO`OT0jEu@u$Y+t(cGx%nE382V|V!J zEDn!I^lXV>5tG9f$9hIX%5rEBkbrKCocYEM>!1-lFcClf;Vnq>3`P;;{da~7JMhrA|0+K}sx<)b>qXJ5{E1Yt#x*89O)Iau>8zN#YJ6Q@on4 z`xw)f4O8h7O2TqN6#zg;D0|77bwaCa1l(Ug#>U2Ws<7XiaReD`l&zOI?LX#p*osxo zR~ONiKVIMck`kko@lI&gwS(5;q-G|?lUE9tUY*W#ydWwlidN$K!hX4rdndfapb(j6 za}=dHNOcH|^v4o^BLE~$ZIS1I{%3Ber0|zJTA!lZrdE_H7PD@!RtlT%~47^If3ym~;bT z=1nmsV6|y|S-s!B)V8Li?!1y#iUHxV>-#Kc+iIKTrNzL-`}0BQaOxE%ue%&J zj>^DO>v2u)T$tI=oFGoi>Fh5d#5_!U6WN2Wmqt}D5&QCe0Pi?9Tz_+HZs-x@NLiPC z#!W+)pzN825uF_Ge$_~}!b~nh5eE7fcsw~GWf)Z8ZU4n|qHzo)f3gdh6QhL3yjoZ{ z)Dhxc%sbLN*b`T})Plws8P8g%I2=k}#v5Un>?}E1=?W#b7#A3Wo%34pobk)eoV0Dl zy^UwvykMW}i4h}l6_ia@7RvNyBbsf^rs=nPX1m-)k>GjGxtF7@vpS1QuTp!9tXT?3 z8SnMd;^E^0Oc>PCi|3Dezk5dnC^WZZq`*%=BX#@jjVB?1`he+TF`|+UU|>|R5Bu@8 z<^Egs!ZHgQb-H$z$2 zGH?+ebw&II?nViyol^Kq7U!xKQgRikm77GJ^FkJfpB1NeuQ#Qu7V?@r5l^*G&~#9H zYCM(s0aaXGe74TH*PiEK@sk`?CrqvM{<7Wm+g3?4iS@DaU=*czR!gk28VUWI;jrPiANSkQ0v(-P<$6Vri83`@I!&XV zqvIdCB4MG%1(W%6#>&c(c1CluCZoADjTRGRyY=kCC%b&Q-U$<>ZxP^MY5C(lYYxKe z0^k2jJ-F`FV^!-w=9hY8{ZbE}pXy;_o&h$Q%nx~_rdBi`8#e^XkI|@;_tmslOJ7oM zJzo3#d67IUE+^;6&vEW4U1eTQBhzD8 znT2gioW8XGlft@bvB?T^^W-m$!)P6dDO!~GFKs7&xGUg;3Zwe8|;$N}nkz_Dd z3nQZK`qIgJR0JuNQr8q-xOA0(3aT_464^VzlpN5( zVRM-zrcMOem9I*CM?8RWVHOCs*9u!Pw(31T0A>XOQ(8rqYH>QB2WvlDsedjcQ{0zy zE0^qa^=cD0j>S0$0Po&JBqUICnx|rzrh27C()3Jvbv4y77mHVr78r2juwrum1cF|4 z)ci#eh&r@7D0iG7)j;{>QO=`45)nZesAA&D--f?pa5I`+HXxLfx_4!N`r(S2cdU|| zJNVN1Fp+qY5F9}bVoM2MG!B8Dug<-o374aHDr`w6GZ_hvQSwUK!~!)p`g&d!i;Xhw zr(jS<%z55w5wJFNY7PgtY#J9QJ^*nc=c<>#C**M%2~mz?v$y%id{jfHlBZ=bhEx0u z2E}o^y~cQp(7kFR98w@=Ng){-UTympu_Y9+ZQGOfk>_Z*;tV*MFs)k3my>O|I${ORFEA zjfzK;$W)WC!<71iqjBjqy1SzCUT|@#kL6#^mQl!mET$bCkI%*G z%fwF;X-*UNLrfditf=bkjbv;02{lQfoJax6UVqXLZ^-X?N0}X!oag@Z($o}xWfDSV ziWcagf%4U21U`~x^^C{d?h@P(EM~;-%WdT8(yw?-C^BjgSk(=_o;Xb$%X!CJ&@R`o z>Qn4kMQElA#$``)f-xX{#{?h}4-SaSsdJlK=ls|A-+I}p{HbZ?)O!M=fhZ(LIVZIz zo^uh_9E2`Mt0GIhub*FBqTq*8dA%yDwEPu=kDBp#3rn+}v*C0WQetwvVK)xt+-_IG z0|qnE5OIOY8qH?S3hGYO`HZc1c~;KMnk7HvnRI&Hw|-o=BDu#5%wvHaIde`Jw(fng z8J1yxj{pkB?PEuqL7KBY+tYJC$T93mJ!~FVA}LgT9;7Xlp?(~1v-d-Qx$Eu9YDxwJ zB?iy*>*Xd6? z{>=g~8bCaA?CgzW*c_C=NGbWjH&BIKpMjH=R@9$LDzv$)@%F~F4Mo~_HsAv$ErRBq?ynIx{iRxQAWRkcWF{1yE~13qPENnteO`=8N|%Dn_j2NzBE-n!(JdE@ZZ2!_QDnC7ox(WH1VDI6?!F(ZS>V>#`>{q|~gF^)v!!vnebc2L`@5`^03p13s)< z?v9o$))(n5z8NeU{NQz>(cJaSXEp*T{r5*YR}3=`9;IJHX7c@%bj`LYH4R5zF|>QwS@jkQ(?e)~CzhQGp=8q#J-Cq$5gP8~V~_u5HQ zHg~=GKqLuxMWu>dzWu6}%<@v_r_A-cP^*TPKSv`g^aUkFkhbGeFyl(LaZkkRE7z@J zv2@dzKI2g6AP!VXW_^)XRh!0qvW&?zkPcckaOJJ7GrsBR_67b)JxAS5+jW#mF47o_ z@x+35_3@a-(v(5B$~Wf85R1W61OSZgz1PkXuFP?(wkgqXb7XRB{l>3lhz7Asa-okh z{#prRr zuj45b&u;m)XV%TM=;1TA7R_-!z+V8S!$q&0mc2CXS=%f(9goOW$&FRulmae>u_`PC zfYfC>ddz%>c^BGvYoS#uE_3W&X*NOjx+7F|S3eG=nAgcOBu8#;cT}`!A&RB{mxBp- znOEeWZ~1(+Rrk;K7oVCOWh;;`JzynBeDvGnX_5N{1KJWr3v)m+yT7BC@yJ%>1%~y$ zOKZ@mL`)85n$)-jE!=klElJx#A$T5@NbPR&d@RsKCNR9R)BTjsD6>^E?{WR!#o8db za9C(@g?7CTq_K&_FzLYw;EKx6x1(^{EE$Oxr3!sMF?kdz5{!$~rH;gDyApYI&d6e# zl#U0>zKxB@O5S@Fi8XJNV`B&K-vr0#BRYjX?T zDSVAj=?^b=8~qVL7sgVQ-iH) z9FsxrVmlrPB7JRnN)fud_~17V?1}!QT|#nZGEhwK2XZg_FVu?HeP1BvZ(28atc>uzucFk zap!;cgRdi5jKUx1uJzlU_i5GS*o#GE)eVi?QKx1NZyarT8u+VA^E$N#-DonJ&0V)8 zGkM;?42L`krf8yX51V>zM;V zcugX@M-6q#*A1d|8Goh=_pR^6LT|Gb*j>=jAxG83Jj~{kc4~D_f#(TsE-s8bk24%o zVtUI3{ZKpON9Wd4z0Ajyf7Er!8>VI<-dnyvQLxS@o_RW#8^Er?yVAqJ}B?NC0%+$#$RRbcF!j=h(3c*D&yw+U^Q`Ei9)`;?T@B= zIr3lX;E%RVr-4{)kMrf+P<>;Z{Vjq& zspVfiAbUytKBfiirubaOXLjYHX*ai&X1@UzGx@jsBJGXy3axdQCtMfTyYj~|fB4!o z=L^SzCF2U%=6s2q@~}LS;wCIny{44~r1&VLN}T>p3cH+B1*hhgQAZr&k$9Bj9FczJ zeb==wNa==QcrtoK;Cl8q*$;`YLOI9D<&)hO_L)_mP^MtQd^>|bKm7~f@llU<`I=zG zMwP}v5%zUL9i8jA~ZAJVW?;QXaOW}|dp+#F4NV5!;P1L7~9jwu_Rip#4zmQCK z#n0HT0Eb3VI8*vYSu!e!b#?yAN0%aT4&7YnPX@WKu<89HzOE!k*5a9f0xhoY*I_*R zwP3{Gp-7YKFX(b1`b|F_=uzk+>W{S zYP90p>a5Ebn}y7#g=fn^JMY=D%%JQa$MeS(atI-(`o+m+%Ffrq+8y=tJ)c+W(P@8t z{6wAKs|SGhfBr9zEvs z^VF8o-g}zpn0w;yblb>{h<*M%@l074$QIbx`)m%GU%$I|y*%q0pQRD{C!t~G@jHXz z1?0!6Mr|IaC9jmLu`Y@!dEq#8i;W4#u50t*9xWG?15B(+VKT zt{7z$v}f?h$jzw0V75P-B<1mP*yirKI!Eh+(lIe9j@=ZArlzI!LEmHi?q?ze{}zeB z|MOXSNY|5KAS=Hdl^Y}aUP;Lq4SrPy$Q0KN9_s#iaT7bdQkHDYt!u;K)2XR4W9+9# z?VE;IiGRJ;?e}Xx067A3=%7+e6g}4SQ}!NEloU`iIYA0Q@+$}G3D8Dj0{=e*^z~~B z#HLEWi#GsM$uhMek4EjKuW{C~DiN{4{Up#B1bXXK`=!ftCoyT)q>34~T83ocp1?|iqOAG_7>5zzxgz^rd&6-ifRSmoLO^8&aZ|eZVM?l=r(rjaO zmv^`nLZB!a3v|zb>hYV!n10ot1)w{FO^B*#Yc4cVZmRlCD@7)SRwWODQr(X7Td;f! z`5iDIHzaUyx3`SLLFIogkJ(B>a#9u8wU|EMB5()A~4oD@1V1jVB5WBPV z+X}MROr49_`A6^+=%|O%1E0{4)%!>LMknv}#=T6aO)&4Aov5!p?>~G|>)PEjJvUSD z*mvE}Nwt+u9j7)C8RzV~u%MsgQ9t?Q;c_5V^5+f`&%}-6n&6EjXH~zNy3;U;GRMIe zz<@P|K{EZv%hB)&H< zfq~@jTFr}SQ@9AZk>A1n{y9r(vh8ma->o|=$e(dApBe&U1k=k9c2YR_Uq541P5oYH z&f$#xtLNu8z9`r*&!SX+*OQ&B>1nW}j7;iCq|JToFc!aTh4KX3cs&5q?J{Tcc2^KYjCr>SZ{~8VfCI{Do3Wi*r z(EAlBzS7HZOu&~6Vr2AivTgCuNf7Js%L*)26mx1%9E_}2>8kx2iL4q`ZF)#k`Pob- zt>`fcwkT_^Ki2rYFdRd>Vp#6ucSVXS{ov4Ne-;3TKr5#EU4-1IH=fmI)emh&0YDmB zRjd>Edzf%Pe~^*yA%if16}0*MT!GTCQ7*2A_SN2s?=?-GY{)`@b&9&g{78-ji-<^c zVFHf|F04NbOhpXx4@*vOJ$oscgDY#CijE8t&2`clMd63$Y|OzC^8JfT${|gjx7>co zac6|b+f)9$SGtmI4((%qG3g!^oE~CJ&|HHrCWMv}_M+XdGm$ciQ$WmCE+P*M(zM*X zP!J>xBuyJe%~JbrpB=!q#ZZPuo(YDOgr%e}ndmY3;`>H*CE{#hcOj6%d=DntioO1b z$h&vv2uw^&*{gB&>uq?+X;KVN^*rupWBnlzKj`Q7%RGSj5JJ$L*dqNmhAWp=SXy-X zE*z{mI5a{ZL_~cR1O_F#z}kWdCpbf|o_FEfs)6*Dp?wm$G+-4cS7gxJWULV!)Vr6r zsEyb@+-Pezu%z7%9}@3qECXs_kgMs&hW{K7|{wjX&Jftf|bZF7Gu*tx=^RCi8mx5LxbhfJfZBp$Ic zhQ~YRGDYb1E+-R#M_a9dHlN=-$ZJ{nbTr4wS!QhcF+bzG(c@WiEM?UfzLolVdIqyK zZu6sp$~@ZCOsV|P;)OWD6nF_hDM{BgYRz#WXE%E7Jgt*rL9=(#@s8&n+hduax99vNoS$GOE4_havoDVpOV)Es#y^TD?ivrNWQyzH?dh4X&4t!Q0{%p)2n+Uh( zwu)gjqUQ&0vvckz%WgtYtfNS>2YPLXswgdzIUT4}Wq34DYD(0WpjFF971=?6XehUp z?##Rb=^;L-bgV1FL4L1OPpSNA>YG&E9HO@hJ)>ipH2G$nIDQ0 zIGnvqs%vH193%;tB05?Ar*W8&qhx9W9bhxvPs@GYf#C61tI1x^+Dyy!ybg@#Cyhiw z*iip6=?BP`$mOooqpRmIgqWfT&_Tgfbw)U9aCZY4CZr4cg9!O0pf6{{Rb+5JQ#eoQ zknUk38^Cvc^DRm}r2}08`_=BtCPL7}YU1d}d3qSV=%G(6HG%;DMVeH89Cehz7Y7+M z>8WG#Lnq2F>M@qw<}Q(6q z%oisw5oPoE{)5WnJ`@)Oc+$@)@0ggL!`2M)w?N1s7tS;XOe+z%MX>6d9*xk;KxaAy zY-C@if8UFHju)RV&UW_Tomi&;7I>d|kJJSHfg@%TogH9+npl2vNS{D43y0>8M@nEQ zGW;Y8AZJ9c<~>qVUa7-*$&+GcoBEmHwV(ozY9wdf^DjyzVkH2!u*VG0+v5STaBuSu z^wdv&1>SJTv7$ePN?K^6jO zFd=1WqW1yfc~HXXwe>goWOL$A3)qzmr~pzv{ELOg;z^M7yPeDK>pn*|qIzeY?9l3a z8R*e&bPqw$i~n;0POjD+VwK&7jx>?91UZZt0btwI52)DLQzuMTBp>boS=<^_cv4-0 zdj@X=p|2k03$ESbCPmPH@QZ87Ib_YQxa~}!LgrI|= z1djl#coqjRhwnZ{>WZ!v(qN~cDxgf#u#Qe@XjrI9I;-owO$|Gz(%5 z$O2-y9f0zm6zso3`G2QS!W&qnfF+*b;^NDv8l%b&dvWTP0YWby`jh#QLa#UA4RjEL zi&{?ox3DSf$ZlU`P`p=?ezJp`>G51XZVm;(Bysih{v%zVsuToF-uMSzq-6hQ0{HO$ z(F7p2{NS}q5BGp3N~mJwZaXSEBT|$Z^ICLi|n4upQyY< z22~?c8BckUP1;t)q6W(<3Yu}0XFrp$S0tk*ya<|QL^T-~CB<9>nO5CEF!A{G(*S1-$79R^iGOJ$SL0lX#V<4=E=9Z-{+Cec5Hwf;#6H6eRW4!@QR|U8qAcKuEcU zcB^=Znm!_aEHoX8MNCJC9#$Z)SKWqu_Bzq9Sm?yn(TZ=M+lB=hUf|!>KIzHv%aUw_ zn~-vH#)T4cK344T^+Dl`#X$u+_`6~(v;mKnx62N{4e)l@cG*P6A*q{`XmwthOiv&J15`eGiaK;6{X>^q6xb`hn5&>g%S~=^7 zaS~Umms@#CNp88~Jr#IDUl2tNSS4Yu0-RYZl{?RdCG{Dam921{P?1@rfA z2+PN;?4dGp2Ro@op{!LwoW3gkLL>G$yp#%sJM`eyYwZKJi~9mD3Dmi0k{mB!bDtl zpCcpR#P}yAx%-Ba@X6Y5jA6O8Dho%VV(BuDM}bro3bhRtm6etKPTJa6ewqBkSgt&Y z301Er5RO(=*_*pCNSfO>M)Cw@)mzfnN&6`Xtccp`Bi&aUfse;fuXf-Iw`dkdJ^usB zP2;_hA&P9-VZ{mJ{=S3n#xqx>cfYNpL#aDDOTo+xRyrJDARyheWvz{wHZW#Sbowbp zb#?H{9`Rr@gG(Dk6EewKvQwqE-wuD_DEA5{dhBy zDp`4-2gf;jU*A7@`t*W$0|}il-Sg($pCF4pv;6}>P9c`|eR2Md@|J|J3V#=sVrBq? z5do*E_=iNXcuWsV)wpH%7fU?CRK&bc`A|&gbd<^d$D?bh?P>E=gPD)mA}j~DV@e)g zLNItCQ}AJxhx)^$VNO4WC-740CAS~GcQZ6bMAK_9xQ**zK& z5h0kwogx4<2WFVIYi>L0`G`bm8y-isyJyJ~t#3XZ?mossyAt~4;~zkP$juk$M6#1t zThLX+=Hhyy&5pJ$W=5bCNzzT4V>FvsmPU|t0tlkfgA+kSLut$9p{LzX?4NK|=NE_H zLbhgTE1&tX(!M@UL63 zhc+dXwyI^_{sWh@FGyyW2hSP|KM?S|M@2&uL65g`IqYRO8gRZ(!^fARUVh@%*tpx^ zYBZf+`F%7HqXGM+ZQ`n{rC{j;fc^5AknahW&`SK8I>)*>IdGT~-}RR51yI{J+8i_2 zPgR|$;$vlv5)EqzDrnqMmz0(+JKpZmb~l=9a0h~8SzFJ`whHioWyAT3M$PK%r2#vo zjEmoUEhn!jhU=?C#-owp{(+qtgYna)r580SnmU$sU#`A?H#dM9)7~Q*N44b2{uu(b zCF};vO=aJC)hRFa4FSgu4altMH-FOq$PlMzwYn5FfHY?~02A@2R`pezpJ*;X!@e5ELAo)@fPyw$fErwqQ?r{l#0u7+y}GW9M{j z%j4Q?swerHe8%B2$>II_k7l1W`u!i$j%uZEAJv~+yqV}(j?t1_P3CTT{iz-4T>;z5 z;$^=lrINICNS=(e4lhtyBe3nYFVXf0s5V(n0s1h@Q7adBG$BKGcw*-x%0!($9j%7 zewdAndBP0ckL_`}6%5a2qhAQ)6i~@*Dakn1?lI*>Wk|&g-p2q7yQNsGeox!Uz|;B6 zWp6UY#;mPMHi1o5)*9M%>vVAn`*(PVRHX`lZu+ z^&#j(OIKcDgEZcw+JU06o9(i7ok1at>y#O%nJUEKl`x)*0CNR{_U{!*`Ckxk!1rTuX);?6XMnqkaQPUivw;lqFj(j&71el^;X9U@q2UT7T9oNVtz_4N%#a4 zjkw2~k-}HiseafGcX9N;wTw;3%vxpOCji6nXvDp%S<$k_-?_#^3uIn2e=7WIE z*)r33e!1yzhAF%HC}&gcq$7fQ4@q(Pa&4}I$1XSLhNB&l_1aGJZAYh*K2l-xZ_{CU+hgQ%^zk-Nxiq=YAcHnvpPS|`@}9*m+yZ%`fapR9duec#PtdHFd@*m5 z)<~bwSO~D@qA_AW_Z@lSjp~|YrH6qhrss!<&macP#Q?y3-81n59-SmkU1muD z^Y}mqhG_v)WHX^nLGE{9cR@WDgj%HxHfnQhlwpQeLZvrOE zO-(l&_G?U@KthSQMZb!Lr1vINelstuTCj;f>rtw>4r!4Dny+&V`-v;;+u6Uier;*? z>0@SSam7ZCebvYO<=JyBztBqwYGjZxncc)Y^cx(AQ|dtYb0PDES8~aE81uw4wMsw^ z)%Yvlvj9KT+0d`W4c95`>xs5~(R2NP-5Pmim0kMGqAHZjeoH)o%`&rc_96`Y_Js0k zd6gu`c$o4a7R20+zD`oQ;xlZWzn9tlD~hce-@ zh^G%Ksr3B5nrMwwzJHjnQW_!ntr;d5PXpFpDFh2noh~a#+i{djgk{}mDXneEb}G|1 zEEKDr=}l#7aUKm1Nndy<;4b5E1Rj$0ed6=OLDw3n?Vn%*OKne@Cnt3u-W7q8Nox z05MYQ9M*HUC!CGx-Bvn@cz!~j6z%V)D3TqiU@zmzo=?9C)})uzoo7qJ=^KrpeLOXe z^Ss3*5KM=KlOpv4HYv%tNT`m7(dau?phdOGaeMXrXylVzB-;76SLY-yYkD;Rvg82G z=MLMPwFS6n#b=Y77N3D$oG$arqjedj%sRsu&u(g1ndMya+nZg}T9x_vB#NCT7$6{Yy~Z;(e*6D}J++S^Eh2=Ghy5^tag2N+iNKFhsKF6b zf<--=Rof zQV9yQHK1tGfC;LK6k0KEblmn0&d;M_be?>HFhuNrxNQ&f3=c&eOgnsb5d#$SNA+h2!ZxGrx zai4#YEv>NOJ>coSz`HIsYIqVd;({Lh5}-M3U(4p{ zZFf&kI-a&u;Ej5<;!3fElvLWVUsi9Gb*22JL&iDEtm|&$(z@o`H(Oa9DNc%Gu5HUP zo$`i^jZ6|fqB>$=jjJ#bv{q;LaKHm+sB~6#x;I*=qtl>z+i+=t>wf#y?A-cu>>#h( zS(}NsLN##JeI3%I%))&F{uvBljW08Bg4qS2=IosqZ8Y5!f3UaxpAji9{>-j|xh zK%!({oC5z{K}51Le1F6#s!1?u9h76DzEqJOf=st?GyYlB_y%|RdcjO1sheZGDJ@Jk zT&AdM|9E_OyA{K7(!oJ1>E%4i#n~sh0FeZSf&32lG^H)C0b6Ftr2Y0hyuE` zn0(RRyxKhtEjP&+JTg|wQAm5Q?HMlzGJp=B!H5j0jB)F)?;kCF5APT>na(A! zVM~T=e%Dx``gq^3TL>dGMz3Rpi?~q+?@QeZ+83Any_bg1bVI%&L4iidk0i@%)p51* zx*8Q`8TVGBU9R_!&-`}{qr0P2#vt$UK`wmlZx!A)xA+ZZa@egE5tg2A4f?!)RLdXo zK}oq>e{09%Va-{XB9_Nu`-h{x+S`of?ii)^hKIRvkWcX}#)4)V5Ak(L(BnXc7-?a= zH_*^>)qHjO>PJ}50~xE|aPaIe4VSo!6^hLoW)`tJfp7hb3Qfif==&$CQ$5ffp9Z8Z zs?YaY|H=&RuUsww!yAE2UK>@&_oqP4#2d(9Lif_4=~8d;rwe?erWkYN$onMHx)20A zc_G4iY(!a%9-REo99JT-xe51+1BSC+Kh~PA%shINBb{BGSf^iOI)posTRL^4 zCnLV_6i9TMReRO0-msCI=Qm#rwWu)Pq&%O@FvZ|;iPZ8z`~Qfuo7>b>g3>l=r^p9RIr9 zF;SXr&>j6hb&se8)is9@OR#I~S|pMpqI(sk*e8ZFn)-RuI36*reVfkl*~^9gf*|&m zpvf5f8!#4Eh@s^!76t*c-@pPL?8RIPR5ynntU6>WCLiL&zrG6`@;V)=yiK4#XXa@uY>23^08l*u=x`tLdbSP;lX^`$lkPx3W`2GF&cOQE{$KJo^ z)$`*2S`Y8>UhBTrTIY41pEV+!r@95W zzTaIMP3)I4ay+c1_BFq#dL)U!5sfqb;wy#*)S`Hewo}Et>W_0nNIZXqdXMfQ}akq@;%)xDiB)(Z>N}&ZI}d{UfhV zoVz{AS>SR%svWcBhxI)4%08iJ2%mpL#u*c|-`c0T`KEiE;W{dauw}{H64mvZ;p*~0 zmO^mMXi?@-qx>SK(67Ux96}MZ_ED#!QKn1|JOh3O(Vp~+k=sJg-->aP{VtcJ!3n)X z6>5To*_$ko)hA70D&usrFM&ua_tos`cMu`G$6u(evDN$&<;)?Ib%*wf zlg(Bjt)KdKnx~+aSJbW-2h~P;8#9N02}~+W!Q|fLd{U68a+znP1xRz#q9|8ntgakw zO)7*?HHh^GPDRC8zOQ$!P-Z0cyPo)!KNteF{N9msGjO| z(hSQliwEI@nX;xQ50mm|rU%Hd#az`aqDZVgp!|V*)*1rQL~@#5?wXj#uW!WvQt%*i z=SuzJp61Ua6+pf5mWD|tpmrlL!j(<);59E<%xI~BS%OTcb0N~8dB(ibR=+xG=4JZ^ zYiMz@+>~FFg)^7t(0iR^GUZWRoxh{X;jt-*3 zGr^GimB*B^nn$cNyr0OveXO+c=q6g(sl{I@ntLR*i3=uyDJAhyM3VRxPEyN2GS@0F z1xTw&h()anJfjoUZdX)`lk-29GVv2q@#v(DT7Vu^X~cMq83c-T)|RP!HkKY>km$2_ z_OA~A?3r4J9~X27PXK4fVac}NtiL~e4Os1J zCj4+6!g)t`at87;SO25+pmIyq{4gaLkJ9-&4l2p1_p9d}Zl(L34it!W8IKJih+pr^ z_~?{0U&vb>?yi`uoU;*|@3V=e6?e38UGVhtF5Vf@jig~}@hu-{$>H8h568b^&ZK&g zlg$_k`#syGj?-}ZhSENo0gv*%=)nYq_$wI)6`vzdz3xFKf5oh5nc(uhE=0H}UV1x- zGnKt?Y>UgXc~cC|gqE=VJ?Es;rIjjz=jU|1VvXgzSC+c8eov(;D`XwjuG}9qPl({% z=9s0#)iFyHTIg{*;&&p8&M-8Ws`<+eAZl*1Mg*`6wifxu)WD3xV(P!99P<-5f zOw^=S!^+n6lm?Pvdn<%R%cIG`L-ELGP908lk*lF{#2ZYM#!+I;V$_F$gTq~qM>zasL8f;MCn{!>Now6mOiIj=Lh`~ydFUtN%fmzotfqJ8>|wW zraM)`!^2L2%Ph#)NlCV65NUiE@a0ME&&5qBjRjq8ft zntxaAmgivLsL{7UvN7BeSsIzlgN+ys`6Z`{1wlGHH#4;H4W*;H@-@tYXumq^)qFp8 zB92~x!=$kY8z53az=m%57NHQbH}jkms;$=T6eln3HY;)-?#m;Nqf#Ejg?OZN=nxa*do zzvvd$i}LJHHOi*@obmL;_-jVXnv+RW$*ka!g@Io&5cK3l_c|xJSOTY)zM23!LRAw* z#{}@klNfm}Eoo}ci$=0N*{ezE_|nA_Ek|Kd+|UwNhN^AlAgCp&74+gL|K3WB3j&|& zeR$=rvI~@G&9biOztJp{*^z99&H=eT%oahxWyaGJj9s!nc*tpTYyB+-O0Jz$!H%#{ zkQ~p~g0)8dZX55W_~g9|Y>q$=U(}(Qy^62j&TK)W zch~G?7i{c#;Mr${r4Bw3oYdIUm8fw~-iqUHmP6)I{aM=C>ZH9gEy>+c5u6+C^j{Kj z$uCqY@)RS}rq@gd@H@_#8^Y_It$3}+C07Dd`>?%N!${!FHD$W}qdd7E`Ct?DAN!dEYwLH}gN&<}|@jJe? z%k_rQfd-t3a=4djYE;dGpB|Pdxku45sRTT9VWV=}Gaf||DNCbzsO z7u-^V%LR6X*iNyw#*P&IuE4p~-QhcGbR3lOa`w^n?gVx+<90p>8?xc#oBt{xXGFO= ztm+;`@W*$?AaoBPImW$dqR_B2ARi^?s;1r-{n-3=LBzO&{~27reYh;WCuldpHty=kR#;x%uoEPHt2gwID4; zLb;hefzUIWw;2s%;&Yh+#3ysZ%>_fbLjYR9CQ#^oryRw+*&?@ zi|%Oa!4ct7{Mgf^ODA8~!Z@71871=i1==&W~h-aL#cmj)C59lPrzNfIB4UpwL4L zre2UlAytg{q-hjz>p=UP4JSbwq;2pFQH`5jW_l>=+*6JkrSg-sU@KMiN)^=OiX;?F zC+%nDH`etuz%%gZ{6e@tlUZV59baC~kyVDmKM*Xv67m}@wjP++zpiAWJ|?zDkfyd0 zEuivsizxOUHuWq;QqvuUyba!PC)a2qfQ89nK^~0A8W+?!G~+;oosjvCkQKR715wNCu);`V6!|^G-lt*cE zM)S5Zd>OQ1!SrAJ%D_>iIdOr)TT4Dfu#+=L{dyq_JO|zKI8e)xkI}dEaXo&RR$UZU zW|~smE+lUU8go&K6}VfZ#{4^KC!fforS8bAOZjPP5*436uhnf26UplL@M%8=t)i*C zKw7UCg7~*7;|&GFKB-CgsZmY5z*&)jFT8D`aZsc@qHxj?oRabK5^fY;QwHWgHJMCrz!7Rv zu2JcQ=3-HKtn`l`KV_Jk)_puF$MHZx15s@bHy@fYrMC&}=BjHJgI-L32)-Hoh@0vn zM2rrkQLV|p#hvA2uWT+|v1KTD{aE*_Q2pGh72Q6{mKi}qpp)t8p`QQVuwV86o&0F0 z@hpO3<6HdMt^}EcaO!|g1=ky67lHO=b;~ZZG-uL^L_0R0$knNwWTZ0)h97=Qwjc9E z*4%x4BznM>w1DVH6x9ems9FqZKvY`zs-d+U8)+;^irE_kE64GScl8$`k0PJa3*VC^ zZLifig+&E0*V`auXD9XFk!5A544+?mmkqM!|E57!Yyi{9pEpHWeM#b5jfncOCTVfC z@kg&Q;VHzRxXK&%J{*B7c?5-*l5*Gt@scxOrBjkpo~%Y?RXx0tLghC;*v2o@YXFzK z6KJD|{OPP$`xY)Q<1qP6g+FcTmaY&qtaAKMuNOv^mu_@0E|(rk1Qk@n>QG-s-K3y2 zEi6-PXYQ6ZxuV{nv`aFfyNwPUmarPFgH@g1H|i z%fBBNr3-_x#C$0=tW~Ni{~3wkzc)Tziwr8$WDLljX*|QaCLmLladF4_Jl{Oys1WTZ z^y)`elp-0BRAjZ$b46dPx_iefq7%K(^?9$T?gU>8GQ){ImSdeoATE9J}%l02k zi0lIJnc(iJ3#3%;ep9y9I8i@1Yg4_{+i1f^br!DiG$|O0E$cgm8!{eS&6cFz=4+vG zH>5)<-oAQf*)GWWBo#mU4O+-$Kx!|Nl$BK+NOFyk%(b)t=w2Ve_<#Xn8BsJnIRMSG@lI* z(sq(_Vb~-aF}*`T3~eGN)~(6kEkD2b^zt=I#H)e;4uY zErr&C#54uj7$@lp8-Ao$1E0T1JZ-v%>8qZoRjb=w!3(!=X;_bI&-c4m$AfcbSQ=im ztT<5^ae!t0D)QG_>eC2Pyl@en6g5%?0UjQ$<=nY4Q^Vn3Au$uqX$l;oAJNBR(f~k*nHE=){C*z547Bkf^{GGd=WOrv|T?seyju8c>uk@PNuOE`)s_ zZr&N4!#yBG8(5@%t-)?nrTQypbI)8gcLQ<#nDZVK#&y?=>gEP=rtzt!3*WV3ZWQ)$ z;8`0YWZHVf6r~Ev1_3VQfrUPLnqyyd^)%8C7d00Tr$#VH{@pIkN<5`u%i(9k= zI23H>4zGHQ8~YZGtbprc*#DE{Nt)PW!6fsD#Bu(k_6^(iI76$SmOfXWkwC{V7v7rK z+;-*)k&2lm;X;~lkm`eOtnxxDvSzvW-9VQP@B+kjYW_6+yy8FWcaK)SqFm(J?#Tz| z6%IY}{;%R2$jYbFHlG}5bsU+IyD}Yf_vE`4lY15i6n;l~rH_Q-(TK7Hyq^rRj68VU zN$St>F<2O=O3Yc%ACDUMuU+$Vc#myD?05FVQb|osH&l8*uAgw061b| zui)TQ??0#8opQdVPDwbQtN2)M<(B=d?{O+HX{526;PjKl<&TfJ#%-8NS$zFYUv=wE zyT`da9GrN?ypO&cb;ZFMGPqZorMVK&GLf@4i5Iw1B>j3(nG;<-EZ{0mN*T(m^s`J? zy9E~)7q=xma(g;hro{exxef8(GoCc>4Ke;DF4=UUi#KVH%dTFd%nRx z-S^_gFHqY)h8jf1f#3L5=+~_-QO|~J-mpjtal^2-(15YS0xI2or>vF0Jgy@^UFDQ> zor#olrg`m5lQg^18xxb^A>_^4M|N`l6DAfeHO`aVXs9G>CqwqU>3p6?sNLN0>EVdk zVq?GtsHe>W5uPVau1Dj)7CjO>NPTuRz>nHQ^oiE8S3R(6_NBsIoi*{w>nG~3^B^MU z^jBZhcOfy-HmWmrMVl^){&h`(nY+{Q+a!`O$wyFGnSH9XPvA7+u(-k8{guY3t>s=3 zIHfR$CiiUt8H+*qpl2{pZ|2Aqw2~N9m&HtT+t?ZU_7OANZ??|Sz71OVEj=yJW&v&v zk+tJU{})ZCg}lq>e_Uv_=?>*XnPZ8Nr4g*qGT;aEG!C&8`!F z69Aisf{t;lh;iJ|zpJIDZic}ctP=yY7?y3ut}me30vDFyd8jZdmkTm|8kq~k%2 z+7wfw1_Q9MC(i+-%GRw0NPFO}Uw_eqejSZRQX^gqT>0kfhF;v(BS-6TEiw)iygGe^ zH$kzJYNX-x1(439#|$~E63|2n zLqzSo3>N7$XnOD5Im==8l9H0D?X6W=VnTwy71S6nv`OX-5s8nDCE+URiw>RkMDE_C zdbgJp+^RkD)Y$DRpZsa}4Iy%e#H zjpG~1INtnDm@~d-N1eN|U3TF2_Ur+%E)&=Zg!$D(uSzaq^UlbCbf20oR=*?7f-t0K z4rp9^-d_xWDr4EPQuFwajdQ`h<*@&<&*$<%CsUdX3fKu|$-my?c`bFtImt85KI9Kh zdIDu{!i0p9bH(r5iN zWHa2Yzb&_9L+ci}Gni9vYY^c*`(DboR18|98gcE{KBTlu&)|HVqAv%m1*4xUdrF$; zpAo4@sY$b5dk#$W0YGtb`-s|MJtu#26!z^=A3H{NJ7 zGc&6`P6_Ck8DesI|2=^l`s_bTDaX3qt<2ddjZXOY5+f{G_GQ|+q<7~6(jyRUahy$w zN(YF~)y^xbLf0a3QQ@Z0Z0xA_^L7VKz3-}H+l3??7L&XT$6l`-^f$+LZtEd65n?dA zW?QOhuZ`ZN#2Vp*8H$gEem}gRJASPlpO<1HB}mc)8ixzt`u>^Ed9cg*++-B5Y`_yz zVA=i5+E_p{gt3-^acrhV?!#|IEb4wpl+bmm;8uBrMhJoS-C+X!7XdF&@Ybqr&Rp|k zS*KH(HW%YO*!nkyE=K`n1O|^=1Ol%QzYQN}WXHI8CD6}IZY;K_l*ar(TVq2M3(`Qn zQaUaEN1X4d~GR~SJ zFnm$7z!A+aMN8&=sMCG+3l3Q|%v6jMQc`5I6e2U9m)^N2`W9Hj*h&IOjXPtK1@XHk znnx$@d5G=CY`14gY$>Cf@sWan$YmhOAy(`+8V_6s4U@7=a02?IChL96{IkN3;w zqf^Sk*GXsnh#R3mP-|R1f7SboHv7Q;{U(aOD$W1G(=lsYS|yqPBiDTLF!;ijZWy*p4m<@ z0sbXri`b@n47QL27wryfVMx>g>@Iqubo;!+!}fX3=_2b$8Zo7m$N*wkg+?0<=Qehr zouH4P*799=mZ~M3GuGA=>VAuhvd@Ufy(!&^g4#@#Y&)TA9CcP={4q1-;XHE3r+JDA zrED`pVK&Z0y-?3YZ;&9ciTbBsSarPT}ATVp|}nfuPcc1zpg z=r`%pg>;OGXbr{$lMw;&`~cF0J1goBHUy9f_&3yu`LBBKWdx9+O;H(gt>;+-juLw1 zIg&w@a&C|2V~P+U?;W*F`{EIw-Lwg#7*y2?!*oi98($~AitGMI^;&BWpBDNWkK5yb zUr{*w{^ewOTs_pu$?0j(*c;rtU3F8o-CIui24uSWRW^?R6UeE$)|NW(1#oYsei0Z` z7ZtHKH836!pfU{;6Y!wwf`9IzFspPx&AEuU2&Q^O$8gF~TUv)}uPybaMU-HBS5S>L zH%~3{m~FQw@ia%#2t~M-b>MhUF=9!D?>N6?L&pQCBLD4H!i^c(aJpW^txe|r;gEcf zTEKpW&D3BXg!)G0?wG~-#KAafSxO?&+xmtal85xwkpws9v*&o9hyYvd>(O{ zZBBAOk9PbaZFqkQ0A+ktG7H%H9~Qu1PEJYn%y7n0Tah?Obsd)D9=+?m?0k7dK;a;@ zVCa#U>yscQXpu3t?DM5w#2Pl_;qnjZ$uaw#6U zSA)=EeWe{@*XHFruGVh!-u`9eK{`0J*E+7P?Dq_*b^aWOZWSf{(nRkPcnac6l|fwv zGIfHfBBW7$LLel&Dm*h^qZf1BQ`N57WD%2vh1JMJfib_7j9k7$&Ldf(?R8&HGIm=_ zpL7?5B~@igeUrQgrF6pchABh-Q<$T1J{Q-Shs%^&1Xh0}u_rU)i^1|$q6e)rW6vD- zDxP(c&-ftTl!Z(Uc|^sqKKT6OnsX=(cAYsP#(Befc&Gg!FQI(mjcEJklZpG!_IUCk zB~&w{Pb{*TlaCEYHIew76!Z(=o}7Tpy&d(-IU-RW@l6YR+ZVVc3^p=M`<8% zEQlG(WI%at+wla%Q4ziZoI3WLy`c$*FZ3}U-DiU*ldA*27iYeWy!0yI^ zBq`t&NEK78Gq=VnpN(cN)VA^Zj}X4&I=@J6FA^eUHH^+x-noK%Dwd=-CQ| z$MV7p=Ue0VX&ITAIvp@YA)3?aLjAWprkB4ygR)T(<&Wz5BFMkO@7!GghP4#0T>bK$ zrlUvbEU^#MnxDV9iJl5fqcMUgrgbF`OVZmpYT zL0}i_T!E7!CTBZHt}9;KDOLp2B8@b%Tgeyot5wPJnlBa~hCoE4-W%w`!aUYvR(JA1 zQLUxr3p={B>}PlX5lxlA`=7KG|0h#S@AoWfxZ;X^1^ay>J3+DrII@{DfzRe! zgZIbAiF05zJQab#tW~NVlyCbS9th^#S>*a%ZqxDD1Y$l5pp@qc@d6Qrla=yX-16w*DnB>vK;`11 z0R#U`nwUAT{fV7QwWiaGdXyniIxJ-&!x^Wqbj5(FYjy4}_Q!W{hci|r(o$;a`1J>$ zUtC?{9JyGbM8wpp#X}3A`|w;HwB4)}*>% zWYp@-W;ylrBSVu|FI^XD{?2+k^uJ01DL=P2C63>Z`!J-F+XKk1Kfb z3xHBpsIp>Ye9C{KP`x;u4U)&RCc=2XbtrTU1Z&T?+s$!k?SLS3164>~~vzaGMsMU^KNFE!It_b6I8^Zu`AI)7IGAJ70gu$}oQaf#P?Gq#M|to?S<-li`3X4>rIZi@BSdHl_=)lM9CI z8?CRY$Rq@-J5$|f9^Am!$=lQM{J>^V6;>BQ`PR%6fbg<)!}npG-Q6)5J@q?HkVpmF z{#qTuj2*zVl0Q$Ln^%GrC{h>tHLTNLfy?2Ks^^c~eqEmll6jf>tkT{pY2|@#Tm8q_ zX4={m;libOMqa9S+d4w>Oft*dyG3RoOFhHFMxl5RKz-gB_zvQjClN_1(PCvIo0=fR;)p-3|;ah~&e07xQv%e~3Op?5SiG^&;w z!dm{X z8RVShpy^2_C}k)vSOx5M6kg`#6a~Cd$Zd+?;!Hti-7P zsYe;PiO&E8p2OfOxWN;jqyRn)Pd1t2`#iU|yD%Y??u%dXLwT$sFlSw*7v=V}g{zVG z4@y!7xTEGxmB5FCdXBj9u2hzy+~0k9?6#1uYYwCQVA%{`f&VCpHOqcQ+2<7)$j}pg zwj{t*`73I9&I$+x*qzx!)I(VX7C#2P6OH!)Q9=n_VXi43Uk4kB<8H0F#z-01+9oH)!If{wa_gdi;rj?gBd8PXZF z#(1B8A&5kUbgj+XQb+vy{7>fdepGxA^!WT4v+Zoc3dwWa zC=v6iATJly6_PH(;7y5NRubXV(8zCgVp6v%aoYBucJ*@Rt~FC<;y3jZpj?u!7P2mO zo0bm6toC)^RezUn|N6^Xg_cLx?e(Zk{_;j?(DnL6Nq}kf*URTcBRl{AqH&gYcsm3w zUFy1;c|N%Mk7#CWj0t~9aU6;K*4;SOZyy-W+9+{WXT{q1Ig?p3t{RQ;#h0w(z}rGr zaD4YAE^`eLf58?09%n^Xh=>qF1tYs^SJISy=ofZP*lbw(-V-k>aN;KXztK-kj@KJPo!@mwJ7(^DCUK%FwZmw2JY#3iq}>$QqjGm%rnC z-qqxw$f*O1g_j4!^g@6M?)g|5eX()l#C@a#HQ;;ZfIZ*1Dh$q<3=Wpv>Q7_tt3;1f zpew(&$!_7_c3;~em=FcopCui6Hq5FpiAq#{HPU{G?s5)TbC`%J}|IelDeE*=YD)ynNan%T~lOox1sD_W0Vr$sYe1EQ&YS zSkP)aBQWRV4h823;B5L~v=YW|aLmkNZo~q`4V?~UC!yFWmBsP1e@xMzJ_~k4cSYRT znwgqlz*ef;j|_*VoJd*g48auCDd`1`{>OhkXpJ$>+_aXKd{Kptab+t{^)zC{rq9e z_mVJhAxHZH2_cO87r7+`woUip4b>nUUs+?MmzM0~}Rko~-;kGKNFv_nFb zEw_GMSR|E2C!0uPIT8*x>0ua634{yts=e>6NBpc{2%X?BmN(L8?ou}!@)MH`4WMwQ zD^gGEQkEkE;HQk!;&#WEFDx&=pZTOXP6L)RRss}u$4NgS+=VRYt+42UNa=1i!Fq6t zApZnjIRh#rR-;woD8y!xC0hO{EGz!Uy$70$Qr}$(pf<3*`!CSOw;N?=QUf3UuN}0|tY-b2Ib0mQdd->Ij%^U)ms0=@SId5g~VtrQTZ;{l#rC8P~laf$Dz?&0Gv{m?$fH>t~Rx zc2=p(!p#t7&G<&cYfJIe+uLde!%Xv;+-FKSVuvq9w$9I5Fz5c8q4+RU;dffn0~+w* zJQZGCGJK+q?4Toe@?!y4)o{*{I8(Y4O9zmsFkqke#u>f`l#B;j9m? z0XyCbyi@!>O^DEgw1Bv5>!`yYm9|!c!(xZiC>d32Iy4*yhwPPP9Q|JIaL>tV`vSb^V#~z@c*>+-A9j=wlpU2|= z1Uf=#qvlJw5n^g7p4$a=)>s-j&Xv#Z)akODaV>p{GF^*uH^l602U736SUZx5=9LE3m=6$^X~Q&A-N!ap7$Y%*4}ZsMEVkNQXx_Qj_wJw^4#|QT7LdLBV&F zjBB_A&e(>@NoI+lC7=<8O8eXO<+-9Z0@@OXt><_%6{z@1DIaMqFJkt``%wIGdyS(MfHPR5`=-XAfxIp5SYmp z{I45l`OVva`XmKP51WT3+aMFzrh-+x!zpB~0N?HFpsPYE(e7e75izN%gob7-K9vy=5j-0lc9}U#T*zHDnU#8X2rvgtD_Fu?0bT@59 z3lt*o#SOG}1ZarOZ996tx&~NDeKIlr$lf8o}oRdXBiy z<<$dS2p7tT5HeoH(|W5;{Rgr&Ei6$cZRKSwjY3&(`JHOyWHf*~RTs&*UsRS4M)?Y+ z{r~EdAti){gsLgcnXZ@KdbT%_#W<)kP70>=(5WNuehTm1L7upZ$k4(%&szWX?_#7jOfkbuO zEn)|jh-MHt<5LN0N_$E_aoohM?%r}&wB#fch!cWF^wWL6hpyb>ZuE=pqmB~53-O5f zmAx0cfSrrosS@2P#(u=jnU{e{_eKUI?|6m=2yuO&T)t)eAMu_oR zdXkZn#24~{PUexO)c{LcyUF-lecmu0DEycP6@uo(y?o2`SN$okD0RVvuo*8dre`-w z8_wMdNMUJeO$@#m&e)S_(k6_~g*p{{L74{ML9|3r6Pn&;cD9gOk~Q|^B(-DK?D@GI zn4YqNAeK6?Bq7)(N57Vf_2*?(;CMP;3_M%dMhKC^))<-lPU0-OIgg4lt*0MOa7d8a z_ojLN-sH%Byl2hUWyYj&yHe;j1wv=GCd#E~>liH6{;<8;N zGxulNyRtp~&4-S{MDOxsN3eT}_4A8Y`V!&pB*^!!CmeHVV200Y3A32tXB2+)obj-^` zdzY%Eh6bUWJk)O4S&#i_C)PJd2ByJI8XWkV&7rI&D#}P!h#{k*QUNg#1=qe8)kaG1 z#HbMB9aETLS}IETJ%GSvtR8WL`pCsG{>r->zwV+Aa_p?LZL0OzuE&K$wSTi9cnEMx z@)AhR4sicgfpMc(uA9PTJfUs)nqxkj(iGl1Z2Lqn1}xtc%+fE$uk=tN_IbNbbI;Qk zT-C53AEQyNviS~}5IoU6!b`YBM&Wvg5C++zRi;4hX8`tPJ7~ z$Vbs|mrQ)$6-qIqQyK2x@yxt}jd}eQI9xVJ9vbK-gL~#O9>o}wkJi%f>yr}`FF0%c z*(Xrk4uw3Lze~mxU`4~Z5~LounYzMq2{yIqCYU2Zcu;e2czmMkL&9NoOu7-VMX@=r@p~4d8Kzx* z4?q&bQ@h6ZC}eJY`iJaT>G~c#zHY_ww|XHB`m4>54M)U zGmDXDt>b(Jp}VH=awc^I?b|I8LrEN6%OQaSyrCJH6TAAs3^_ip^HUe4VmrGas_Jl4 zCO}PsUY>e1-;9I&4m;EOSov#@pB_yl`XKXcZn(79c)69z$16{aW^TYgwoLeXbWJW6 zBJCcm>|cvIQ6-WMDyfskw4#eDZR(91>#8;k!8Jo+MWRih`eQ*s81-gco=^a3 zqzSc+dK)I1sr6d@-)+zTFty-qubA5yjJR9Ai+g1T`RTutGwBnO3=FsxR3A+z?OU$y zxb#VXxXJ2Oy~;-RQIml=p1OWUu|7s4 zu_#mi^KzGTQy_4}%XWfB|NNLRat7k_^I&Fu)g61hBlOwW&yRMK=H6xRcJphUp4isS zPwDet8`x5XVPVh_XkS7Ab2dK!XKa4)!oq?rK25)HQG&kVwhkbQ5ka*BVd`@YSAK6k zh^22Fc3OZ)+}2KG-fRO00@(e6m4MzprLbK)IDQvu<5a<15ph-aD5eOJ4bIS)<#<5V9scd(>}Waex{3@TLeJlpw< zk*DyS3C?8rlzV%CKT1AL3#cC9d5J6=cS}^^ru1MwBQVRNggkZ$>XzR7H3IvVk(^qO+qoul$Kks%( zD^FJ0J`(#7puR&41si;NwSUkTC;XoR5bV86rtDRQD{v;s@2}tM8KFXiUGSRs`vkn# z>g56-F!w*)Qnh(<11`7vAU1_UuKISV!nL|wy4X-*M~@pX379|0FSoTQs5?1X2hm!6 z$Fu+R;zIjvx&p2LIiX&snH=Gq-bly3>fq5Y(BZ)jNmdKJJI5{q9RUf}4%i;ok=6yG!oo0g^{@3;4cI?oS44hC7wg%@_YE_F_q}%njHc1VQp542m=su;GE*diz&#snbndbM7#(Pr(SkoN= zfEd#nL+Seqzn{3ytsTcAzJQ?l4J{S; z_uu`1_i|sX09odwA4*o$z^XDhw8aYWdqA=(TmZg^hpL}aVhQ9m2nl9L$s2Amzmx=x zxW|PRC?$}LRC&DitvGp_e%{YC@EV$#Y4|q^acF6-XDC2?cW(5Wg}eiQh?D|-7rwsN zq~2FwPlj0D@CESxuYc7XMO|6gBNdgn;AEjG-2{%*ZdOOIMJswj1f{~eN-IT|4=B%; z{-><0$lEj^X{Id^`{p*yQ}6D7`ol~6g`*}@E-cm-t*p7>AX(-7Ar_tnSFf1nOI`Kf`8{R4jJ3~7-qFPVKz2_msf7v5z}R219 zulD{?%zbl8o!wM43R;_hivQNt7(j2g#2;iqGBU*hKElkjxP8bk<86sKzxg!$Z-AT~ zOZ0c%bmrZkw}xJKSf-~c_N%7kr{bc4+nBf*z+(44B31ZC+h;)Y?zejNd5gC1*$n`R z(f?~rZ73^?PD*0B9!Ed?Q8}T4RQ%7Iprtd_CD?rlc+^L`Fqr3As(X$@Pt}_{H;##U zSTgYlFn#~driN%;`weCaMVB;J7A5%CuUr=;%-5^D4_u8CLl*IRfK4NjhjFvoUoeXl zyujywh@g!ta%cqfp_BX;`o)1qJ*}vyU}0iv^UH`#2cTI(I=5RdS*8D8rmey2wgIf} zhiF#!mFpO2pt}G2mhVUSmLD*HX2bqh1FC@q%_sja8S?)Jf_(*p29&*|oqajSHxO&@8>tQAl zn5J}prm2@&3VK$qE`?1E0O`yK^~VPw$b z@hdm0A5dsufpY%#uYqS){pW!%`I1;obxeE!%e;@i2HopH8ZUpjbZrT`|^%5~6Hk^@&2xi)FCUD|sSf*$K-pegXV-;vr(;6l)B z@IgYYO*WtUVt@mq+vKCLgQFb~R=)Ob1eM=Jqmg&CB z74QC!+oa~)>D@Plq82`|Y*tqo!rT{N32WTM? z8PwQoEk|i9G*$8w8-}j2CsP1v<&#tO{kVmR^hnNF3B6tFGaVAHRU#G%MnS4k21{U zaz&tjU1G>b={S`%(OT(AvxV;r-?xQ36ka)|T^t0X$@yvepIrNh>5<3eSKUV!FW<;6 zKqdo^$NT?g(?dezWll1V1^h-DF6G59j;({FoH`H?lEZb zzxZs6Ew2sACc)FcFzdmWD{o_ph6rX21Gq5(!oAE6gelSZe5~uzN&i^iKm2a`+nouZ z+ykubJYm!W%1__PM4OhD^=tQf(Mx`5`yFYc)x(ld{n% z5@HfySiQZjIR5)A*%fbv(Yu0avagdTUCqhO(sCxr_h@=L=~v%b)2q0M=H{+z{-2dP z3T$6oxr*U><2uAOc%tG3@pjciH{@G@Ce9JrY2dM~U5dAw_+UjnshWrFw-}``nEfb5 zSjVS(>1f06<9Psb?1Q9d^aGgHyOILZt^>$tB`q7W`(OF5c6i!mQI*rpGT$5YcyZ+= zJx1dxlBF5~BShXVh)Au~qxp#g6<#RP#<-5jdyfugC13QzJk_DvZnS{UvL`WW=4|FT zR}ixaK>2S990V|n(l;(p?{FsW|9&fwos9hc7H}M|eJu60 z_1bn$7un=;e@FatKcU$$CF^`QP~vgFM?23u10l}FiVcCr&*BCdx*cgtZZ{z47tZwm z`ogP<-{Nr8&`~@0a9IJnkyV0`+at$J8VQeS*HNQ2fZ|aTnDH>10Bq7B_ojo?4}HQL zA8hMCVux(`Zap_qwoLQ1nDE*te$HNj@|5`b8GlgOct3i6_PFV|k(6Vy_tu$fbDh-_ zgHC+7QQ`Halei~2%HP>-EMDXCD+0&m^V9MFhrRcXYI@uLMYk1fARtYe4Jn}riuB@E zKtfYW2tf!%q=P_^4gxCD6$rg3RUq`AEo%g#P zDmk6Evz7P^*aKs|tR#K~!J za3n0NVi{M^Cb4sCuR!C;o)C#|-#YkPB@$)_-LAm}Z)&GC&4-p6IalC>&7F5*-)S+Z zn1Pd|=Py0bkjhaB|G<4l=$qvk1Fh3OAyr1=vzjKAYrPP+uf8m0uawYGR@XTL7WvW5 z2Ek5V_pPRaFHfS+xt7cqk7M&TJLX<7Sl$O#Y%7l+1;bOX-b{5}!PsO^DYH9!rzdSH zV+el&7os5fg4?RARrqjRV$-f!@< zutb-G2UfPfJ=-;GKRUc*FkN#lD3<|3<$t^$hdCG=-7g}wRT7;qxH3c^Q1F-Cqjp_Y z53OFw;@EN0^B*9W;A!&RVJBjq7QeJkKBHOkRL^NfH9l&xH8$F9$q#uUQ%#2RR#>c< zqfLwVA~;PQu}%aSAGvM?x4iq63sN;x!Z)u z(Y+HT;p#MCSw8=&HOtX?q{2~})>PT4GX!yQly%nKDW^Tj9+9HbVHt~(2kfPyd zgJX{5&fk&tpcV2xYS5xMd94pwy)b)TL;2O6x!DryaZ_qRbDfGRu&by=j-ic5E>+QsWKP>Yowl(S z4LoMl8WWjtW|UgdT8tJOZLLt6U@$GT5QM)y;xJzD?G-~WR~{FnLBQXdUIV4@r8)Pp zSoo$w(Dp+LULQ<^(VI z_@GIQTC#?ggG4#)m8*Fd0)08+lhkaOGAC%W@^+jTjAxJ-s@ch7-?Jq8Y~pQmR3bOp zYxnsckaQohd*^NUHL}g+$&z+^53i^CQ-dYf=<+bciFikMaeRK8MEQrs((LUNdvhSt zK_6_zAEZrW)pBm#%VD2tRCcpr4eC;9yj-K}C`_6*!Gh&v+5heh0h&&0*7)z#O)^Zc z2Q{?q)N|)p4mPp+r4h(|zqRE}p89ce$=#>;>iyYj&aDZH?@NrUY7*Z)$kmqLefJt{ zV$+uvE=4QyWp~Vz)4Z{oc#{@fZuyjV^xBP*7ifYqODChy z+3tI@cl|OaLPnqqQSTfoP=ly0MaUEo59i>=l(=>hXe&PidqwMS4S^mp$%7S1i2G(O zi|^EOix!H^>w&JgIXwukubrr}RT&SWMc5w1A?@P;FuU`&^RRUSTDFXG@+S8HAQVX0)ZY9e`@VMEfI<*-y3Y+ z&NZ(pF@Jh6E?mdA2)4}=hd(CMdL_=Bt;IZU_!B|s9sG12u;TKr2Nn_p)fak4gRbOJyEE`tk!6h z1XyZmwJ+55bG9f5Pu!2+8Jg-m*l|C|p0*V_Srl}KwR&fGG373~_^nvmR^fWI+13OE z|2$#37wnX@sIVoQzgmzgo>}6_dy-qndA`sOGL}&OLfLgjl_UN+b@|)7t@;(=)AGV6 z_gz}dbX5H}D2r`{-aG+VplWa4Ds}qVw|;{lMe9Och1`IWnYqpJAm#Yo2{zTl!FX^U zL9QE_gNnImnV8=cg~s{G5X=Sb-|BW*_NHYy;%1fIx4Xtlr$Tb{4_4%ibCu?-=0@|n z?(Y-=bdln`SSeU^)k%&2{`PKD#4-#f}9j}sDeS+C>{ zdt-eb{HG#lopS(J#$uOBqul3|w|={#za?^n|PJ6O!kj_$8pAml$nhyC)De$$0wt$;>PH%R~fCmlp=ifBIm*J zwz26)^Y!W}v@puQYO@7qm(X)xEOpc6p!#5^dQ36qE%ks7n^uOsp9&QgrBWf#28%Me|$5ugWs3hc^8>20m03i<|fobA1g4$E_yqdKnqVSPTBWrEg!D_0cwpiG);F_)Y%{n$;$E_EJPj!Y~ zL*9g2WK!(JmgjRK{OSg`(?Edb33yD%GmfPa&~F6b># zk0Ba+{a(o!FT$le8diYZdTiN0V_pAHNDI@jWSbjSxC4xEC9yGU{Ub!p`~N8UOyHc#OqLVhO;!{716r zMJ0oy_#S8LU8|WGDMvTzk(B)l@S=KX!D;`t%=e=txlr1GCmHm$}c^}(Lk0o znKbmsV{L@A3v?T6Hru~g+G_$JJyqixj~XfI;11=6EuZ)u?Vvq|Ea+@Ka>u&)0=Ft| zJZ*W!EH(C7`n}oE_o_GJy*fUzO599TAKl$r@upGTYWJ2ph#_#JDlz5m|1 z-)1Wz9$J<2L1@H}MBjbPVGj>bUk*MY26VQgzPnsTX%lN7!$MPZGP_Lcy*e`s%NL>M;?1~O! zdi-+bHp3cU+}+|J@unY!lI0#7i-V*WzzLZA&Qs?vwZDE&0|{Wy+~v=D{K&EPS?GC< zRy@l$KN|9u(C~7EoFs5h+onlPfgRPrjEbVx@%TFGT0_6LB~hDJ?mL4yEfZ2##PazS|&&!~~ZEm|LOEw{zE z^kEU)u`7B+?a|xfxxA(k7L)3Pdbt7_dJ+6Wsc)F**5kaA;9M83EFu5>c!SX~+(#Q6 zn`u4TTW}bqiW|P3p9M^OtN_+H^B{Z7kL#H&hl63sgVbr4KW9`5s z=xJOxu{|A2TV-<>#%0Bqe3TWIG|}bn7%Rp^?a4z%mcxp<${qONsDe2^w-v=o3L+%5 z{}XNdp$)@>mhOy22nS~CGX_WCXzaISw-TgnN|o~Cy;9?xOgr>N1GUFXtfns*;&a~9 z#a{~yAE&!P(k&2lB+<^MKqio;gHHxp&>v`kHRh%R0lKnc?n0(Pf*q=$&S%Xmpr#+7 zL&sgFz>d?dUpNpSC$j8i&2u|vwygt$%O68pUyg~gBO2@aZ(3VYW_~wF7i@rDHRRfH zDRZsXh^NLs)99grlpUSb4_R9Nh|KAn5278sD)s#1@=;GvMiAk{p0a0Ci{@RC+#i?h zq^;hpAM@VZ^r-IOP?AVEwPzTv4YNg0mvJJ~G*IWxvzc^I!((1Z3u@d=bx}uj>{jjO zaxpV-z)Jz=>c^kp?D<~RQcq>qAxumEf@Dg-1>IYDdp1v--{X0mMcvN&-{EZ1H8kG1 z)x;yZ>UMJno&AKXjPFeRCSPt0E@Xmk_g3*m&L>FFeDPBL?6RiP7OpB(>k9P_JzF=} zgWA=)Q@rz(S&UfKW<4+S+-WYi~p=U>y z?|Pw0!^cT7hc0GMyQTE%+}pUh;TU6TqW$BuMuhjK2#anhrVkl_& zGZfT7nT>AD=VF=bO&#YdLbb?J#j=KT^l@$3C3H_f-pS&E`O;P+MAnp<>L+zAa7QL8 z;Y=n#kd8^-BvaV+@kf_t)Gc#WLK(CTrAsLFu>_s}SG9(_^N-9FED_jQH^;BMaDq_bx$~C*J@;HUndhMv&C^fAG!1B; zYrlJx3h}P%&lVPYb`PctuWT3(3q9Zp>+K!JGyZ<^@%6C z#FGmlU)RrE>X&{&GOu3!iM6Dshf@vo>L71+mfwEPRiShN=?53aD?@9b{ttu8ojw#! z=Qx5?sc=2xEX;;h(92avR<1)n$PC^W43DlVyZf?VFJM+@B3pQV#NRk+sS8py!B7(y8 zWHfu*s}0QpNo7X%7VODgjr-?85*y#dS0ACl7pUhiN{KDEmTXY*Ib%8|B$Ogl-*6JCA=8^R%C816MqJ_%%%ucm9dhJ!!z9q;SkcR?b92g^NbS8S#Ni)9D-Bf-?e z{En~c_eHG6_lpkax;@m($>Ezt)jHb$;Q`^?Au*EXh${<`%`b=_>^p(=gjG&#V$Ru9tzXK zoRxr?`J-L(G`YgH`p@!cjJ$SxlPG~U@bi&UePswDAvx;c4Zn_^H13@fmm;@OPV-PlniQT=tg&5A=PP zWc9^9GS-EX3#Qr@EJcb=M56)7vi-98!`Lt z1!4y%FcWUU5Rf@MW3_L|8Tz`mbXT{k%;~8v-i;K73z_lbDKn0chfhFGpUGdY6Dhu* zrVt~P8;sSaFEY1mQE7CwqHPM3d>(d0wMxXg|6@|F{#l)i?<3m{=?rS^aUsnesD%FS zA+w1mnpQHt-(FL?gfx8QQdIY~_HODjNimT-^~xV#xn69EsWX~w=Wx5=vX+6=4qfp> z?j|^V*wy2g$gu76ze~&Ow&`~#Dp5>z@%WgtW=^N3J6M<3EK*jV9f#yrK9L}?l@HZ_ zw>9KjPKnLFn_4o>#$&_j^?JxcgVkOR);NqM)5cJ~m&3$sbzE4n^?N%87qqt)FfWZ@ zStO9Vd_9IHZN94Zd3c{p>-zfTlkY}}>A@Besg;3}V&+7cPt!SemiV2e;q!yM@ohQh zrbJt^;>)YK1AW4oDkNIX^hLo6G$c|Lv9VrU?b`n&n!}*qSG=U_Pmk93rv~X!8x_@& zqsL+5N3=ccrUd$=@7c#N3`5Co`d(k(f=lA55)%AUr;UV>isAadth|acuWh%wx7x<3`ZG`-`!MzOx-_zFe)=?5MMBK9ph)qU9N%k~)ln90@(DmqilNv~A&^R&cwfJL?V!g_nPth7y)-gFk`m+faT$Cm?W~qFx33odvezg@S-^=TNSN-`vf>wM`RwHom)SJ zM4ig-kYS)U#BO+}2_6kx5m{W=Nm^+LuzWw~Yid|_)Xzd-MVPKB{8&HM+S;X|A$Oq7 zeSamvZ*$6yKAxDlGA{ zR<5&JU4WETL)p<3Eme%gTnnr*3&^D$GzF|+n_$&aK)t;R<1BW6$}iM+>ijS*b_pfi z01agt4#9^_)jQUJy&zTmle^66A1Y<1=(xf{2U}{O1E_d|Xs`|VBRgo>6i&6%SVSkz zt<4{7qk9!pO$xgT?D`rhwBr)Ce6v1^u&{+`sF~z;vS?(O34FDCKmwa5W|>pf8~Wfx zG#uK0_`RhA#IYg-`pA`9!@qu3b>ELCDZwn=F#EJlf$_r*&ubfwk|pMk(!#I|X3oX>W%CR_WfwQo0Np_r`^*C>99~NbzzoGxA}C6?W|UTd2TR?R@MZ| z1(NW9ENvU)78>lx>Da%#fI?orMn%mS;*uWAV$$)bVew#Z15H0}9W>J&+gr50y$40A;tkS;3{{UT0*@I2#8Tsk^igME2Bo{hSI1UhL8+AYFt?9-$(shdX8H z@kVw97yN0sJ8(~@yFp@Leihj-fN3`7uo-cjqF#|P%~i3?j$D)6UkaklgyHsr!gjV2 z(x6q(&EbA%cG1a2Kn2xHOe`7TTGXRB44x)44^8(NNtbGK!_Ml&j~X53jW#*K>rKaM zqCjm!>Ye>jK(}EfSYD^PeA0d-*AvS<08Ko)uNra z<(bM3)CnDfHcWLGXIu8Noa>Ty7t*=dbV0Eo$FKM0lK4ZiH)6r}Tx9S(m;BlejK$$r zWnkw5-$e}ITq{eKOGcKN7b`%`sFeY5suk9}#qD!bx{NnvA=R6VjC6E#)_t}c$oeU> z0tKJ2OIH$(GL%?tzsq6Ip;h4PLo)(8y)UMfj147p7&Hp?2DPoR z+k(;$LvHh1W@FTz@mgHDFr@Y7*rp(217i$M65R1!j4&=#QECZ$nAO)(eOPxXPcMRZ zP)B5aEZdSO<_5LK)c^q9MR{LREi4EPp7v0=ME*2BA-ZU@+#dO1V!t9YB1ia^gGXa-XL=^Jh7;LqBLQMOo=N-bV9N5EtG znrcpG6f)5aJ*9!b+~Ry9d~}`~q{KRHh~zDav}AqeBS)J~ug>Hr3}xv`=j55kn(yo# zW}eRwCn$kEnH;Hsc0K0K{Q9Dt^)`e8&f(GKx?o%XJfXv)rkwx+v^-m!!&tkfrTzUw zkdGqWMAnB*X?UVrMIf;s?}Kfhg)cue2ii89NQBQgu1z!Vp{{I0Xf?p!bGJc(Ypy zy`7$i^)*%r@N6a^Nq{KAi+>wsv-9DSdM-ut!&z#4!A-2{4qwaKl;INiA8qhj+6hHm z;I+8@PhXbO4Ludg76~4@>gnP2KxvlkMlhJ0w~q$|M>phlxX|}>xz7AGP@PK~8viMC zlX-dm(hb~?1lI^za#+67G)ktTlly2ra-V-bdT}SoH4J*?M%am;Ym^!PsYaRjJL!2k z!=re)6y4-}-bapZtUrh%|7%}`tgc0_w}7-rLY|d^uIW$fp}?Sa_tgJWQSjmA2aM9L9W~H= zcL=6GYA}0(mcu~<@o|ymqD03Ag%KuMxAU~N4LU*{T=&fccE-j?^2{( zTI+XsKPjpj9mlb7ySCajXrvu)2$tdv6mM!{Ka97F$wY&8RC*&>%LXntgW3Z?ut1G#4^virTZytqc) zTpqbI(AP;J%b4pcZl)YA<$j70s9FgAH(RH(V=vxywgokxBydKc;IM6*xWTlhBj<5a z6!fd-8J_#uE_jHj`mPQBG>j9C!+f#r`-j?csR z&iN2s{BxA!z8DVa6;Pw;3xWsR_msr_t4xk{-%P-|Q z>zhf{O@?yA2?%ul<)Z>X`J_(64Ksz2-#q0{?*WvllJli~RRn=-H}wV2)p3JKN5?C~ zum5?SX_6zvkuzI+kfV}JgF$8DzMS`n0;?jYpKkJ#wHqNHBPcdO2{#HGynQc<-w;~% zZCf%4$Uy@lrzU;=z7ANm`~G;KcmMK0HDHGVz3U!)SHI_lS6=}|f%zdTambASpu3q{!BX)0%6WiY(?YbiyrcA}+oRPvo7%$^vUUP_Ew$8iX^8gs^Z z<3s?!e1TxhAX(*thXj>FZ+=|tLfOuOo-@cSt0bAM>%RT3U5XO845X8ROjN&Ae88eu zyNi_;Su*5u^QV~uBkp)OcUy?Av5R2BVsw-1Pd^qoYC(A(qjvcO#3KEH@8x!*qga-M zDpbu&+Mb~d7VY)Df?6`sZkJ|-Lo52z42&O10nw64MG~|oks;9zW?y=@^dPq<^j7N{exNS8dGoLPk zBE6YOwd;oVH-6@SJ{a{G;zUBSE?c;$fv;b{6y&OK_)482T6~ml@=Kfn^-))@8wL&j z1$`qsJ;Svc)!S%72%Ag7f(^~z&qOq@w}C`lG%}uMU+_u+9 zJ|{TzF3pZh;UABs{v*;Xo&cVV0RGKsboaO^(IB`y(NR6M3P%%FfZS+65^NK_nzFv< z#p2*Dvi!L=8iD=v4|{k3a3CqHY+VPKd77+Qj!I*BHKNbd?;OQ+40uIT2%PYUNtNHj z^7QVcV0daRnPeV85k&kff@t`o{#7T~Kc_L2XsKD1c>J3JX#w44pEpb@n?r#?CBgQ*J<;FXOwxE3@ zlDE4hK)~w(D#a?1cJ%ExDT++t*Of!F)C_ke z5x~}ZG@v>B2vf}7HLy+t*@LXh_kr0gMOmh`u(2h{{ci&l;aWzqg|aTq|8&v^TEzRN zY2OsNp#e-|eoB|MY!C6Kd^bv%xA;M1<+YZ$>RZyERm`G<+zJaiM2?w^+ci)hX*xda z)uYd!O%8mihZaPAZ*&(Ag>mA?#jGk3CY#UirG1J=pUE@G!!G<-_Wp=6L?_mErUlc{ zy>}7Ag;0^(m|Cb>h|a$zzS-WX{|GI^i47H-#5K&4B?a^{-=D<@ZRDg5s9JCD%QM zQH7~EXB{HJ(AAk7B-j;s+XCUVH;W7(ahRYmndwgRYBP;@jYls4a1f(bYUQVaIdKVdmaRxB`%$U=lZu^)2 ze!M~SI79f$=?SJ9sCsd{Cz>w8ZQD!U_FY~QU6MrEQLRXm-xrw8hdB&s^^TdWo!XP! z$7$OS#|Db-R}$>2%bi{-yOz%vDI&L)zeq#E;7=D_<02OTgevHPX(b~MW~LyIilfO- z!)DCRz|v(r(c5XttMkRiYssXgF(cW{ta|xvkQ*0}iIvs|TIDrfCuogsI}MmNaeIru-C2x} z43wD6Bv>45^%flLAKc!I&^3Ic-#rXs7m~jQ_aUR~S%j1-Sbb7T{t+%cQh+mp-*PtN zxVYeHGiz7iKHndYcqS79cW6^19U0Naxp{fRZUP9}ghE_skL@gajiht3>ttu01c7;O zQp-{f$YK2Q?y_2xd^<__bf#5V%d%8DQQ7RJw#V?(+H(Y~JC9L92u9Q93K}~F%I<9BU|gMUGe)aH@Ns4a5wU5M8V?vQr9pCT5?BM0Yv+hkWMm%!%;w9I_m-{;}c=c@P`#v|E7idVf5TuXq zMVeizf4)%}L)5FK9=IKl_-ViPAV_}qIIuLe2mIzTC<5_UvU;-uDUnWAg+SkIi{q1mulPu+6BhIeb z(Wm#EK%>CPsrrDWW5-PThM&nVIVgCdJ@rZtz=A;uFui*1*`o$fsT}?k9VO*gKUuuQ zA^a?Zu@Wub@&rWQwivlFb%$V8YV^=DGtWlbO_2`HohfPe2od2GS?Z_|hBld}G{`Z} zTi9HkXu+n+Q8|ZaxqS6vfdWlTA;f%IBLm-BWejnfP}kw#h!gZkO4E~G;2XH`MNB&bd+`C4MhNw)5 z(;x&7pN8p6ZI>^@Ig6(js%a#85pv;||>B+&j&X zJ5e?K$`bf))PPAU?FPB)uvr&YV!dPRS$+*JE!B-D)@H_}zKo=D_2X(}TTEy}{qx(7 zWfQ@RNgIqCBQ;L}xXR($hzAyS1+JN88UaiZfcl2+CPL$%kpP4QfR_)`s`MnHDQ4;> zyXZ|xs&ITl?@V&Z1>r4gE)q_29Yv>wVDWdRNIFwW?HV^d@(N}~Nlh&)tsa(HJ!8%` zv^IFJ7+NaBY-SjJ4x8H2%W&ViI$X6}3&W+Ecg?=G(AjD8n|Bv*ttg(-%}@Ba5gq5n z=qJr3)pSO9BgsAMB?s_I!1f$`!}vPp`_}pnF56QTHRQ~nJ?9T5#rWhRSokh@qH8$6 zo%G-XYZZM6Zw8z6-;rM`p7d~q%%_!cI_X6 zN^FK`wy|Pf!%RkUKp*|*_qffmXN509X?W%8OV|%g86P&geSe=Utg9I_$@BS$Jeh^5 zBE>re+_7G(iUp|w&TwJj@Pq7O&UG`Bk+=PQmdf+Y2KYty181k2BfLCu4Ip_AlUS`G z=h_I>pF;!PUuyj>gAF_7uxBPa+>vmHgm(D#IVJ+UvE8yu|@+E|U|e zZTfiL$uqF9agHb8eR(|518cR9jv7bu&52^Pd zO6waDjyan?!rkl!eps8YHj8ttgg8xkUu%skx*VybYAKoY->^~$Hv1j*EUKnvx02u# z$&w^eX(Bf2jahXtraICxOwPko#^fwi9Oz?Fcj3L#an7#gel9Ig#-(f;c0<2cz~q1J zfJUu-556UlDjYq*UoJlE(!@v+?%6Sn~xS?$)2k4wPgY~xnuR`L1OLM7&o&qpONN(!! z$>;E^O)d?3qSIgM;+I6e(Rk#t1HG(7`8MO70~V z4=p2sutThWH)~MgzO%XLd-me9Q2i}gn+;uii5#I}NqZBDEQOYDP1j2+&aJGo1-iQo zb_DdtY{~mwn)Gm#?+m%k6pwuxd0rajy8T^tpDosT!Lj}s=pY(jowstoYEg&}t6eu( zpA#lb8;nG0`83N<#qRVJfPNO*AYKMcU9*?Yv!@Gjv#)L$;zC!47au)BZK=~do44yr z)~K9$bLF$}*XOiJZShN`WABaTN|mEPDV$FUx0tisbSECT+e z;oVeM{sw06f)3aAEOl9s36(L_-~@!>K9*%yuKBblkqY%Pho>l96cj3wE4UbmEIgP$-G!D}wT|hVhn28oB8nFEa`JR*KA}FIL540=tYqm6 z5|I_qwamKqOJ=F0i!^4l$&n$#A8N~wap z%>cjE9oCkJ@+4BVNy%-8$VxJpzWrj^>;h_NX;Bfh3G@i8?=NX~B;bc^tD=eqEeG|V zWa_dqZ+<0_jH%iWu#%`6<1Z#k!cvNvZCAezvFvTK5h4usZNIxYUUl0I;=$?X?7Xjm zin)PxEj2`0xbP9Ge-!k}36k*V`#E}|^=^6D1FUhn2B(XD>h&=KggXDm@Sc#LElT+h zi;^Z<6m1Q3>k2l4Z5Gd`oY0iL)^|}5Q!j^Vva6M}{SbT%Ey@B{ISWH(Y3v7NeblMg zNm^YULa>%?I63IC#_sNU`-O3|^;_z9D8l`#DOC$zN5*Sa4M`fwftun5l49s=?Dl22 z&t4@Spjd)yoSF4rK%=mlHz`Ij;g&TZa!#wrV*|Wemet20cQ?$weR@e~V6fsWoA<4m6C89|8QRmF?-jb+ zmtnWFzhgu*>@#1`rDOuSD$@!-f2}Y!wCPfG=WRfIF-c{k_39#*DDh+f{hiK$jwDDw zrQ&|1um*A;^{X||y)yk%M-u^2z#oMa-%JD_jhwv>&To{$_jmZU)@FXf;MCQeJ8Ak8O=uG zOI(0GFK2@j$CK{OY!)x{#d4EgIuH-lkPm8bbQ%3|WFCgVZkwK$$p8Hezj@pS8NzXh z1x5+N1IVP4$0nE=U^9Ij%I-AddtnXJH)C<3fIpZWMMgh&khNoN9fj}*4Odm_*ICrk4; z{gF*l!-*HQsKM1Oafmxl$q~LNYl@S@<2yfLm!GeVIt%SB6-n2gOav{WKQ4T}2G=`i zULKqbC#KDbvR~H+E$PKY&6Iy;{coK9!_ifNqdOIxfG7{3Tsjb}h|IdxE=h^tR*;R} z+WCgn%BXXxfwD))co9?a3h4UV-#nh3su=qko*JSglB-?M;r;hO09c>*BYFyY{=rfS zuw{nDg=5tndzCV1MOn1nW4HIbyG9of6^|+@FkG_wGV!PHuiBh3Q9u21VC0@8K`p20(Q3 z1@`wEa0uf)a@l@lCNRKAbSJXHHQh6<3C~kA7D@{sAx>H*?$w9~EQ5US;;oaN$rzdL=23+bAn zfRt2sd@w+uuPvN_IO3WBz=s3%L@6?PS{El~e9FzO=WK6`}$l!NE|n zu$tx@o`XRH>Cbs{W=zM1tg63k7Hq6jb;T)1y#YW;gNq;G1vI@@pqZbLAHRnZ`Fi5O zYI-ik(j7D)D;dSDz0{8d$BM9OZx8$V0sNanH&~XVn~Vcx8?!H_!|U9WSUO-Q!~xmxn! zFwma2+=uG1a@QJ)&uh3yAa~z%jS|Wg=~XCQ4A`$EvWDMe86%&x#f3-!;a2i752hCOr=q_z^C8P()mS1pFVBVWj{`r;PCVFYNYv)COr> zqAj1by<*+#Jy&NLvqDv0T-LI@@g{D+&WAuhsCa+Sg5T{%L}TQ=sA63c-?b;rE$?xk zpuUb*756AH_w13Q<*UJZP=~01zyn6?z;tAEB1G%Bva*i_uRHy@ZqV1 zuGCbv#mx!Ex(e|<&@QJ;Ociq7636q-4&Jdp4G8AKE2OZB%fAGyBS(KJE^c$feJXt& z9^)o@I=4$tG(DL;mt8T={JaF5B|NqMR#Q&LsoPzS%|vj=CZKErF_VKc}#B%t*Kx3(BE`u-D?O?~OD?`WizV!~svb03j=MU6}m_ZK@AjMX6H)NA!!7hpL zFXJR9JG08?L1iEU&=L+&Wq@h50ev;;w`s2b9qoTiFWA8xtm?p1y99yPe{(wv!}UoZ zG8AIPcq~fRr3S=ExbA8ti4~OuYvp5ZW2BqB9EZ^Et&d{#-)+APg87Cq8fpgY&WG!= zuG@7f3UWVlYz(gF@I*)SYTL)I*7L?hJ_AQ&kGbvkHR|Gl{8ygWc?;4BI(XI0tuiu; zpw9T;rdWzcqV_kjlF8F5n3&^`#4=Mk1?JIC zkZf18W6%NV=$xOJo^D}5@5K!2Z@31>vtg&kSr(TeIK-Q~p`teLY}}dU4QmqRHG~3j zzC}zmII`#fW#u7pAGmX8_7f^3||^bPJP))1u|Y%#Yld`r=?0KA*pfOr6?j= z9u%2QCP@J#NYX9KM}=$ctl=);h>VgAvR8l;t_e>f5L3)Z-DE<#!HHiMtLIXe57)o> zOn_8W>_y-{fhCiZJf)VE_SPW%3m&piH{-Q_!AxcQF;9=VU&1qJ`XpD%&XXz2ngy*G zW<<#?ZWHlxIjWN?15Xl{Ofmq0)f0Zo5-~VF*^9Uh7oJhHV&oz=tZ+66YUb$b4cIhs zcM2nSpu|l({IH_*n2P;#-l*c5|_!%kru+%U8~YWYJ94OpatO@i&RVr zkO#r*vc14rDJwe*u|(d*9Mqv54K+(!nfX+nwDXmksI7@VbWB%QZj1vbqUCENu3?MSD-E zI-ShUC5b(fS55`ZRC7bB&qf4V;w2?5@>^~b!mBq;a;uexp6w)vW+C(x==}=-r6*4& z#!hS_RW!>geX`xflC5~G6pw2M2@YI`gU)F17I>E4x&dN#gU2b zeja<4QG=6}*v>_@3Qy4(x@;#=b=8nmSClXv3ii%#CK zrEy#(h#~E4#w@T6P;=w0oR;QgblHt_UXUZg9|YLPgl}JeJSt~X6dx<82$cUI!7?$O0-RPWn?nsuTtzkUolG0;hM0CePcau@GjbtTN@ zce!q~urvFC!hzQX#ZeGe8jnL1HJBhj(a%4kCRi(onj9-)rMxU0%z(aRR5?lctj8gD z*|{sLAx7$&UjsePNwxfLV^)D4l+al2&nh!be-YxuuVk*-8|SFZF4>HL`b-51*BDRF zcz<73a$inlGU9-Xg?X1jveycfg-ja!cUJO3x{wDpcbEDX-Q6vmdNRm4EUUpL1@hRc zy=vdVMTe>KF8gX{6KiMV`!;_v6cN>P?z36tmSk2>c#B)}no6tn!SdE43oKh z=67!IjDc1?V{|`@{Rdp)DW8jpTWj9yoJG1DbGp9{ntCJ2ukwN=UUbjD;BjN{n2BPn zh(IOi9F*_VcGfQYp5Uj!3HLdwEV;Kr7@R`B0W~U}P!@wsMyKi7&Xj2LV5Q9opZbdT zdJ25C?&r>gW=~akGg~^u=9T~n3|Ccp9I@@@=LNg{+~M3XtYP|z#yG4DJoDiTj+;~5 z9EhCe$m(0%F6Itpo%e}VMCBH3SBp0!$)o0#*7fr<`Ap>pvQ2NJEw`Nki=iYcUP|}0 zYPfA3oZhBvKPoN`n6!0ivxsNm)#+h?&!}4VM60hceRZVzDRUyOr4X>$4h05d&SdJ~ z3JfG2BvH#J9)&U>0fzuW{CekwW5bd~B^PtL&%k=&u7!uvf|S`n=`439^w~`wk~Vw*|_%sxR`Ld$!?8 zr2v|Q4pACnTi7Og*~fj^o{kUu*=*?fb(O@tyZ0t=moLDn)wp=Wn>HrDi$Yy(~!3;ElDaGI|(Q z{7z+P^acrMhkpG%-CF>zr)P(OB)12#1O4@~AwMp=O(X?_zlb!eujx6Iu^5E$LU9wQh5eEgDKE7rpKJ4IFw-Ik^Yb}{u&{PJG=qmJ~~O8 zm}v$R>-_Ba^Tvu`5bUTh1u%EaPsOkwBk%*v@esk>e<$Mqnu*}?LTA+so<9PVv6JGf zPg}M!a0)fJ^kt}X+^<$!erQwA`qC(xs4!|KSKz|mUMTxdPXK=_yZXJh$*nu~+!_Jo zFs|F5`hV7s98yG!KPVz~6=1#}EM5bf`>_il=m`161Br%Bf$7H3#YS+z2`|$50l(Ky zVSd7xW%`_C(;wG^no)3+?SZFB5d>ja~SFrFl7Lxi67lASI+$E zx`&d#;}8Le4BL~_Fp>A^L?LRyb+BtKC}YnLkS;q_Bp01YBcvJ>O*Y2>isbU z$U0CB^hT@L*8VfUMv;%~6vU-hMozsj`j?RVXB2z4aTsS7&EAXu0(JlTLgGClYN2OU z@3H*3knmG%^%yyt5}{{Tz=ltf@+7^p zr@a9^1S=0!)aU;p5B(`gn65m8ojV9)@}GYD^Q--LQ2xyv{dZ9Q|34@=PDYG>u!yPP z&qc|fAdb}NDf6<$T$3)>Knv6Ah0>W&R2w=qDad8wDxsET|EcQiy6HE|yqLX4<^lY9b5wS1reHlOyXHs8Io=v8nUF zQp}dXLrufGVT>%+D_$Gf9PPkm9_$yQJNBH!h0U?X~ zU7;BObLH32|Lkn~E5rhn3oaFC+mZ>AuDpo<6d|`^QaPQq;W9a$cU_|Y(=c!YG7ceS z*2f7lGda$GmYV+_V3aJl6cgCs2f*RSd_-{h_rv`CSF4!-8+-ExfFGN;761KM9Y+AN zzTkOpmilqXe6SkXzrWJ?=|FHP(N=+CkO2Rp?*IFK=clzmwoQ(_2!1SC{0l$&$7~f` z0*LiUrgjD?Y=AY5@!vi>P>Tfe^F9rs|Ty+cl7#qCxAD-R# z&i^xDU@=AQ4g>B%@D8%X7E-_@vA`N=)H{pocYtY2t$QXeRr$cgY+I8FB#vy)#O3xT zU|_LsDO(F12%5g62q}ONE(bRA%YkX@ZsEF}z_jHnW{fMz-v^E=SN*sIBs%gh<8l=% zFtE5Y-bMjOg7iF}V$X*RjqiYIYuDqLV&DM4ES)r5t$}@@JosTLur4PeBYH`$*|ORO zmyL`}cLjf(aJQRe_vayhd3v(;&Z(Fsn5sWo0d~(Y3VOMb34NMWuz$D1FPzQZ}7ZINVYgV9c9zrew8`*$q zlh-M-2h~c1ZeYd>WQY3%gTV_-X;Lkl(C76LRyb?`UPXkQgi$P#0DgNHlpnSR8KXJ=-HCmU|YqZD}c!&#HkvzowDaxL&Ed&Mmh+juqp gj1Z> ${HOME}/.bashrc + + cd examples/platform/sandbox && \ + export MLP_TYPE_BASE_DIR=$(pwd) && \ + echo "export MLP_TYPE_BASE_DIR=${MLP_TYPE_BASE_DIR}" >> ${HOME}/.bashrc + ``` + +### GitHub Configuration + +- Create a [Personal Access Token][personal-access-token] in [GitHub][github]: + + Note: It is recommended to use a [machine user account][machine-user-account] for this but you can use a personal user account just to try this reference architecture. + + **Fine-grained personal access token** + + - Go to https://github.com/settings/tokens and login using your credentials + - Click "Generate new token" >> "Generate new token (Beta)". + - Enter a Token name. + - Select the expiration. + - Select the Resource owner. + - Select All repositories + - Set the following Permissions: + - Repository permissions + - Administration: Read and write + - Content: Read and write + - Click "Generate token" + + **Personal access tokens (classic)** + + - Go to https://github.com/settings/tokens and login using your credentials + - Click "Generate new token" >> "Generate new token (classic)". + - You will be directed to a screen to created the new token. Provide the note and expiration. + - Choose the following two access: + - [x] repo - Full control of private repositories + - [x] delete_repo - Delete repositories + - Click "Generate token" + +- Store the token in a secure file. + + ``` + # Create a secure directory + mkdir -p ${HOME}/secrets/ + chmod go-rwx ${HOME}/secrets + + # Create a secure file + touch ${HOME}/secrets/mlp-github-token + chmod go-rwx ${HOME}/secrets/mlp-github-token + + # Put the token in the secure file using your preferred editor + nano ${HOME}/secrets/mlp-github-token + ``` + +- Set the GitHub environment variables in Cloud Shell + + Replace the following values: + + - `` is the GitHub organization or user namespace to use for the repositories + - `` is the GitHub account to use for authentication + - `` is the email address to use for commit + + ``` + export MLP_GITHUB_ORG="" + export MLP_GITHUB_USER="" + export MLP_GITHUB_EMAIL="" + ``` + +- Set the configuration variables + + ``` + sed -i "s/YOUR_GITHUB_EMAIL/${MLP_GITHUB_EMAIL}/g" ${MLP_TYPE_BASE_DIR}/mlp.auto.tfvars + sed -i "s/YOUR_GITHUB_ORG/${MLP_GITHUB_ORG}/g" ${MLP_TYPE_BASE_DIR}/mlp.auto.tfvars + sed -i "s/YOUR_GITHUB_USER/${MLP_GITHUB_USER}/g" ${MLP_TYPE_BASE_DIR}/mlp.auto.tfvars + ``` + +### Project Configuration + +You only need to complete the section for the option that you have selected. + +#### Bring your own project (BYOP) + +- Set the project environment variables in Cloud Shell + + Replace the following values + + - `` is the ID of your existing Google Cloud project + + ``` + export MLP_PROJECT_ID="" + export MLP_STATE_BUCKET="${MLP_PROJECT_ID}-tf-state" + ``` + +- Set the default `gcloud` project + + ``` + gcloud config set project ${MLP_PROJECT_ID} + ``` + +- Authorize `gcloud` + + ``` + gcloud auth login --activate --no-launch-browser --quiet --update-adc + ``` + +- Create a Cloud Storage bucket to store the Terraform state + + ``` + gcloud storage buckets create gs://${MLP_STATE_BUCKET} --project ${MLP_PROJECT_ID} + ``` + +- Set the configuration variables + + ``` + sed -i "s/YOUR_STATE_BUCKET/${MLP_STATE_BUCKET}/g" ${MLP_TYPE_BASE_DIR}/backend.tf + sed -i "s/YOUR_PROJECT_ID/${MLP_PROJECT_ID}/g" ${MLP_TYPE_BASE_DIR}/mlp.auto.tfvars + ``` + +#### Terraform managed project + +- Set the configuration variables + + ``` + nano ${MLP_BASE_DIR}/terraform/features/initialize/initialize.auto.tfvars + ``` + + ``` + project = { + billing_account_id = "XXXXXX-XXXXXX-XXXXXX" + folder_id = "############" + name = "mlp" + org_id = "############" + } + ``` + + > `project.billing_account_id` the billing account ID + > + > Enter either `project.folder_id` **OR** `project.org_id` + > `project.folder_id` the folder ID + > `project.org_id` the organization ID + +- Authorize `gcloud` + + ``` + gcloud auth login --activate --no-launch-browser --quiet --update-adc + ``` + +- Create a new project + + ``` + cd ${MLP_BASE_DIR}/terraform/features/initialize + terraform init && \ + terraform plan -input=false -out=tfplan && \ + terraform apply -input=false tfplan && \ + rm tfplan && \ + terraform init -force-copy -migrate-state && \ + rm -rf state + ``` + +### Run Terraform + +- Create the resources + + ``` + cd ${MLP_TYPE_BASE_DIR} && \ + terraform init && \ + terraform plan -input=false -var github_token="$(tr --delete '\n' < ${HOME}/secrets/mlp-github-token)" -out=tfplan && \ + terraform apply -input=false tfplan + rm tfplan + ``` + +### Review the resources + +#### GKE clusters and ConfigSync + +- Go to Google Cloud Console, click on the navigation menu and click on Kubernetes Engine > Clusters. You should see one cluster. + +- Go to Google Cloud Console, click on the navigation menu and click on Kubernetes Engine > Config. If you haven't enabled GKE Enterprise in the project earlier, Click `LEARN AND ENABLE` button and then `ENABLE GKE ENTERPRISE`. You should see a RootSync and RepoSync object. + ![configsync](/best-practices/ml-platform/docs/images/configsync.png) + +#### Software installed via RepoSync and RootSync + +Open Cloud Shell to execute the following commands: + +- Store your GKE cluster name in env variable: + + `export GKE_CLUSTER=` + +- Get cluster credentials: + + ``` + gcloud container fleet memberships get-credentials ${GKE_CLUSTER} + ``` + +- Fetch KubeRay operator CRDs + + ``` + kubectl get crd | grep ray + ``` + + The output will be similar to the following: + + ``` + rayclusters.ray.io 2024-02-12T21:19:06Z + rayjobs.ray.io 2024-02-12T21:19:09Z + rayservices.ray.io 2024-02-12T21:19:12Z + ``` + +- Fetch KubeRay operator pod + + ``` + kubectl get pods + ``` + + The output will be similar to the following: + + ``` + NAME READY STATUS RESTARTS AGE + kuberay-operator-56b8d98766-2nvht 1/1 Running 0 6m26s + ``` + +- Check the namespace `ml-team` created: + + ``` + kubectl get ns | grep ml-team + ``` + +- Check the RepoSync object created `ml-team` namespace: + ``` + kubectl get reposync -n ml-team + ``` +- Check the `raycluster` in `ml-team` namespace + + ``` + kubectl get raycluster -n ml-team + ``` + + The output will be similar to the following: + + ``` + NAME DESIRED WORKERS AVAILABLE WORKERS STATUS AGE + ray-cluster-kuberay 1 1 ready 29m + ``` + +- Check the head and worker pods of kuberay in `ml-team` namespace + ``` + kubectl get pods -n ml-team + ``` + The output will be similar to the following: + ``` + NAME READY STATUS RESTARTS AGE + ray-cluster-kuberay-head-sp6dg 2/2 Running 0 3m21s + ray-cluster-kuberay-worker-workergroup-rzpjw 2/2 Running 0 3m21s + ``` + +### Cleanup + +- Destroy the resources + + ``` + cd ${MLP_TYPE_BASE_DIR} && \ + terraform init && \ + terraform destroy -auto-approve -var github_token="$(tr --delete '\n' < ${HOME}/secrets/mlp-github-token)" && \ + rm -rf .terraform .terraform.lock.hcl + ``` + +#### Project + +You only need to complete the section for the option that you have selected. + +##### Bring your own project (BYOP) + +- Delete the project + + ``` + gcloud projects delete ${MLP_PROJECT_ID} + ``` + +#### Terraform managed project + +- Destroy the project + + ``` + cd ${MLP_BASE_DIR}/terraform/features/initialize && \ + TERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F"=" '{print $2}' | xargs) && \ + cp backend.tf.local backend.tf && \ + terraform init -force-copy -lock=false -migrate-state && \ + gsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \ + terraform init && \ + terraform destroy -auto-approve && \ + rm -rf .terraform .terraform.lock.hcl + ``` + +[gitops]: https://about.gitlab.com/topics/gitops/ +[repo-sync]: https://cloud.google.com/anthos-config-management/docs/reference/rootsync-reposync-fields +[root-sync]: https://cloud.google.com/anthos-config-management/docs/reference/rootsync-reposync-fields +[config-sync]: https://cloud.google.com/anthos-config-management/docs/config-sync-overview +[cloud-deploy]: https://cloud.google.com/deploy?hl=en +[terraform]: https://www.terraform.io/ +[gke]: https://cloud.google.com/kubernetes-engine?hl=en +[git]: https://git-scm.com/ +[github]: https://github.com/ +[gcp-project]: https://cloud.google.com/resource-manager/docs/creating-managing-projects +[personal-access-token]: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens +[machine-user-account]: https://docs.github.com/en/get-started/learning-about-github/types-of-github-accounts diff --git a/best-practices/ml-platform/examples/platform/sandbox/backend.tf b/best-practices/ml-platform/examples/platform/sandbox/backend.tf new file mode 100644 index 000000000..959028bb0 --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/backend.tf @@ -0,0 +1,20 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + backend "gcs" { + prefix = "terraform" + bucket = "YOUR_STATE_BUCKET" + } +} diff --git a/best-practices/ml-platform/examples/platform/sandbox/main.tf b/best-practices/ml-platform/examples/platform/sandbox/main.tf new file mode 100644 index 000000000..8da37d4b1 --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/main.tf @@ -0,0 +1,520 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Project +########################################################################## +data "google_project" "environment" { + project_id = var.environment_project_id +} + +resource "google_project_service" "containerfilesystem_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "containerfilesystem.googleapis.com" +} + +resource "google_project_service" "serviceusage_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "serviceusage.googleapis.com" +} + +resource "google_project_service" "project_services-cr" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "cloudresourcemanager.googleapis.com" +} + +resource "google_project_service" "project_services-an" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "anthos.googleapis.com" +} + +resource "google_project_service" "project_services-anc" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "anthosconfigmanagement.googleapis.com" +} + +resource "google_project_service" "project_services-con" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "container.googleapis.com" +} + +resource "google_project_service" "project_services-com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "compute.googleapis.com" +} + +resource "google_project_service" "project_services-gkecon" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "gkeconnect.googleapis.com" +} + +resource "google_project_service" "project_services-gkeh" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "gkehub.googleapis.com" +} + +resource "google_project_service" "project_services-iam" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "iam.googleapis.com" +} + +resource "google_project_service" "project_services-gate" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "connectgateway.googleapis.com" +} + +# +# Networking +########################################################################## +module "create-vpc" { + source = "../../../terraform/modules/network" + + depends_on = [ + google_project_service.project_services-com + ] + + network_name = format("%s-%s", var.network_name, var.environment_name) + project_id = data.google_project.environment.project_id + routing_mode = var.routing_mode + subnet_01_ip = var.subnet_01_ip + subnet_01_name = format("%s-%s", var.subnet_01_name, var.environment_name) + subnet_01_region = var.subnet_01_region + subnet_02_ip = var.subnet_02_ip + subnet_02_name = format("%s-%s", var.subnet_02_name, var.environment_name) + subnet_02_region = var.subnet_02_region +} + +module "cloud-nat" { + source = "../../../terraform/modules/cloud-nat" + + create_router = true + name = format("%s-%s", "nat-for-acm", var.environment_name) + network = module.create-vpc.vpc + project_id = data.google_project.environment.project_id + region = split("/", module.create-vpc.subnet-1)[3] + router = format("%s-%s", "router-for-acm", var.environment_name) +} + +# +# GKE +########################################################################## +resource "google_gke_hub_feature" "configmanagement_acm_feature" { + depends_on = [ + google_project_service.project_services-gkeh, + google_project_service.project_services-anc, + google_project_service.project_services-an, + google_project_service.project_services-com, + google_project_service.project_services-gkecon + ] + + location = "global" + name = "configmanagement" + project = data.google_project.environment.project_id +} + +module "gke" { + source = "../../../terraform/modules/cluster" + + depends_on = [ + google_gke_hub_feature.configmanagement_acm_feature, + google_project_service.project_services-con, + google_project_service.project_services-com + ] + + cluster_name = format("%s-%s", var.cluster_name, var.environment_name) + env = var.environment_name + initial_node_count = 1 + machine_type = "n2-standard-8" + master_auth_networks_ipcidr = var.subnet_01_ip + network = module.create-vpc.vpc + project_id = data.google_project.environment.project_id + region = var.subnet_01_region + remove_default_node_pool = false + subnet = module.create-vpc.subnet-1 + zone = "${var.subnet_01_region}-a" +} + +module "reservation" { + source = "../../../terraform/modules/vm-reservations" + + cluster_name = module.gke.cluster_name + project_id = data.google_project.environment.project_id + zone = "${var.subnet_01_region}-a" +} + +module "node_pool-reserved" { + source = "../../../terraform/modules/node-pools" + + depends_on = [ + module.reservation + ] + + cluster_name = module.gke.cluster_name + node_pool_name = "reservation" + project_id = data.google_project.environment.project_id + region = var.subnet_01_region + reservation_name = module.reservation.reservation_name + resource_type = "reservation" + taints = var.reserved_taints +} + +module "node_pool-ondemand" { + source = "../../../terraform/modules/node-pools" + + depends_on = [ + module.gke + ] + + cluster_name = module.gke.cluster_name + node_pool_name = "ondemand" + project_id = data.google_project.environment.project_id + region = var.subnet_01_region + resource_type = "ondemand" + taints = var.ondemand_taints +} + +module "node_pool-spot" { + source = "../../../terraform/modules/node-pools" + + depends_on = [ + module.gke + ] + + cluster_name = module.gke.cluster_name + node_pool_name = "spot" + project_id = data.google_project.environment.project_id + region = var.subnet_01_region + resource_type = "spot" + taints = var.spot_taints +} + +resource "google_gke_hub_membership" "membership" { + depends_on = [ + google_gke_hub_feature.configmanagement_acm_feature, + google_project_service.project_services-gkeh, + google_project_service.project_services-gkecon + ] + + membership_id = module.gke.cluster_name + project = data.google_project.environment.project_id + + endpoint { + gke_cluster { + resource_link = "//container.googleapis.com/${module.gke.cluster_id}" + } + } +} + +resource "google_gke_hub_feature_membership" "feature_member" { + depends_on = [ + google_project_service.project_services-gkecon, + google_project_service.project_services-gkeh, + google_project_service.project_services-an, + google_project_service.project_services-anc + ] + + feature = "configmanagement" + location = "global" + membership = google_gke_hub_membership.membership.membership_id + project = data.google_project.environment.project_id + + configmanagement { + version = var.config_management_version + + config_sync { + source_format = "unstructured" + + git { + policy_dir = "manifests/clusters" + secret_type = "token" + sync_branch = github_branch.environment.branch + sync_repo = github_repository.acm_repo.http_clone_url + } + } + + policy_controller { + enabled = true + referential_rules_enabled = true + template_library_installed = true + + } + } +} + +# +# Git Repository +########################################################################## +# data "github_organization" "default" { +# name = var.github_org +# } + +resource "github_repository" "acm_repo" { + # depends_on = [ + # data.github_organization.default + # ] + + allow_merge_commit = true + allow_rebase_merge = true + allow_squash_merge = true + auto_init = true + delete_branch_on_merge = false + description = "Repo for Config Sync" + has_issues = false + has_projects = false + has_wiki = false + name = var.configsync_repo_name + visibility = "private" + vulnerability_alerts = true +} + +resource "github_branch" "environment" { + branch = var.environment_name + repository = github_repository.acm_repo.name +} + +resource "github_branch_default" "environment" { + branch = github_branch.environment.branch + repository = github_repository.acm_repo.name +} + +resource "github_branch_protection_v3" "environment" { + repository = github_repository.acm_repo.name + branch = github_branch.environment.branch + + required_pull_request_reviews { + require_code_owner_reviews = true + required_approving_review_count = 1 + } + + restrictions { + } +} + +# +# Scripts +########################################################################## +resource "null_resource" "create_cluster_yamls" { + depends_on = [ + google_gke_hub_feature_membership.feature_member + ] + + provisioner "local-exec" { + command = "${path.module}/scripts/create_cluster_yamls.sh ${var.github_org} ${github_repository.acm_repo.full_name} ${var.github_user} ${var.github_email} ${var.environment_name} ${module.gke.cluster_name}" + environment = { + GIT_TOKEN = var.github_token + } + } + + triggers = { + md5_files = md5(join("", [for f in fileset("${path.module}/templates/acm-template", "**") : md5("${path.module}/templates/acm-template/${f}")])) + md5_script = filemd5("${path.module}/scripts/create_cluster_yamls.sh") + } +} + +resource "null_resource" "create_git_cred_cms" { + depends_on = [ + google_gke_hub_feature_membership.feature_member, + module.gke, + module.node_pool-reserved, + module.node_pool-ondemand, + module.node_pool-spot, + module.cloud-nat + ] + + provisioner "local-exec" { + command = "${path.module}/scripts/create_git_cred.sh ${module.gke.cluster_name} ${data.google_project.environment.project_id} ${var.github_user} config-management-system" + environment = { + GIT_TOKEN = var.github_token + } + } + + triggers = { + md5_credentials = md5(join("", [var.github_user, var.github_token])) + md5_script = filemd5("${path.module}/scripts/create_git_cred.sh") + } +} + +resource "null_resource" "install_kuberay_operator" { + depends_on = [ + google_gke_hub_feature_membership.feature_member, + null_resource.create_git_cred_cms + ] + + provisioner "local-exec" { + command = "${path.module}/scripts/install_kuberay_operator.sh ${github_repository.acm_repo.full_name} ${var.github_email} ${var.github_org} ${var.github_user}" + environment = { + GIT_TOKEN = var.github_token + } + } + + triggers = { + md5_files = md5(join("", [for f in fileset("${path.module}/templates/acm-template/templates/_cluster_template/kuberay", "**") : md5("${path.module}/templates/acm-template/templates/_cluster_template/kuberay/${f}")])) + md5_script = filemd5("${path.module}/scripts/install_kuberay_operator.sh") + } +} + +locals { + namespace_default_kubernetes_service_account = "default" +} + +resource "google_service_account" "namespace_default" { + account_id = "wi-${var.namespace}-${local.namespace_default_kubernetes_service_account}" + display_name = "${var.namespace}/${local.namespace_default_kubernetes_service_account} workload identity service account" + project = data.google_project.environment.project_id +} + +resource "google_service_account_iam_member" "namespace_default_iam_workload_identity_user" { + depends_on = [ + module.gke + ] + + member = "serviceAccount:${data.google_project.environment.project_id}.svc.id.goog[${var.namespace}/${local.namespace_default_kubernetes_service_account}]" + role = "roles/iam.workloadIdentityUser" + service_account_id = google_service_account.namespace_default.id +} + +resource "null_resource" "create_namespace" { + depends_on = [ + google_gke_hub_feature_membership.feature_member, + null_resource.install_kuberay_operator + ] + + provisioner "local-exec" { + command = "${path.module}/scripts/create_namespace.sh ${github_repository.acm_repo.full_name} ${var.github_email} ${var.github_org} ${var.github_user} ${var.namespace} ${var.environment_name}" + environment = { + GIT_TOKEN = var.github_token + } + } + + triggers = { + md5_files = md5(join("", [for f in fileset("${path.module}/templates/acm-template/templates/_cluster_template/team", "**") : md5("${path.module}/templates/acm-template/templates/_cluster_template/team/${f}")])) + md5_script = filemd5("${path.module}/scripts/create_namespace.sh") + } +} + +resource "null_resource" "create_git_cred_ns" { + depends_on = [ + google_gke_hub_feature_membership.feature_member, + null_resource.create_namespace + ] + + provisioner "local-exec" { + command = "${path.module}/scripts/create_git_cred.sh ${module.gke.cluster_name} ${module.gke.gke_project_id} ${var.github_user} ${var.namespace}" + environment = { + GIT_TOKEN = var.github_token + } + } + + triggers = { + md5_credentials = md5(join("", [var.github_user, var.github_token])) + md5_script = filemd5("${path.module}/scripts/create_git_cred.sh") + } +} + +locals { + ray_head_kubernetes_service_account = "ray-head" + ray_worker_kubernetes_service_account = "ray-worker" +} + +resource "google_service_account" "namespace_ray_head" { + account_id = "wi-${var.namespace}-${local.ray_head_kubernetes_service_account}" + display_name = "${var.namespace}/${local.ray_head_kubernetes_service_account} workload identity service account" + project = data.google_project.environment.project_id +} + +resource "google_service_account_iam_member" "namespace_ray_head_iam_workload_identity_user" { + depends_on = [ + module.gke + ] + + member = "serviceAccount:${data.google_project.environment.project_id}.svc.id.goog[${var.namespace}/${local.ray_head_kubernetes_service_account}]" + role = "roles/iam.workloadIdentityUser" + service_account_id = google_service_account.namespace_ray_head.id +} + +resource "google_service_account" "namespace_ray_worker" { + account_id = "wi-${var.namespace}-${local.ray_worker_kubernetes_service_account}" + display_name = "${var.namespace}/${local.ray_worker_kubernetes_service_account} workload identity service account" + project = data.google_project.environment.project_id +} + +resource "google_service_account_iam_member" "namespace_ray_worker_iam_workload_identity_user" { + depends_on = [ + module.gke + ] + + member = "serviceAccount:${data.google_project.environment.project_id}.svc.id.goog[${var.namespace}/${local.ray_worker_kubernetes_service_account}]" + role = "roles/iam.workloadIdentityUser" + service_account_id = google_service_account.namespace_ray_worker.id +} + +resource "null_resource" "install_ray_cluster" { + depends_on = [ + google_gke_hub_feature_membership.feature_member, + null_resource.create_git_cred_ns + ] + + provisioner "local-exec" { + command = "${path.module}/scripts/install_ray_cluster.sh ${github_repository.acm_repo.full_name} ${var.github_email} ${var.github_org} ${var.github_user} ${var.namespace} ${google_service_account.namespace_ray_head.email} ${local.ray_head_kubernetes_service_account} ${google_service_account.namespace_ray_worker.email} ${local.ray_worker_kubernetes_service_account}" + environment = { + GIT_TOKEN = var.github_token + } + } + + triggers = { + md5_files = md5(join("", [for f in fileset("${path.module}/templates/acm-template/templates/_namespace_template/app", "**") : md5("${path.module}/templates/acm-template/templates/_namespace_template/app/${f}")])) + md5_script = filemd5("${path.module}/scripts/install_ray_cluster.sh") + } +} + +resource "null_resource" "manage_ray_ns" { + depends_on = [ + google_gke_hub_feature_membership.feature_member, + null_resource.create_git_cred_ns, + null_resource.install_ray_cluster + ] + + provisioner "local-exec" { + command = "${path.module}/scripts/manage_ray_ns.sh ${github_repository.acm_repo.full_name} ${var.github_email} ${var.github_org} ${var.github_user} ${var.namespace}" + environment = { + GIT_TOKEN = var.github_token + } + } + + triggers = { + md5_script = filemd5("${path.module}/scripts/manage_ray_ns.sh") + } +} diff --git a/best-practices/ml-platform/examples/platform/sandbox/mlp.auto.tfvars b/best-practices/ml-platform/examples/platform/sandbox/mlp.auto.tfvars new file mode 100644 index 000000000..a20c54ee0 --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/mlp.auto.tfvars @@ -0,0 +1,5 @@ +environment_name = "dev" +environment_project_id = "YOUR_PROJECT_ID" +github_email = "YOUR_GITHUB_EMAIL" +github_org = "YOUR_GITHUB_ORG" +github_user = "YOUR_GITHUB_USER" diff --git a/best-practices/ml-platform/examples/platform/sandbox/outputs.tf b/best-practices/ml-platform/examples/platform/sandbox/outputs.tf new file mode 100644 index 000000000..633bb7f1e --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/outputs.tf @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/best-practices/ml-platform/examples/platform/sandbox/scripts/create_cluster_yamls.sh b/best-practices/ml-platform/examples/platform/sandbox/scripts/create_cluster_yamls.sh new file mode 100755 index 000000000..36cece324 --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/scripts/create_cluster_yamls.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +github_org=${1} +acm_repo_name=${2} +github_user=${3} +github_email=${4} +cluster_env=${5} +cluster_name=${6} + +random=$( + echo $RANDOM | md5sum | head -c 20 + echo +) +log="$(pwd)/log" +flag=0 + +download_acm_repo_name="/tmp/$(echo ${acm_repo_name} | awk -F "/" '{print $2}')-${random}" +git config --global user.name ${github_user} +git config --global user.email ${github_emai} +git clone https://${github_user}:${GIT_TOKEN}@github.com/${acm_repo_name} ${download_acm_repo_name} || exit 1 + +if [ ! -d "${download_acm_repo_name}/manifests" ] && [ ! -d "${download_acm_repo_name}/templates" ]; then + echo "copying files" + cp -r templates/acm-template/* ${download_acm_repo_name} + flag=1 +fi + +cd ${download_acm_repo_name}/manifests/clusters +if [ "${flag}" -eq 0 ]; then + echo "not copying files" +fi + +cp ../../templates/_cluster_template/cluster.yaml ./${cluster_name}-cluster.yaml +cp ../../templates/_cluster_template/selector.yaml ./${cluster_env}-selector.yaml + +find . -type f -name ${cluster_name}-cluster.yaml -exec sed -i "s/CLUSTER_NAME/${cluster_name}/g" {} + +find . -type f -name ${cluster_name}-cluster.yaml -exec sed -i "s/ENV/${cluster_env}/g" {} + +find . -type f -name ${cluster_env}-selector.yaml -exec sed -i "s/ENV/${cluster_env}/g" {} + + +git add ../../. +git config --global user.name ${github_user} +git config --global user.email ${github_email} +git commit -m "Adding ${cluster_name} cluster to the ${cluster_env} environment." +git push origin + +rm -rf ${download_acm_repo_name} diff --git a/best-practices/ml-platform/examples/platform/sandbox/scripts/create_git_cred.sh b/best-practices/ml-platform/examples/platform/sandbox/scripts/create_git_cred.sh new file mode 100755 index 000000000..3c9711558 --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/scripts/create_git_cred.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +gke_cluster=${1} +project_id=${2} +git_user=${3} +namespace=${4} + +gcloud container fleet memberships get-credentials ${gke_cluster} --project ${project_id} + +echo "Waiting for namespace '${namespace}' to be created..." +while ! kubectl get ns ${namespace} >/dev/null 2>&1; do + sleep 2 +done + +if kubectl get secret git-creds -n ${namespace} >/dev/null 2>&1; then + kubectl create secret generic git-creds --namespace="${namespace}" --save-config --dry-run=client --from-literal=username="${git_user}" --from-literal=token="${GIT_TOKEN}" -o yaml | kubectl apply -f - +else + kubectl create secret generic git-creds --namespace="${namespace}" --save-config --from-literal=username="${git_user}" --from-literal=token="${GIT_TOKEN}" +fi diff --git a/best-practices/ml-platform/examples/platform/sandbox/scripts/create_namespace.sh b/best-practices/ml-platform/examples/platform/sandbox/scripts/create_namespace.sh new file mode 100755 index 000000000..938c62b4e --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/scripts/create_namespace.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +configsync_repo_name=${1} +github_email=${2} +github_org=${3} +github_user=${4} +namespace=${5} +cluster_env=${6} + +logfile=$(pwd)/log +random=$( + echo $RANDOM | md5sum | head -c 20 + echo +) +download_acm_repo_name="/tmp/$(echo ${configsync_repo_name} | awk -F "/" '{print $2}')-${random}" +git config --global user.name ${github_user} +git config --global user.email ${github_emai} +git clone https://${github_user}:${GIT_TOKEN}@github.com/${configsync_repo_name} ${download_acm_repo_name} || exit 1 +cd ${download_acm_repo_name}/manifests/clusters + +if [ -d "${namespace}" ]; then + exit 0 +fi + +#TODO: This most likely needs to be fixed for multiple environments +chars_in_namespace=$(echo -n ${namespace} | wc -c) +chars_in_cluster_env=$(echo -n ${cluster_env} | wc -c) +chars_in_reposync_name=$(expr $chars_in_namespace + ${chars_in_cluster_env} + 1) +mkdir ${namespace} || exit 1 +cp -r ../../templates/_cluster_template/team/* ${namespace} +sed -i "s?NAMESPACE?$namespace?g" ${namespace}/* +sed -ni '/#END OF SINGLE ENV DECLARATION/q;p' ${namespace}/reposync.yaml +sed -i "s?ENV?$cluster_env?g" ${namespace}/reposync.yaml +sed -i "s?GIT_REPO?https://github.com/$configsync_repo_name?g" ${namespace}/reposync.yaml +sed -i "s??$chars_in_reposync_name?g" ${namespace}/reposync.yaml + +mkdir ../apps/${namespace} +touch ../apps/${namespace}/.gitkeep + +cat <>kustomization.yaml +- ./${namespace} +EOF +cd .. +git add . +git config --global user.name ${github_user} +git config --global user.email ${github_email} +git commit -m "Adding manifests to create a new namespace." +git push origin + +rm -rf ${download_acm_repo_name} diff --git a/best-practices/ml-platform/examples/platform/sandbox/scripts/install_kuberay_operator.sh b/best-practices/ml-platform/examples/platform/sandbox/scripts/install_kuberay_operator.sh new file mode 100755 index 000000000..11f0397aa --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/scripts/install_kuberay_operator.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +configsync_repo_name=${1} +github_email=${2} +github_org=${3} +github_user=${4} + +random=$( + echo $RANDOM | md5sum | head -c 20 + echo +) +download_acm_repo_name="/tmp/$(echo ${configsync_repo_name} | awk -F "/" '{print $2}')-${random}" +git config --global user.name ${github_user} +git config --global user.email ${github_emai} +git clone https://${github_user}:${GIT_TOKEN}@github.com/${configsync_repo_name} ${download_acm_repo_name} || exit 1 +cd ${download_acm_repo_name}/manifests/clusters +if [ -f "kustomization.yaml" ]; then + exit 0 +fi + +yamlfiles=$(find . -type f -name "*.yaml") +cp ../../templates/_cluster_template/kustomization.yaml . +for yamlfile in $(echo ${yamlfiles}); do + cat <>kustomization.yaml + +- ${yamlfile} +EOF +done + +cp -r ../../templates/_cluster_template/kuberay . +git add . +git config --global user.name ${github_user} +git config --global user.email ${github_email} +git commit -m "Adding manifests to install kuberay operator." +git push origin + +rm -rf ${download_acm_repo_name} diff --git a/best-practices/ml-platform/examples/platform/sandbox/scripts/install_ray_cluster.sh b/best-practices/ml-platform/examples/platform/sandbox/scripts/install_ray_cluster.sh new file mode 100755 index 000000000..3e95bf170 --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/scripts/install_ray_cluster.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +configsync_repo_name=${1} +github_email=${2} +github_org=${3} +github_user=${4} +namespace=${5} +google_service_account_head=${6} +kubernetes_service_account_head=${7} +google_service_account_worker=${8} +kubernetes_service_account_worker=${9} + +random=$( + echo $RANDOM | md5sum | head -c 20 + echo +) +download_acm_repo_name="/tmp/$(echo ${configsync_repo_name} | awk -F "/" '{print $2}')-${random}" +git config --global user.name ${github_user} +git config --global user.email ${github_emai} +git clone https://${github_user}:${GIT_TOKEN}@github.com/${configsync_repo_name} ${download_acm_repo_name} || exit 1 +cd ${download_acm_repo_name}/manifests/apps +if [ ! -d "${namespace}" ]; then + echo "${namespace} folder doesnt exist in the configsync repo" + exit 1 +fi + +if [ -f "${namespace}/kustomization.yaml" ]; then + echo "${namespace} is already set up" + exit 0 +fi + +cp -r ../../templates/_namespace_template/app/* ${namespace}/ +sed -i "s?NAMESPACE?${namespace}?g" ${namespace}/* +sed -i "s?GOOGLE_SERVICE_ACCOUNT_RAY_HEAD?$google_service_account_head?g" ${namespace}/* +sed -i "s?KUBERNETES_SERVICE_ACCOUNT_RAY_HEAD?$kubernetes_service_account_head?g" ${namespace}/* +sed -i "s?GOOGLE_SERVICE_ACCOUNT_RAY_WORKER?$google_service_account_worker?g" ${namespace}/* +sed -i "s?KUBERNETES_SERVICE_ACCOUNT_RAY_WORKER?$kubernetes_service_account_worker?g" ${namespace}/* + +git add . +git config --global user.name ${github_user} +git config --global user.email ${github_email} +git commit -m "Installing ray cluster in ${namespace} namespace." +git push origin + +rm -rf ${download_acm_repo_name} diff --git a/best-practices/ml-platform/examples/platform/sandbox/scripts/manage_ray_ns.sh b/best-practices/ml-platform/examples/platform/sandbox/scripts/manage_ray_ns.sh new file mode 100755 index 000000000..a1ca8b2cb --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/scripts/manage_ray_ns.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +configsync_repo_name=${1} +github_email=${2} +github_org=${3} +github_user=${4} +namespace=${5} + +random=$( + echo $RANDOM | md5sum | head -c 20 + echo +) +download_acm_repo_name="/tmp/$(echo ${configsync_repo_name} | awk -F "/" '{print $2}')-${random}" +git config --global user.name ${github_user} +git config --global user.email ${github_emai} +git clone https://${github_user}:${GIT_TOKEN}@github.com/${configsync_repo_name} ${download_acm_repo_name} || exit 1 +cd ${download_acm_repo_name}/manifests/clusters/kuberay +ns_exists=$(grep ${namespace} values.yaml | wc -l) +if [ "${ns_exists}" -ne 0 ]; then + echo "namespace already present in values.yaml" + exit 0 +fi + +sed -i "s/watchNamespace:/watchNamespace:\n - ${namespace}/g" values.yaml + +git add . +git config --global user.name ${github_user} +git config --global user.email ${github_email} +git commit -m "Installing ray cluster in ${namespace} namespace." +git push origin + +rm -rf ${download_acm_repo_name} diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/manifests/apps/.gitkeep b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/manifests/apps/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/manifests/clusters/.gitkeep b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/manifests/clusters/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/cluster.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/cluster.yaml new file mode 100644 index 000000000..c27d6a578 --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/cluster.yaml @@ -0,0 +1,21 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: Cluster +apiVersion: clusterregistry.k8s.io/v1alpha1 +metadata: + name: CLUSTER_NAME + labels: + environment: ENV + clusterName: CLUSTER_NAME \ No newline at end of file diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/config-selector.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/config-selector.yaml new file mode 100644 index 000000000..3f22b4d64 --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/config-selector.yaml @@ -0,0 +1,23 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: ClusterSelector +apiVersion: configmanagement.gke.io/v1 +metadata: + name: config +spec: + selector: + matchLabels: + clusterName: CLUSTER_NAME + diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/kustomization.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/kustomization.yaml new file mode 100644 index 000000000..cc63d55d2 --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/kustomization.yaml @@ -0,0 +1,30 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- rbac.yaml +patches: +- path: rayclusters.yaml +- path: rayservices.yaml +- path: rayjobs.yaml + +helmCharts: +- name: kuberay-operator + repo: https://ray-project.github.io/kuberay-helm/ + version: 1.0.0 + releaseName: kuberay-operator + includeCRDs: true + valuesFile: values.yaml diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rayclusters.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rayclusters.yaml new file mode 100644 index 000000000..d552cc2d9 --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rayclusters.yaml @@ -0,0 +1,22 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: rayclusters.ray.io + annotations: + controller-gen.kubebuilder.io/version: v0.6.0 +status: + $patch: delete \ No newline at end of file diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rayjobs.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rayjobs.yaml new file mode 100644 index 000000000..c18a0c21b --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rayjobs.yaml @@ -0,0 +1,22 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: rayjobs.ray.io + annotations: + controller-gen.kubebuilder.io/version: v0.6.0 +status: + $patch: delete \ No newline at end of file diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rayservices.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rayservices.yaml new file mode 100644 index 000000000..4e10d8ab6 --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rayservices.yaml @@ -0,0 +1,22 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: rayservices.ray.io + annotations: + controller-gen.kubebuilder.io/version: v0.6.0 +status: + $patch: delete \ No newline at end of file diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rbac.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rbac.yaml new file mode 100644 index 000000000..a0a1a686d --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rbac.yaml @@ -0,0 +1,44 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kuberay-operator-role +rules: +- apiGroups: + - '*' + resources: + - '*' + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kuberay-operator-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kuberay-operator-role +subjects: +- kind: ServiceAccount + name: kuberay-operator + namespace: default diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/values.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/values.yaml new file mode 100644 index 000000000..626a6cb2a --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/values.yaml @@ -0,0 +1,112 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +image: + repository: kuberay/operator + tag: v1.0.0 + pullPolicy: IfNotPresent + +nameOverride: "kuberay-operator" +fullnameOverride: "kuberay-operator" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "kuberay-operator" + +service: + type: ClusterIP + port: 8080 + +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do whelm to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: + cpu: 100m + # Anecdotally, managing 500 Ray pods requires roughly 500MB memory. + # Monitor memory usage and adjust as needed. + memory: 512Mi + # requests: + # cpu: 100m + # memory: 512Mi + +livenessProbe: + initialDelaySeconds: 10 + periodSeconds: 5 + failureThreshold: 5 + +readinessProbe: + initialDelaySeconds: 10 + periodSeconds: 5 + failureThreshold: 5 + +batchScheduler: + enabled: false + + # Set up `securityContext` to improve Pod security. + # See https://github.com/ray-project/kuberay/blob/master/docs/guidance/pod-security.md for further guidance. +securityContext: {} + + + # If rbacEnable is set to false, no RBAC resources will be created, including the Role for leader election, the Role for Pods and Services, and so on. +rbacEnable: true + + # When crNamespacedRbacEnable is set to true, the KubeRay operator will create a Role for RayCluster preparation (e.g., Pods, Services) + # and a corresponding RoleBinding for each namespace listed in the "watchNamespace" parameter. Please note that even if crNamespacedRbacEnable + # is set to false, the Role and RoleBinding for leader election will still be created. + # + # Note: + # (1) This variable is only effective when rbacEnable and singleNamespaceInstall are both set to true. + # (2) In most cases, it should be set to true, unless you are using a Kubernetes cluster managed by GitOps tools such as ArgoCD. +crNamespacedRbacEnable: true + + # When singleNamespaceInstall is true: + # - Install namespaced RBAC resources such as Role and RoleBinding instead of cluster-scoped ones like ClusterRole and ClusterRoleBinding so that + # the chart can be installed by users with permissions restricted to a single namespace. + # (Please note that this excludes the CRDs, which can only be installed at the cluster scope.) + # - If "watchNamespace" is not set, the KubeRay operator will, by default, only listen + # to resource events within its own namespace. +singleNamespaceInstall: true + +# The KubeRay operator will watch the custom resources in the namespaces listed in the "watchNamespace" parameter. +watchNamespace: + + +# Environment variables +env: +# If not set or set to true, kuberay auto injects an init container waiting for ray GCS. +# If false, you will need to inject your own init container to ensure ray GCS is up before the ray workers start. +# Warning: we highly recommend setting to true and let kuberay handle for you. +# - name: ENABLE_INIT_CONTAINER_INJECTION +# value: "true" +# If not set or set to "", kuberay will pick up the default k8s cluster domain `cluster.local` +# Otherwise, kuberay will use your custom domain +# - name: CLUSTER_DOMAIN +# value: "" +# If not set or set to false, when running on OpenShift with Ingress creation enabled, kuberay will create OpenShift route +# Otherwise, regardless of the type of cluster with Ingress creation enabled, kuberay will create Ingress +# - name: USE_INGRESS_ON_OPENSHIFT +# value: "true" +# Unconditionally requeue after the number of seconds specified in the +# environment variable RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV. If the +# environment variable is not set, requeue after the default value (300). +# - name: RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV +# value: 300 +# If not set or set to "true", KubeRay will clean up the Redis storage namespace when a GCS FT-enabled RayCluster is deleted. +# - name: ENABLE_GCS_FT_REDIS_CLEANUP +# value: "true" \ No newline at end of file diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kustomization.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kustomization.yaml new file mode 100644 index 000000000..448f68961 --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kustomization.yaml @@ -0,0 +1,19 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/master/nvidia-driver-installer/cos/daemonset-preloaded-latest.yaml" +- ./kuberay \ No newline at end of file diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/selector.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/selector.yaml new file mode 100644 index 000000000..cfd6f6ede --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/selector.yaml @@ -0,0 +1,22 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: ClusterSelector +apiVersion: configmanagement.gke.io/v1 +metadata: + name: ENV +spec: + selector: + matchLabels: + environment: ENV \ No newline at end of file diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/kustomization.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/kustomization.yaml new file mode 100644 index 000000000..93f6f77e9 --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/kustomization.yaml @@ -0,0 +1,22 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- namespace.yaml +- network-policy.yaml +- rbac.yaml +- reposync.yaml \ No newline at end of file diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/namespace.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/namespace.yaml new file mode 100644 index 000000000..08474cb90 --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/namespace.yaml @@ -0,0 +1,20 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: NAMESPACE + labels: + app: NAMESPACE \ No newline at end of file diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/network-policy.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/network-policy.yaml new file mode 100644 index 000000000..de02d2a5a --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/network-policy.yaml @@ -0,0 +1,32 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: deny-from-other-namespaces + namespace: NAMESPACE +spec: + podSelector: + # Apply policy to all pods in this namespace. + matchLabels: {} + ingress: + - from: + # Allow traffic between all pods in this namespace. + - podSelector: {} + # Example that allows traffic from another app's namespace. + #- from: + # - namespaceSelector: + # matchLabels: + # app: another-app-namespace \ No newline at end of file diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/rbac.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/rbac.yaml new file mode 100644 index 000000000..398a617bb --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/rbac.yaml @@ -0,0 +1,59 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: NAMESPACE-FullAccess + namespace: NAMESPACE +rules: +- apiGroups: + - '*' + resources: + - '*' + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: NAMESPACE-user-access + namespace: NAMESPACE +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: NAMESPACE-FullAccess +subjects: +- kind: User + name: USERNAME1 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kuberay-sa-access + namespace: NAMESPACE +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: NAMESPACE-FullAccess +subjects: +- kind: ServiceAccount + name: kuberay-operator + namespace: default \ No newline at end of file diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/reposync.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/reposync.yaml new file mode 100644 index 000000000..4b50dcee5 --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/reposync.yaml @@ -0,0 +1,172 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#ROOT_SOURCE/namespaces/NAMESPACE/repo-sync.yaml +apiVersion: configsync.gke.io/v1beta1 +kind: RepoSync +metadata: + name: ENV-NAMESPACE + namespace: NAMESPACE + annotations: + configmanagement.gke.io/cluster-selector: ENV +spec: + sourceType: git + # Since this is for a namespace repository, the format is unstructured + sourceFormat: unstructured + git: + repo: "GIT_REPO" + revision: "ENV" + #branch: NAMESPACE_BRANCH + dir: "manifests/apps/NAMESPACE" + auth: token + secretRef: + name: git-creds +--- +#ROOT_REPO/namespaces/NAMESPACE/sync-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ENV-rb-NAMESPACE + namespace: NAMESPACE + annotations: + configmanagement.gke.io/cluster-selector: ENV +subjects: +- kind: ServiceAccount + name: ns-reconciler-NAMESPACE-ENV-NAMESPACE- + namespace: config-management-system +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +--- +#END OF SINGLE ENV DECLARATION +apiVersion: configsync.gke.io/v1beta1 +kind: RepoSync +metadata: + name: dev-NAMESPACE + namespace: NAMESPACE + annotations: + configmanagement.gke.io/cluster-selector: dev +spec: + sourceType: git + # Since this is for a namespace repository, the format is unstructured + sourceFormat: unstructured + git: + repo: "GIT_REPO" + revision: "dev" + #branch: NAMESPACE_BRANCH + dir: "manifests/apps/NAMESPACE" + auth: token + secretRef: + name: git-creds +--- +#ROOT_REPO/namespaces/NAMESPACE/sync-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: dev-rb-NAMESPACE + namespace: NAMESPACE + annotations: + configmanagement.gke.io/cluster-selector: dev +subjects: +- kind: ServiceAccount + name: ns-reconciler-NAMESPACE-dev-NAMESPACE- + namespace: config-management-system +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +--- +#ROOT_SOURCE/namespaces/NAMESPACE/repo-sync.yaml +apiVersion: configsync.gke.io/v1beta1 +kind: RepoSync +metadata: + name: staging-NAMESPACE + namespace: NAMESPACE + annotations: + configmanagement.gke.io/cluster-selector: staging +spec: + sourceType: git + # Since this is for a namespace repository, the format is unstructured + sourceFormat: unstructured + git: + repo: "GIT_REPO" + revision: "staging" + #branch: NAMESPACE_BRANCH + dir: "manifests/apps/NAMESPACE" + auth: token + secretRef: + name: git-creds +--- +#ROOT_REPO/namespaces/NAMESPACE/sync-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: staging-rb-NAMESPACE + namespace: NAMESPACE + annotations: + configmanagement.gke.io/cluster-selector: staging +subjects: +- kind: ServiceAccount + name: ns-reconciler-NAMESPACE-staging-NAMESPACE- + namespace: config-management-system +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +--- + +#ROOT_SOURCE/namespaces/NAMESPACE/repo-sync.yaml +apiVersion: configsync.gke.io/v1beta1 +kind: RepoSync +metadata: + name: prod-NAMESPACE + namespace: NAMESPACE + annotations: + configmanagement.gke.io/cluster-selector: prod +spec: + sourceType: git + # Since this is for a namespace repository, the format is unstructured + sourceFormat: unstructured + git: + repo: "GIT_REPO" + revision: "prod" + #branch: NAMESPACE_BRANCH + dir: "manifests/apps/NAMESPACE" + auth: token + secretRef: + name: git-creds +--- +#ROOT_REPO/namespaces/NAMESPACE/sync-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: prod-rb-NAMESPACE + namespace: NAMESPACE + annotations: + configmanagement.gke.io/cluster-selector: prod +subjects: +- kind: ServiceAccount + name: ns-reconciler-NAMESPACE-prod-NAMESPACE- + namespace: config-management-system +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +--- +#What should be the name of the reconciler's service account? +#If the RepoSync name is repo-sync, SERVICE_ACCOUNT_NAME is ns-reconciler-NAMESPACE. +# Otherwise, it is ns-reconciler-NAMESPACE-REPO_SYNC_NAME-REPO_SYNC_NAME_LENGTH. +#For example, if your RepoSync name is prod, then the SERVICE_ACCOUNT_NAME would be ns-reconciler-NAMESPACE-prod-4. The integer 4 is used as prod contains 4 characters. +# https://cloud.google.com/anthos-config-management/docs/how-to/multiple-repositories \ No newline at end of file diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/fluentd_config.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/fluentd_config.yaml new file mode 100644 index 000000000..e85cf38d0 --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/fluentd_config.yaml @@ -0,0 +1,44 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluentbit-config +data: + fluent-bit.conf: | + [SERVICE] + Parsers_File parsers.conf + [INPUT] + Name tail + Path /tmp/ray/session_latest/logs/* + Tag ray + Path_Key filename + Refresh_Interval 5 + [FILTERS] + Name parser + Match ray + Key_Name filename + Parser rayjob + Reserve_Data On + [OUTPUT] + Name stdout + Format json_lines + Match * + + parsers.conf: | + [PARSER] + Name rayjob + Format regex + Regex (?raysubmit_[^.]*) \ No newline at end of file diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/kustomization.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/kustomization.yaml new file mode 100644 index 000000000..5d213bc4b --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/kustomization.yaml @@ -0,0 +1,29 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: NAMESPACE + +resources: +- fluentd_config.yaml +- serviceaccount_ray_head.yaml +- serviceaccount_ray_worker.yaml + +helmCharts: +- name: ray-cluster + repo: https://ray-project.github.io/kuberay-helm/ + version: 1.0.0 + releaseName: ray-cluster + valuesFile: values.yaml diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/serviceaccount_ray_head.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/serviceaccount_ray_head.yaml new file mode 100644 index 000000000..b88329a3c --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/serviceaccount_ray_head.yaml @@ -0,0 +1,21 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: KUBERNETES_SERVICE_ACCOUNT_RAY_HEAD + namespace: NAMESPACE + annotations: + iam.gke.io/gcp-service-account: GOOGLE_SERVICE_ACCOUNT_RAY_HEAD diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/serviceaccount_ray_worker.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/serviceaccount_ray_worker.yaml new file mode 100644 index 000000000..eefd56a56 --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/serviceaccount_ray_worker.yaml @@ -0,0 +1,21 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: KUBERNETES_SERVICE_ACCOUNT_RAY_WORKER + namespace: NAMESPACE + annotations: + iam.gke.io/gcp-service-account: GOOGLE_SERVICE_ACCOUNT_RAY_WORKER diff --git a/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/values.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/values.yaml new file mode 100644 index 000000000..e8cf46a4b --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/values.yaml @@ -0,0 +1,319 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +image: + # Replace this with your own image if needed. + repository: rayproject/ray + tag: 2.7.1-py310-gpu + pullPolicy: IfNotPresent + +nameOverride: "kuberay" +fullnameOverride: "" + +imagePullSecrets: [] +# - name: an-existing-secret + +head: + groupName: headgroup + rayVersion: 2.7.1 + enableInTreeAutoscaling: true + # If enableInTreeAutoscaling is true, the autoscaler sidecar will be added to the Ray head pod. + # Ray autoscaler integration is supported only for Ray versions >= 1.11.0 + # Ray autoscaler integration is Beta with KubeRay >= 0.3.0 and Ray >= 2.0.0. + # enableInTreeAutoscaling: true + # autoscalerOptions is an OPTIONAL field specifying configuration overrides for the Ray autoscaler. + # The example configuration shown below below represents the DEFAULT values. + autoscalerOptions: + # upscalingMode: Default + # idleTimeoutSeconds: 60 + # securityContext: {} + # env: [] + # envFrom: [] + # resources specifies optional resource request and limit overrides for the autoscaler container. + # For large Ray clusters, we recommend monitoring container resource usage to determine if overriding the defaults is required. + resources: + limits: + cpu: "500m" + memory: "512Mi" + requests: + cpu: "500m" + memory: "512Mi" + labels: + cloud.google.com/gke-ray-node-type: head + created-by: ray-on-gke + serviceAccountName: "KUBERNETES_SERVICE_ACCOUNT_RAY_HEAD" + rayStartParams: + dashboard-host: '0.0.0.0' + block: 'true' + num-cpus: '0' + # containerEnv specifies environment variables for the Ray container, + # Follows standard K8s container env schema. + image: + repository: rayproject/ray + tag: 2.7.1-py310 + pullPolicy: IfNotPresent + containerEnv: + # - name: EXAMPLE_ENV + # value: "1" + - name: RAY_memory_monitor_refresh_ms + value: "0" + envFrom: [] + # - secretRef: + # name: my-env-secret + # ports optionally allows specifying ports for the Ray container. + ports: [] + # resource requests and limits for the Ray head container. + # Modify as needed for your application. + # Note that the resources in this example are much too small for production; + # we don't recommend allocating less than 8G memory for a Ray pod in production. + # Ray pods should be sized to take up entire K8s nodes when possible. + # Always set CPU and memory limits for Ray pods. + # It is usually best to set requests equal to limits. + # See https://docs.ray.io/en/latest/cluster/kubernetes/user-guides/config.html#resources + # for further guidance. + resources: + limits: + cpu: "4" + # To avoid out-of-memory issues, never allocate less than 2G memory for the Ray head. + memory: "10G" + ephemeral-storage: 20Gi + requests: + cpu: "4" + memory: "10G" + ephemeral-storage: 10Gi + annotations: {} + nodeSelector: + iam.gke.io/gke-metadata-server-enabled: "true" + tolerations: + - key: "reserved" + operator: "Exists" + effect: "NoSchedule" + affinity: {} + # Ray container security context. + securityContext: {} + volumes: + - name: ray-logs + emptyDir: {} + - name: fluentbit-config + configMap: + name: fluentbit-config + # Ray writes logs to /tmp/ray/session_latests/logs + volumeMounts: + - mountPath: /tmp/ray + name: ray-logs + # sidecarContainers specifies additional containers to attach to the Ray pod. + # Follows standard K8s container spec. + sidecarContainers: + - name: fluentbit + image: fluent/fluent-bit:1.9.6 + # These resource requests for Fluent Bit should be sufficient in production. + resources: + requests: + cpu: 100m + memory: 128Mi + ephemeral-storage: 2Gi + limits: + cpu: 100m + memory: 128Mi + ephemeral-storage: 4Gi + volumeMounts: + - mountPath: /tmp/ray + name: ray-logs + - mountPath: /fluent-bit/etc/ + name: fluentbit-config + +worker: + # If you want to disable the default workergroup + # uncomment the line below + # disabled: true + groupName: workergroup + minReplicas: 1 + maxReplicas: 3 + replicas: 1 + type: worker + labels: + cloud.google.com/gke-ray-node-type: worker + created-by: ray-on-gke + serviceAccountName: "KUBERNETES_SERVICE_ACCOUNT_RAY_WORKER" + rayStartParams: + block: 'true' + resources: '"{\"accelerator_type_l4\": 2}"' + initContainerImage: 'busybox:1.28' # Enable users to specify the image for init container. Users can pull the busybox image from their private repositories. + # Security context for the init container. + initContainerSecurityContext: {} + # containerEnv specifies environment variables for the Ray container, + # Follows standard K8s container env schema. + containerEnv: [] + # - name: EXAMPLE_ENV + # value: "1" + envFrom: [] + # - secretRef: + # name: my-env-secret + # ports optionally allows specifying ports for the Ray container. + ports: [] + # resource requests and limits for the Ray head container. + # Modify as needed for your application. + # Note that the resources in this example are much too small for production; + # we don't recommend allocating less than 8G memory for a Ray pod in production. + # Ray pods should be sized to take up entire K8s nodes when possible. + # Always set CPU and memory limits for Ray pods. + # It is usually best to set requests equal to limits. + # See https://docs.ray.io/en/latest/cluster/kubernetes/user-guides/config.html#resources + # for further guidance. + resources: + limits: + cpu: "22" + nvidia.com/gpu: "2" + memory: "90G" + ephemeral-storage: 20Gi + requests: + cpu: "22" + nvidia.com/gpu: "2" + memory: "90G" + ephemeral-storage: 10Gi + annotations: + key: value + nodeSelector: + iam.gke.io/gke-metadata-server-enabled: "true" + cloud.google.com/gke-accelerator: "nvidia-l4" + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" + - key: "reserved" + operator: "Exists" + effect: "NoSchedule" + affinity: {} + # Ray container security context. + securityContext: {} + volumes: + - name: ray-logs + emptyDir: {} + - name: fluentbit-config + configMap: + name: fluentbit-config + # Ray writes logs to /tmp/ray/session_latests/logs + volumeMounts: + - mountPath: /tmp/ray + name: ray-logs + # sidecarContainers specifies additional containers to attach to the Ray pod. + # Follows standard K8s container spec. + sidecarContainers: + - name: fluentbit + image: fluent/fluent-bit:1.9.6 + # These resource requests for Fluent Bit should be sufficient in production. + resources: + requests: + cpu: 100m + memory: 128Mi + ephemeral-storage: 2Gi + limits: + cpu: 100m + memory: 128Mi + ephemeral-storage: 4Gi + volumeMounts: + - mountPath: /tmp/ray + name: ray-logs + - mountPath: /fluent-bit/etc/ + name: fluentbit-config + +# The map's key is used as the groupName. +# For example, key:small-group in the map below +# will be used as the groupName +additionalWorkerGroups: + smallGroup: + # Disabled by default + disabled: true + replicas: 1 + minReplicas: 1 + maxReplicas: 3 + type: worker + labels: {} + rayStartParams: + block: 'true' + initContainerImage: 'busybox:1.28' # Enable users to specify the image for init container. Users can pull the busybox image from their private repositories. + # Security context for the init container. + initContainerSecurityContext: {} + # containerEnv specifies environment variables for the Ray container, + # Follows standard K8s container env schema. + containerEnv: [] + # - name: EXAMPLE_ENV + # value: "1" + envFrom: [] + # - secretRef: + # name: my-env-secret + # ports optionally allows specifying ports for the Ray container. + ports: [] + # resource requests and limits for the Ray head container. + # Modify as needed for your application. + # Note that the resources in this example are much too small for production; + # we don't recommend allocating less than 8G memory for a Ray pod in production. + # Ray pods should be sized to take up entire K8s nodes when possible. + # Always set CPU and memory limits for Ray pods. + # It is usually best to set requests equal to limits. + # See https://docs.ray.io/en/latest/cluster/kubernetes/user-guides/config.html#resources + # for further guidance. + resources: + limits: + cpu: 1 + memory: "1G" + requests: + cpu: 1 + memory: "1G" + annotations: + key: value + nodeSelector: {} + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" + - key: "reserved" + operator: "Exists" + effect: "NoSchedule" + affinity: {} + # Ray container security context. + securityContext: {} + volumes: + - name: ray-logs + emptyDir: {} + - name: fluentbit-config + configMap: + name: fluentbit-config + # Ray writes logs to /tmp/ray/session_latests/logs + volumeMounts: + - mountPath: /tmp/ray + name: ray-logs + # sidecarContainers specifies additional containers to attach to the Ray pod. + # Follows standard K8s container spec. + sidecarContainers: + - name: fluentbit + image: fluent/fluent-bit:1.9.6 + # These resource requests for Fluent Bit should be sufficient in production. + resources: + requests: + cpu: 100m + memory: 128Mi + ephemeral-storage: 2Gi + limits: + cpu: 100m + memory: 128Mi + ephemeral-storage: 4Gi + volumeMounts: + - mountPath: /tmp/ray + name: ray-logs + - mountPath: /fluent-bit/etc/ + name: fluentbit-config + +service: + type: ClusterIP diff --git a/best-practices/ml-platform/examples/platform/sandbox/variables.tf b/best-practices/ml-platform/examples/platform/sandbox/variables.tf new file mode 100644 index 000000000..e19c2538e --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/variables.tf @@ -0,0 +1,182 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "cluster_name" { + default = "gke-ml" + description = "Name of the GKE cluster" + type = string +} + +variable "config_management_version" { + default = "1.17.1" + description = "Version of Config Management to enable" + type = string +} + +variable "configsync_repo_name" { + default = "config-sync-repo" + description = "Name of the GitHub repo that will be synced to the cluster with Config sync." + type = string +} + +variable "environment_name" { + default = "dev" + description = "Name of the environment" + type = string +} + +variable "environment_project_id" { + description = "The GCP project where the resources will be created" + type = string +} + +variable "env" { + default = ["dev"] + description = "List of environments" + type = set(string) +} + +variable "github_email" { + description = "GitHub user email." + type = string +} + +variable "github_org" { + description = "GitHub org." + type = string +} + +variable "github_token" { + description = "GitHub token. It is a token with write permissions as it will create a repo in the GitHub org." + type = string +} + +variable "github_user" { + description = "GitHub user name." + type = string +} + +variable "namespace" { + default = "ml-team" + description = "Name of the namespace to demo." + type = string +} + +variable "network_name" { + default = "ml-vpc" + description = "VPC network where GKE cluster will be created" + type = string +} + +variable "ondemand_taints" { + default = [{ + key = "ondemand" + value = true + effect = "NO_SCHEDULE" + }] + description = "Taints to be applied to the on-demand node pool." + type = list(object({ + key = string + value = any + effect = string + })) +} + +variable "reserved_taints" { + default = [{ + key = "reserved" + value = true + effect = "NO_SCHEDULE" + }] + description = "Taints to be applied to the reserved node pool." + type = list(object({ + key = string + value = any + effect = string + })) +} + +variable "routing_mode" { + default = "GLOBAL" + description = "VPC routing mode." + type = string +} + +variable "secret_for_rootsync" { + default = 1 + description = "Create git-cred in config-management-system namespace." + type = number +} + +variable "spot_taints" { + default = [{ + key = "spot" + value = true + effect = "NO_SCHEDULE" + }] + description = "Taints to be applied to the spot node pool." + type = list(object({ + key = string + value = any + effect = string + })) +} + +variable "subnet_01_description" { + default = "subnet 01" + description = "Description of the first subnet." + type = string +} + +variable "subnet_01_ip" { + default = "10.40.0.0/22" + description = "CIDR of the first subnet." + type = string +} + +variable "subnet_01_name" { + default = "ml-vpc-subnet-01" + description = "Name of the first subnet in the VPC network." + type = string +} + +variable "subnet_01_region" { + default = "us-central1" + description = "Region of the first subnet." + type = string +} + +variable "subnet_02_description" { + default = "subnet 02" + description = "Description of the second subnet." + type = string +} + +variable "subnet_02_ip" { + default = "10.12.0.0/22" + description = "CIDR of the second subnet." + type = string +} + +variable "subnet_02_name" { + default = "gke-vpc-subnet-02" + description = "Name of the second subnet in the VPC network." + type = string +} + +variable "subnet_02_region" { + default = "us-west2" + description = "Region of the second subnet." + type = string +} diff --git a/best-practices/ml-platform/examples/platform/sandbox/versions.tf b/best-practices/ml-platform/examples/platform/sandbox/versions.tf new file mode 100644 index 000000000..1b4d21b72 --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/versions.tf @@ -0,0 +1,39 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + github = { + source = "integrations/github" + version = "6.0.1" + } + google = { + source = "hashicorp/google" + version = "5.19.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "5.19.0" + } + null = { + source = "hashicorp/null" + version = "3.2.2" + } + } +} + +provider "github" { + owner = var.github_org + token = var.github_token +} diff --git a/best-practices/ml-platform/examples/use-case/ray/dataprocessing/CONVERSION.md b/best-practices/ml-platform/examples/use-case/ray/dataprocessing/CONVERSION.md new file mode 100644 index 000000000..7e99345cd --- /dev/null +++ b/best-practices/ml-platform/examples/use-case/ray/dataprocessing/CONVERSION.md @@ -0,0 +1,31 @@ +# Steps to convert the code from Notebook to run with Ray on GKE + +1. Decorate the function which needs to be run as remote function in Ray workers + + ``` + import ray + @ray.remote(num_cpus=1) + ``` + +1. Create the run time environment with the libraries needed by remote function + + ``` + runtime_env = {"pip": ["google-cloud-storage==2.16.0", "spacy==3.7.4", "jsonpickle==3.0.3"]} + ``` + +1. Initialize the Ray with the Ray cluster created & pass the runtime environment along + + ``` + ray.init("ray://"+RAY_CLUSTER_HOST, runtime_env=runtime_env)`` + ``` + +1. Get remote object using ray.get() method + + ``` + results = ray.get([get_clean_df.remote(res[i]) for i in range(len(res))]) + ``` + +1. After completing the execution, shutdown Ray clusters + ``` + ray.shutdown() + ``` diff --git a/best-practices/ml-platform/examples/use-case/ray/dataprocessing/README.md b/best-practices/ml-platform/examples/use-case/ray/dataprocessing/README.md new file mode 100644 index 000000000..8e02fe3e7 --- /dev/null +++ b/best-practices/ml-platform/examples/use-case/ray/dataprocessing/README.md @@ -0,0 +1,133 @@ +# Distributed Data Processing with Ray on GKE + +## Dataset + +[This](https://www.kaggle.com/datasets/PromptCloudHQ/flipkart-products) is a pre-crawled public dataset, taken as a subset of a bigger dataset (more than 5.8 million products) that was created by extracting data from [Flipkart](https://www.flipkart.com/), a leading Indian eCommerce store. + +## Architecture + +![DataPreprocessing](/best-practices/ml-platform/docs/images/ray-dataprocessing-workflow.png) + +## Data processing steps + +The dataset has product information such as id, name, brand, description, image urls, product specifications. + +The preprocessing.py file does the following: + +- Read the csv from Cloud Storage +- Clean up the product description text +- Extract image urls, validate and download the images into cloud storage +- Cleanup & extract attributes as key-value pairs + +## How to use this repo: + +1. Clone the repository and change directory to the guide directory + + ``` + git clone https://github.com/GoogleCloudPlatform/ai-on-gke && \ + cd ai-on-gke/best-practices/ml-platform/examples/use-case/ray/dataprocessing + ``` + +1. Set environment variables + + ``` + CLUSTER_NAME= + PROJECT_ID= + PROCESSING_BUCKET= + DOCKER_IMAGE_URL=us-docker.pkg.dev/${PROJECT_ID}/dataprocessing/dp:v0.0.1 + ``` + +1. Create a Cloud Storage bucket to store raw data + + ``` + gcloud storage buckets create gs://${PROCESSING_BUCKET} --project ${PROJECT_ID} + ``` + +1. Download the raw data csv file from above and store into the bucket created in the previous step. + The kaggle cli can be installed using the following [instructions](https://github.com/Kaggle/kaggle-api#installation) + To use the cli you must create an API token (Kaggle > User Profile > API > Create New Token), the downloaded file should be stored in HOME/.kaggle/kaggle.json. + Alternatively, it can be [downloaded](https://www.kaggle.com/datasets/atharvjairath/flipkart-ecommerce-dataset) from the kaggle website + + ``` + kaggle datasets download --unzip atharvjairath/flipkart-ecommerce-dataset && \ + gcloud storage cp flipkart_com-ecommerce_sample.csv \ + gs://${PROCESSING_BUCKET}/flipkart_raw_dataset/flipkart_com-ecommerce_sample.csv + ``` + +1. Provide respective GCS bucket access rights to GKE Kubernetes Service Accounts. + Ray head with access to read the raw source data in the storage bucket + Ray worker(s) with the access to write data to the storage bucket. + + ``` + gcloud projects add-iam-policy-binding ${PROJECT_ID} \ + --member "serviceAccount:wi-ml-team-ray-head@${PROJECT_ID}.iam.gserviceaccount.com" \ + --role roles/storage.objectViewer + + gcloud projects add-iam-policy-binding ${PROJECT_ID} \ + --member "serviceAccount:wi-ml-team-ray-worker@${PROJECT_ID}.iam.gserviceaccount.com" \ + --role roles/storage.objectAdmin + ``` + +1. Create Artifact Registry repository for your docker image + + ``` + gcloud artifacts repositories create dataprocessing \ + --repository-format=docker \ + --location=us \ + --project=${PROJECT_ID} \ + --async + ``` + +1. Enable the Cloud Build APIs + + ``` + gcloud services enable cloudbuild.googleapis.com --project ${PROJECT_ID} + ``` + +1. Build container image using Cloud Build and push the image to Artifact Registry + + ``` + cd src && \ + gcloud builds submit --tag ${DOCKER_IMAGE_URL} . && \ + cd .. + ``` + +1. Update respective variables in the Job submission manifest to reflect your configuration. + + - Image is the docker image that was built in the previous step + - Processing bucket is the location of the GCS bucket where the source data and results will be stored + - Ray Cluster Host - if used in this example, it should not need to be changed, but if your Ray cluster service is named differently or in a different namespace, update accordingly. + + ``` + sed -i "s|#IMAGE|${DOCKER_IMAGE_URL}|" job.yaml && \ + sed -i "s|#PROCESSING_BUCKET|${PROCESSING_BUCKET}|" job.yaml + ``` + +1. Get credentials for the GKE cluster + + ``` + gcloud container fleet memberships get-credentials ${CLUSTER_NAME} + ``` + +1. Create the Job in the “ml-team” namespace using kubectl command + + ``` + kubectl apply -f job.yaml + ``` + +1. Monitor the execution in Ray Dashboard + + - Jobs -> Running Job ID + - See the Tasks/actors overview for Running jobs + - See the Task Table for a detailed view of task and assigned node(s) + - Cluster -> Node List + - See the Ray actors running on the worker process + +1. Once the Job is completed, both the prepared dataset as a CSV and the images are stored in Google Cloud Storage. + + ``` + gcloud storage ls gs://${PROCESSING_BUCKET}/flipkart_preprocessed_dataset/flipkart.csv + gcloud storage ls gs://${PROCESSING_BUCKET}/flipkart_images + ``` + +> For additional information about converting you code from a notebook to run as a Job on GKE see the [Conversion Guide](CONVERSION.md) diff --git a/best-practices/ml-platform/examples/use-case/ray/dataprocessing/job.yaml b/best-practices/ml-platform/examples/use-case/ray/dataprocessing/job.yaml new file mode 100644 index 000000000..cc44a972d --- /dev/null +++ b/best-practices/ml-platform/examples/use-case/ray/dataprocessing/job.yaml @@ -0,0 +1,21 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: job + namespace: ml-team +spec: + template: + metadata: + labels: + app: job + spec: + containers: + - name: job + image: #IMAGE + env: + - name: "PROCESSING_BUCKET" + value: #PROCESSING_BUCKET + - name: "RAY_CLUSTER_HOST" + value: "ray-cluster-kuberay-head-svc.ml-team:10001" + restartPolicy: Never + serviceAccountName: ray-worker diff --git a/best-practices/ml-platform/examples/use-case/ray/dataprocessing/src/Dockerfile b/best-practices/ml-platform/examples/use-case/ray/dataprocessing/src/Dockerfile new file mode 100644 index 000000000..98aed63de --- /dev/null +++ b/best-practices/ml-platform/examples/use-case/ray/dataprocessing/src/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.10-slim-bullseye as build-stage + +ENV PATH=/venv/bin:${PATH} +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +COPY requirements.txt /venv/requirements.txt +RUN pip install --no-cache-dir -r /venv/requirements.txt + +COPY preprocessing.py /app/preprocessing.py + +WORKDIR /app + +CMD python preprocessing.py diff --git a/best-practices/ml-platform/examples/use-case/ray/dataprocessing/src/preprocessing.py b/best-practices/ml-platform/examples/use-case/ray/dataprocessing/src/preprocessing.py new file mode 100644 index 000000000..7aef363d0 --- /dev/null +++ b/best-practices/ml-platform/examples/use-case/ray/dataprocessing/src/preprocessing.py @@ -0,0 +1,158 @@ +import os +import ray +import pandas as pd +from typing import List +import urllib.request, urllib.error +import time +from google.cloud import storage +import spacy +import jsonpickle +import re + +IMAGE_BUCKET = os.environ['PROCESSING_BUCKET'] +RAY_CLUSTER_HOST = os.environ['RAY_CLUSTER_HOST'] +GCS_IMAGE_FOLDER = 'flipkart_images' + +@ray.remote(num_cpus=1) +def get_clean_df(df): + + def extract_url(image_list: str) -> List[str]: + image_list = image_list.replace('[', '') + image_list = image_list.replace(']', '') + image_list = image_list.replace('"', '') + image_urls = image_list.split(',') + return image_urls + + def download_image(image_url, image_file_name, destination_blob_name): + storage_client = storage.Client() + image_found_flag = False + try: + urllib.request.urlretrieve(image_url, image_file_name) + bucket = storage_client.bucket(IMAGE_BUCKET) + blob = bucket.blob(destination_blob_name) + blob.upload_from_filename(image_file_name) + print( + f"File {image_file_name} uploaded to {destination_blob_name}." + ) + image_found_flag = True + except urllib.error.HTTPError: + print("HTTPError exception") + except urllib.error.URLError: + print("URLError exception") + except: + print("Unknown exception") + return image_found_flag + + def prep_product_desc(df): + # Cleaning the description text + spacy.cli.download("en_core_web_sm") + model = spacy.load("en_core_web_sm") + + def parse_nlp_description(description) -> str: + if not pd.isna(description): + doc = model(description.lower()) + lemmas = [] + for token in doc: + if token.lemma_ not in lemmas and not token.is_stop and token.is_alpha: + lemmas.append(token.lemma_) + return ' '.join(lemmas) + + df['description'] = df['description'].apply(parse_nlp_description) + return df + + # Extract product attributes as key-value pair + def parse_attributes(specification: str): + spec_match_one = re.compile("(.*?)\\[(.*)\\](.*)") + spec_match_two = re.compile("(.*?)=>\"(.*?)\"(.*?)=>\"(.*?)\"(.*)") + if pd.isna(specification): + return None + m = spec_match_one.match(specification) + out = {} + if m is not None and m.group(2) is not None: + phrase = '' + for c in m.group(2): + if c == '}': + m2 = spec_match_two.match(phrase) + if m2 and m2.group(2) is not None and m2.group(4) is not None: + out[m2.group(2)] = m2.group(4) + phrase = '' + else: + phrase += c + json_string = jsonpickle.encode(out) + return json_string + + def get_product_image(df): + products_with_no_image_count = 0 + products_with_no_image = [] + gcs_image_url = [] + image_found_flag = False + for id, image_list in zip(df['uniq_id'], df['image']): + + if pd.isnull(image_list): # No image url + # print("WARNING: No image url: product ", id) + products_with_no_image_count += 1 + products_with_no_image.append(id) + gcs_image_url.append(None) + continue + image_urls = extract_url(image_list) + for index in range(len(image_urls)): + image_url = image_urls[index] + image_file_name = '{}_{}.jpg'.format(id, index) + destination_blob_name = GCS_IMAGE_FOLDER + '/' + image_file_name + image_found_flag = download_image(image_url, image_file_name, destination_blob_name) + if image_found_flag: + gcs_image_url.append('gs://' + IMAGE_BUCKET + '/' + destination_blob_name) + break + if not image_found_flag: + # print("WARNING: No image: product ", id) + products_with_no_image_count += 1 + products_with_no_image.append(id) + gcs_image_url.append(None) + + # appending gcs image uri into dataframe + gcs_image_loc = pd.DataFrame(gcs_image_url, index=df.index) + gcs_image_loc.columns = ["image_uri"] + df_with_gcs_image_uri = pd.concat([df, gcs_image_loc], axis=1) + return df_with_gcs_image_uri + + df_with_gcs_image_uri = get_product_image(df) + df_with_desc = prep_product_desc(df_with_gcs_image_uri) + df_with_desc['attributes'] = df_with_desc['product_specifications'].apply(parse_attributes) + + return df_with_desc + + +def split_dataframe(df, chunk_size=199): + chunks = list() + num_chunks = len(df) // chunk_size + 1 + for i in range(num_chunks): + chunks.append(df[i * chunk_size:(i + 1) * chunk_size]) + return chunks + + +# This function invokes ray task +def run_remote(): + df = pd.read_csv('gs://'+IMAGE_BUCKET+'/flipkart_raw_dataset/flipkart_com-ecommerce_sample.csv') + df = df[['uniq_id','product_name','description','brand','image','product_specifications']] + runtime_env = {"pip": ["google-cloud-storage==2.16.0", "spacy==3.7.4", "jsonpickle==3.0.3"]} + ray.init("ray://"+RAY_CLUSTER_HOST, runtime_env=runtime_env) + print("STARTED") + start_time = time.time() + res = split_dataframe(df) + results = ray.get([get_clean_df.remote(res[i]) for i in range(len(res))]) + print("FINISHED IN ") + duration = time.time() - start_time + print(duration) + ray.shutdown() + result_df = pd.concat(results, axis=0, ignore_index=True) + result_df.to_csv('gs://'+IMAGE_BUCKET+'/flipkart_preprocessed_dataset/flipkart.csv', index=False) + return result_df + + +def main(): + clean_df = run_remote() + + +if __name__ == "__main__": + """ This is executed when run from the command line """ + main() diff --git a/best-practices/ml-platform/examples/use-case/ray/dataprocessing/src/requirements.txt b/best-practices/ml-platform/examples/use-case/ray/dataprocessing/src/requirements.txt new file mode 100644 index 000000000..f2abde391 --- /dev/null +++ b/best-practices/ml-platform/examples/use-case/ray/dataprocessing/src/requirements.txt @@ -0,0 +1,8 @@ +ray==2.7.1 +ray[client]==2.7.1 +spacy==3.7.4 +google-cloud-storage==2.16.0 +pandas==2.2.1 +gcsfs==2024.3.1 +fsspec==2024.3.1 +jsonpickle==3.0.3 diff --git a/best-practices/ml-platform/terraform/features/initialize/backend.tf b/best-practices/ml-platform/terraform/features/initialize/backend.tf new file mode 100644 index 000000000..8d5d67421 --- /dev/null +++ b/best-practices/ml-platform/terraform/features/initialize/backend.tf @@ -0,0 +1,19 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + backend "local" { + path = "state/default.tfstate" + } +} diff --git a/best-practices/ml-platform/terraform/features/initialize/backend.tf.bucket b/best-practices/ml-platform/terraform/features/initialize/backend.tf.bucket new file mode 100644 index 000000000..991e86976 --- /dev/null +++ b/best-practices/ml-platform/terraform/features/initialize/backend.tf.bucket @@ -0,0 +1,20 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + backend "gcs" { + prefix = "terraform/initialize" + bucket = "" + } +} diff --git a/best-practices/ml-platform/terraform/features/initialize/initialize.auto.tfvars b/best-practices/ml-platform/terraform/features/initialize/initialize.auto.tfvars new file mode 100644 index 000000000..8ef26e4b0 --- /dev/null +++ b/best-practices/ml-platform/terraform/features/initialize/initialize.auto.tfvars @@ -0,0 +1,7 @@ +environment_name = "dev" +project = { + billing_account_id = "" + folder_id = "" + name = "mlp" + org_id = "" +} diff --git a/best-practices/ml-platform/terraform/features/initialize/main.tf b/best-practices/ml-platform/terraform/features/initialize/main.tf new file mode 100644 index 000000000..bd9f5e55c --- /dev/null +++ b/best-practices/ml-platform/terraform/features/initialize/main.tf @@ -0,0 +1,131 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + backend_file = "../../../examples/platform/${var.platform_type}/backend.tf" + project_id_prefix = "${var.project.name}-${var.environment_name}" + project_id_suffix_length = 29 - length(local.project_id_prefix) + tfvars_file = "../../../examples/platform/${var.platform_type}/mlp.auto.tfvars" +} + +resource "random_string" "project_id_suffix" { + length = local.project_id_suffix_length + lower = true + numeric = true + special = false + upper = false +} + +resource "google_project" "environment" { + billing_account = var.project.billing_account_id + folder_id = var.project.folder_id == "" ? null : var.project.folder_id + name = local.project_id_prefix + org_id = var.project.org_id == "" ? null : var.project.org_id + project_id = "${local.project_id_prefix}-${random_string.project_id_suffix.result}" +} + + +resource "google_storage_bucket" "mlp" { + force_destroy = false + location = var.storage_bucket_location + name = "${google_project.environment.project_id}-mlp" + project = google_project.environment.project_id + uniform_bucket_level_access = true + + versioning { + enabled = true + } +} + +resource "null_resource" "write_environment_name" { + triggers = { + md5 = var.environment_name + tfvars_file = local.tfvars_file + } + + provisioner "local-exec" { + command = <=0.13, please open an issue. + +## Usage + +```hcl +module "cloud-nat" { + source = "terraform-google-modules/cloud-nat/google" + version = "~> 1.2" + project_id = var.project_id + region = var.region + router = google_compute_router.router.name +} +``` + +Then perform the following commands on the root folder: + +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| create\_router | Create router instead of using an existing one, uses 'router' variable for new resource name. | `bool` | `false` | no | +| enable\_dynamic\_port\_allocation | Enable Dynamic Port Allocation. If minPorts is set, minPortsPerVm must be set to a power of two greater than or equal to 32. | `bool` | `false` | no | +| enable\_endpoint\_independent\_mapping | Specifies if endpoint independent mapping is enabled. | `bool` | `null` | no | +| icmp\_idle\_timeout\_sec | Timeout (in seconds) for ICMP connections. Defaults to 30s if not set. Changing this forces a new NAT to be created. | `string` | `"30"` | no | +| log\_config\_enable | Indicates whether or not to export logs | `bool` | `false` | no | +| log\_config\_filter | Specifies the desired filtering of logs on this NAT. Valid values are: "ERRORS\_ONLY", "TRANSLATIONS\_ONLY", "ALL" | `string` | `"ALL"` | no | +| max\_ports\_per\_vm | Maximum number of ports allocated to a VM from this NAT. This field can only be set when enableDynamicPortAllocation is enabled.This will be ignored if enable\_dynamic\_port\_allocation is set to false. | `string` | `null` | no | +| min\_ports\_per\_vm | Minimum number of ports allocated to a VM from this NAT config. Defaults to 64 if not set. Changing this forces a new NAT to be created. | `string` | `"64"` | no | +| name | Defaults to 'cloud-nat-RANDOM\_SUFFIX'. Changing this forces a new NAT to be created. | `string` | `""` | no | +| nat\_ips | List of self\_links of external IPs. Changing this forces a new NAT to be created. Value of `nat_ip_allocate_option` is inferred based on nat\_ips. If present set to MANUAL\_ONLY, otherwise AUTO\_ONLY. | `list(string)` | `[]` | no | +| network | VPN name, only if router is not passed in and is created by the module. | `string` | `""` | no | +| project\_id | The project ID to deploy to | `string` | n/a | yes | +| region | The region to deploy to | `string` | n/a | yes | +| router | The name of the router in which this NAT will be configured. Changing this forces a new NAT to be created. | `string` | n/a | yes | +| router\_asn | Router ASN, only if router is not passed in and is created by the module. | `string` | `"64514"` | no | +| router\_keepalive\_interval | Router keepalive\_interval, only if router is not passed in and is created by the module. | `string` | `"20"` | no | +| source\_subnetwork\_ip\_ranges\_to\_nat | Defaults to ALL\_SUBNETWORKS\_ALL\_IP\_RANGES. How NAT should be configured per Subnetwork. Valid values include: ALL\_SUBNETWORKS\_ALL\_IP\_RANGES, ALL\_SUBNETWORKS\_ALL\_PRIMARY\_IP\_RANGES, LIST\_OF\_SUBNETWORKS. Changing this forces a new NAT to be created. | `string` | `"ALL_SUBNETWORKS_ALL_IP_RANGES"` | no | +| subnetworks | Specifies one or more subnetwork NAT configurations | 
list(object({
    name                     = string,
    source_ip_ranges_to_nat  = list(string)
    secondary_ip_range_names = list(string)
  }))
| `[]` | no | +| tcp\_established\_idle\_timeout\_sec | Timeout (in seconds) for TCP established connections. Defaults to 1200s if not set. Changing this forces a new NAT to be created. | `string` | `"1200"` | no | +| tcp\_time\_wait\_timeout\_sec | Timeout (in seconds) for TCP connections that are in TIME\_WAIT state. Defaults to 120s if not set. | `string` | `"120"` | no | +| tcp\_transitory\_idle\_timeout\_sec | Timeout (in seconds) for TCP transitory connections. Defaults to 30s if not set. Changing this forces a new NAT to be created. | `string` | `"30"` | no | +| udp\_idle\_timeout\_sec | Timeout (in seconds) for UDP connections. Defaults to 30s if not set. Changing this forces a new NAT to be created. | `string` | `"30"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| name | Name of the Cloud NAT | +| nat\_ip\_allocate\_option | NAT IP allocation mode | +| region | Cloud NAT region | +| router\_name | Cloud NAT router name | + + + +## Requirements + +Before this module can be used on a project, you must ensure that the following pre-requisites are fulfilled: + +1. Terraform and kubectl are [installed](#software-dependencies) on the machine where Terraform is executed. +2. The Service Account you execute the module with has the right [permissions](#iam-roles). +3. The APIs are [active](#enable-apis) on the project you will launch the cluster in. +4. If you are using a Shared VPC, the APIs must also be activated on the Shared VPC host project and your service account needs the proper permissions there. + +### Terraform plugins + +- [Terraform](https://www.terraform.io/downloads.html) >= 0.13.0 +- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) plugin v4.27.0 + +### Configure a Service Account + +In order to execute this module you must have a Service Account with the +following project roles: + +- [roles/compute.networkAdmin](https://cloud.google.com/nat/docs/using-nat#iam_permissions) + +### Enable APIs + +In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: + +- Compute Engine API - compute.googleapis.com + +## Contributing + +Refer to the [contribution guidelines](./CONTRIBUTING.md) for information on contributing to this module. diff --git a/best-practices/ml-platform/terraform/modules/cloud-nat/main.tf b/best-practices/ml-platform/terraform/modules/cloud-nat/main.tf new file mode 100644 index 000000000..a85277dce --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/cloud-nat/main.tf @@ -0,0 +1,80 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "random_string" "name_suffix" { + length = 6 + special = false + upper = false +} + +locals { + default_name = "cloud-nat-${random_string.name_suffix.result}" + name = var.name != "" ? var.name : local.default_name + nat_ip_allocate_option = length(var.nat_ips) > 0 ? "MANUAL_ONLY" : "AUTO_ONLY" + router = var.create_router ? google_compute_router.router[0].name : var.router +} + +resource "google_compute_router" "router" { + count = var.create_router ? 1 : 0 + + name = var.router + network = var.network + project = var.project_id + region = var.region + + bgp { + asn = var.router_asn + keepalive_interval = var.router_keepalive_interval + } +} + +resource "google_compute_router_nat" "main" { + enable_dynamic_port_allocation = var.enable_dynamic_port_allocation + enable_endpoint_independent_mapping = var.enable_endpoint_independent_mapping + icmp_idle_timeout_sec = var.icmp_idle_timeout_sec + max_ports_per_vm = var.enable_dynamic_port_allocation ? var.max_ports_per_vm : null + min_ports_per_vm = var.min_ports_per_vm + name = local.name + nat_ip_allocate_option = local.nat_ip_allocate_option + nat_ips = var.nat_ips + project = var.project_id + region = var.region + router = local.router + source_subnetwork_ip_ranges_to_nat = var.source_subnetwork_ip_ranges_to_nat + tcp_established_idle_timeout_sec = var.tcp_established_idle_timeout_sec + tcp_time_wait_timeout_sec = var.tcp_time_wait_timeout_sec + tcp_transitory_idle_timeout_sec = var.tcp_transitory_idle_timeout_sec + udp_idle_timeout_sec = var.udp_idle_timeout_sec + + dynamic "log_config" { + for_each = var.log_config_enable == true ? [{ + enable = var.log_config_enable + filter = var.log_config_filter + }] : [] + + content { + enable = log_config.value.enable + filter = log_config.value.filter + } + } + + dynamic "subnetwork" { + for_each = var.subnetworks + content { + name = subnetwork.value.name + source_ip_ranges_to_nat = subnetwork.value.source_ip_ranges_to_nat + secondary_ip_range_names = contains(subnetwork.value.source_ip_ranges_to_nat, "LIST_OF_SECONDARY_IP_RANGES") ? subnetwork.value.secondary_ip_range_names : [] + } + } +} diff --git a/best-practices/ml-platform/terraform/modules/cloud-nat/outputs.tf b/best-practices/ml-platform/terraform/modules/cloud-nat/outputs.tf new file mode 100644 index 000000000..acd7f8ce6 --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/cloud-nat/outputs.tf @@ -0,0 +1,33 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "name" { + description = "Name of the Cloud NAT" + value = local.name +} + +output "nat_ip_allocate_option" { + description = "NAT IP allocation mode" + value = local.nat_ip_allocate_option +} + +output "region" { + description = "Cloud NAT region" + value = google_compute_router_nat.main.region +} + +output "router_name" { + description = "Cloud NAT router name" + value = local.router +} diff --git a/best-practices/ml-platform/terraform/modules/cloud-nat/variables.tf b/best-practices/ml-platform/terraform/modules/cloud-nat/variables.tf new file mode 100644 index 000000000..a329cfbfb --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/cloud-nat/variables.tf @@ -0,0 +1,146 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "create_router" { + default = false + description = "Create router instead of using an existing one, uses 'router' variable for new resource name." + type = bool +} + +variable "enable_dynamic_port_allocation" { + default = false + description = "Enable Dynamic Port Allocation. If minPorts is set, minPortsPerVm must be set to a power of two greater than or equal to 32." + type = bool +} + +variable "enable_endpoint_independent_mapping" { + default = null + description = "Specifies if endpoint independent mapping is enabled." + type = bool +} + +variable "icmp_idle_timeout_sec" { + default = "30" + description = "Timeout (in seconds) for ICMP connections. Defaults to 30s if not set. Changing this forces a new NAT to be created." + type = string +} + +variable "log_config_enable" { + default = false + description = "Indicates whether or not to export logs" + type = bool +} + +variable "log_config_filter" { + default = "ALL" + description = "Specifies the desired filtering of logs on this NAT. Valid values are: \"ERRORS_ONLY\", \"TRANSLATIONS_ONLY\", \"ALL\"" + type = string +} + +variable "max_ports_per_vm" { + default = null + description = "Maximum number of ports allocated to a VM from this NAT. This field can only be set when enableDynamicPortAllocation is enabled.This will be ignored if enable_dynamic_port_allocation is set to false." + type = string +} + +variable "min_ports_per_vm" { + default = "64" + description = "Minimum number of ports allocated to a VM from this NAT config. Defaults to 64 if not set. Changing this forces a new NAT to be created." + type = string +} + +variable "name" { + default = "" + description = "Defaults to 'cloud-nat-RANDOM_SUFFIX'. Changing this forces a new NAT to be created." + type = string +} + +variable "nat_ips" { + default = [] + description = "List of self_links of external IPs. Changing this forces a new NAT to be created. Value of `nat_ip_allocate_option` is inferred based on nat_ips. If present set to MANUAL_ONLY, otherwise AUTO_ONLY." + type = list(string) +} + +variable "network" { + default = "" + description = "VPN name, only if router is not passed in and is created by the module." + type = string +} + +variable "project_id" { + description = "The project ID to deploy to" + type = string +} + +variable "region" { + description = "The region to deploy to" + type = string +} + +variable "router" { + description = "The name of the router in which this NAT will be configured. Changing this forces a new NAT to be created." + type = string +} + +variable "router_asn" { + default = "64514" + description = "Router ASN, only if router is not passed in and is created by the module." + type = string +} + +variable "router_keepalive_interval" { + default = "20" + description = "Router keepalive_interval, only if router is not passed in and is created by the module." + type = string +} + +variable "source_subnetwork_ip_ranges_to_nat" { + default = "ALL_SUBNETWORKS_ALL_IP_RANGES" + description = "Defaults to ALL_SUBNETWORKS_ALL_IP_RANGES. How NAT should be configured per Subnetwork. Valid values include: ALL_SUBNETWORKS_ALL_IP_RANGES, ALL_SUBNETWORKS_ALL_PRIMARY_IP_RANGES, LIST_OF_SUBNETWORKS. Changing this forces a new NAT to be created." + type = string +} + +variable "subnetworks" { + default = [] + description = "Specifies one or more subnetwork NAT configurations" + type = list(object({ + name = string, + secondary_ip_range_names = list(string) + source_ip_ranges_to_nat = list(string) + })) +} + +variable "tcp_established_idle_timeout_sec" { + default = "1200" + description = "Timeout (in seconds) for TCP established connections. Defaults to 1200s if not set. Changing this forces a new NAT to be created." + type = string +} + +variable "tcp_time_wait_timeout_sec" { + default = "120" + description = "Timeout (in seconds) for TCP connections that are in TIME_WAIT state. Defaults to 120s if not set." + type = string +} + +variable "tcp_transitory_idle_timeout_sec" { + default = "30" + description = "Timeout (in seconds) for TCP transitory connections. Defaults to 30s if not set. Changing this forces a new NAT to be created." + type = string +} + +variable "udp_idle_timeout_sec" { + default = "30" + description = "Timeout (in seconds) for UDP connections. Defaults to 30s if not set. Changing this forces a new NAT to be created." + type = string +} diff --git a/best-practices/ml-platform/terraform/modules/cloud-nat/versions.tf b/best-practices/ml-platform/terraform/modules/cloud-nat/versions.tf new file mode 100644 index 000000000..b19ea5231 --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/cloud-nat/versions.tf @@ -0,0 +1,34 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + github = { + source = "integrations/github" + version = "6.0.1" + } + google = { + source = "hashicorp/google" + version = "5.19.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "5.19.0" + } + random = { + source = "hashicorp/random" + version = "3.6.0" + } + } +} diff --git a/best-practices/ml-platform/terraform/modules/cluster/gke.tf b/best-practices/ml-platform/terraform/modules/cluster/gke.tf new file mode 100644 index 000000000..9c51f4d64 --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/cluster/gke.tf @@ -0,0 +1,197 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +data "google_client_config" "default" {} + +data "google_project" "project" { + project_id = var.project_id +} + +resource "google_container_cluster" "mlp" { + provider = google-beta + + deletion_protection = false + enable_shielded_nodes = true + initial_node_count = var.initial_node_count + location = var.region + name = var.cluster_name + network = var.network + node_locations = ["${var.region}-a", "${var.region}-b", "${var.region}-c"] + project = var.project_id + remove_default_node_pool = var.remove_default_node_pool + subnetwork = var.subnet + + addons_config { + gcp_filestore_csi_driver_config { + enabled = true + } + + gcs_fuse_csi_driver_config { + enabled = true + } + + gce_persistent_disk_csi_driver_config { + enabled = true + } + } + + cluster_autoscaling { + autoscaling_profile = "OPTIMIZE_UTILIZATION" + enabled = true + + auto_provisioning_defaults { + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform" + ] + + management { + auto_repair = true + auto_upgrade = true + } + + shielded_instance_config { + enable_integrity_monitoring = true + enable_secure_boot = true + } + + upgrade_settings { + max_surge = 0 + max_unavailable = 1 + strategy = "SURGE" + } + } + + resource_limits { + resource_type = "cpu" + minimum = 4 + maximum = 600 + } + + resource_limits { + resource_type = "memory" + minimum = 16 + maximum = 2400 + } + + resource_limits { + resource_type = "nvidia-a100-80gb" + maximum = 30 + } + + resource_limits { + resource_type = "nvidia-l4" + maximum = 30 + } + + resource_limits { + resource_type = "nvidia-tesla-t4" + maximum = 300 + } + + resource_limits { + resource_type = "nvidia-tesla-a100" + maximum = 50 + } + + resource_limits { + resource_type = "nvidia-tesla-k80" + maximum = 30 + } + + resource_limits { + resource_type = "nvidia-tesla-p4" + maximum = 30 + } + + resource_limits { + resource_type = "nvidia-tesla-p100" + maximum = 30 + } + + resource_limits { + resource_type = "nvidia-tesla-v100" + maximum = 30 + } + } + + logging_config { + enable_components = [ + "APISERVER", + "CONTROLLER_MANAGER", + "SCHEDULER", + "SYSTEM_COMPONENTS", + "WORKLOADS" + ] + } + + ip_allocation_policy { + } + + master_authorized_networks_config { + cidr_blocks { + cidr_block = var.master_auth_networks_ipcidr + display_name = "vpc-cidr" + } + } + + monitoring_config { + enable_components = [ + "APISERVER", + "CONTROLLER_MANAGER", + "DAEMONSET", + "DEPLOYMENT", + "HPA", + "POD", + "SCHEDULER", + "STATEFULSET", + "STORAGE", + "SYSTEM_COMPONENTS" + ] + + managed_prometheus { + enabled = true + } + } + + node_config { + machine_type = var.machine_type + + shielded_instance_config { + enable_integrity_monitoring = true + enable_secure_boot = true + } + } + + node_pool_defaults { + node_config_defaults { + gcfs_config { + enabled = true + } + } + } + + private_cluster_config { + enable_private_nodes = true + enable_private_endpoint = true + master_ipv4_cidr_block = "172.16.0.32/28" + } + + release_channel { + channel = "STABLE" + } + + workload_identity_config { + workload_pool = "${var.project_id}.svc.id.goog" + } +} diff --git a/best-practices/ml-platform/terraform/modules/cluster/outputs.tf b/best-practices/ml-platform/terraform/modules/cluster/outputs.tf new file mode 100644 index 000000000..9c813071a --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/cluster/outputs.tf @@ -0,0 +1,33 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "cluster_id" { + value = google_container_cluster.mlp.id +} + +output "cluster_location" { + value = google_container_cluster.mlp.location +} + +output "cluster_name" { + value = google_container_cluster.mlp.name +} + +output "env" { + value = var.env +} + +output "gke_project_id" { + value = var.project_id +} diff --git a/best-practices/ml-platform/terraform/modules/cluster/variables.tf b/best-practices/ml-platform/terraform/modules/cluster/variables.tf new file mode 100644 index 000000000..e2f8c0daa --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/cluster/variables.tf @@ -0,0 +1,75 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "cluster_name" { + default = "" + description = "GKE cluster name" + type = string +} + +variable "env" { + description = "environment" + type = string +} + +variable "initial_node_count" { + default = 1 + description = "The number of nodes to create in this cluster's default node pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Must be set if node_pool is not set. If you're using google_container_node_pool objects with no default node pool, you'll need to set this to a value of at least 1, alongside setting remove_default_node_pool to true." + type = number +} + +variable "machine_type" { + default = "e2-medium" + description = "The name of a Google Compute Engine machine type." + type = string +} + +variable "master_auth_networks_ipcidr" { + description = "master authorized network" + type = string +} + +variable "network" { + description = "VPC network where the cluster will be created" + type = string +} + +variable "project_id" { + default = "" + description = "The GCP project where the resources will be created" + type = string +} + +variable "region" { + default = "us-central1" + description = "The GCP region where the GKE cluster will be created" + type = string +} + +variable "remove_default_node_pool" { + default = true + description = "If true, deletes the default node pool upon cluster creation. If you're using google_container_node_pool resources with no default node pool, this should be set to true, alongside setting initial_node_count to at least 1." + type = bool +} + +variable "subnet" { + description = "subnetwork where the cluster will be created" + type = string +} + +variable "zone" { + default = "us-central1-a" + description = "The GCP zone where the reservation will be created" + type = string +} diff --git a/best-practices/ml-platform/terraform/modules/cluster/versions.tf b/best-practices/ml-platform/terraform/modules/cluster/versions.tf new file mode 100644 index 000000000..b19f861ad --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/cluster/versions.tf @@ -0,0 +1,26 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "5.19.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "5.19.0" + } + } +} diff --git a/best-practices/ml-platform/terraform/modules/network/README.md b/best-practices/ml-platform/terraform/modules/network/README.md new file mode 100644 index 000000000..6de9bdc13 --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/network/README.md @@ -0,0 +1,110 @@ +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +## Requirements + +| Name | Version | +|------|---------| +| [google](#requirement\_google) | >= 4.28.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [vpc](#module\_vpc) | terraform-google-modules/network/google | 5.2.0 | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [network\_name](#input\_network\_name) | Name of the VPC network. | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | Id of the GCP project where VPC is to be created. | `string` | n/a | yes | +| [routing\_mode](#input\_routing\_mode) | The network routing mode. | `string` | n/a | yes | +| [subnet\_01\_description](#input\_subnet\_01\_description) | Subnet description. | `string` | n/a | yes | +| [subnet\_01\_ip](#input\_subnet\_01\_ip) | IP range of first subnet. | `string` | n/a | yes | +| [subnet\_01\_name](#input\_subnet\_01\_name) | Name of first subnet. | `string` | n/a | yes | +| [subnet\_01\_region](#input\_subnet\_01\_region) | Region of first subnet. | `string` | n/a | yes | +| [subnet\_01\_secondary\_pod\_name](#input\_subnet\_01\_secondary\_pod\_name) | Name of pods IP range. | `string` | n/a | yes | +| [subnet\_01\_secondary\_pod\_range](#input\_subnet\_01\_secondary\_pod\_range) | IP range of the pods. | `string` | n/a | yes | +| [subnet\_01\_secondary\_svc\_1\_name](#input\_subnet\_01\_secondary\_svc\_1\_name) | Name of service IP range. | `string` | n/a | yes | +| [subnet\_01\_secondary\_svc\_1\_range](#input\_subnet\_01\_secondary\_svc\_1\_range) | IP range of the service. | `string` | n/a | yes | +| [subnet\_01\_secondary\_svc\_2\_name](#input\_subnet\_01\_secondary\_svc\_2\_name) | Name of service IP range. | `string` | n/a | yes | +| [subnet\_01\_secondary\_svc\_2\_range](#input\_subnet\_01\_secondary\_svc\_2\_range) | IP range of the service. | `string` | n/a | yes | +| [subnet\_02\_description](#input\_subnet\_02\_description) | Subnet description. | `string` | n/a | yes | +| [subnet\_02\_ip](#input\_subnet\_02\_ip) | IP range of second subnet. | `string` | n/a | yes | +| [subnet\_02\_name](#input\_subnet\_02\_name) | Name of the second subnet. | `string` | n/a | yes | +| [subnet\_02\_region](#input\_subnet\_02\_region) | Region of second subnet. | `string` | n/a | yes | +| [subnet\_02\_secondary\_pod\_name](#input\_subnet\_02\_secondary\_pod\_name) | Name of pods IP range. | `string` | n/a | yes | +| [subnet\_02\_secondary\_pod\_range](#input\_subnet\_02\_secondary\_pod\_range) | IP range of the pods. | `string` | n/a | yes | +| [subnet\_02\_secondary\_svc\_1\_name](#input\_subnet\_02\_secondary\_svc\_1\_name) | Name of service IP range. | `string` | n/a | yes | +| [subnet\_02\_secondary\_svc\_1\_range](#input\_subnet\_02\_secondary\_svc\_1\_range) | IP range of the service. | `string` | n/a | yes | +| [subnet\_02\_secondary\_svc\_2\_name](#input\_subnet\_02\_secondary\_svc\_2\_name) | Name of service IP range. | `string` | n/a | yes | +| [subnet\_02\_secondary\_svc\_2\_range](#input\_subnet\_02\_secondary\_svc\_2\_range) | IP range of the service. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [network](#output\_network) | Object containing details of the VPC network. | + +## Usage + +```hcl + source = "git::https://github.com/YOUR_GITHUB_ORG/terraform-modules.git//vpc/" + project_id = "my-project" + network_name = "my-network" + routing_mode = "GLOBAL" + subnet_01_name = "subnet-1" + subnet_01_ip = "10.40.0.0/22" + subnet_01_region = "us-central1" + subnet_01_description = "subnet 1" + subnet_02_name = "subnet-2" + subnet_02_ip = "10.12.0.0/22" + subnet_02_region = "us-central1" + subnet_02_description = "subnet 2" + subnet_01_secondary_svc_1_name = "subnet1-service1" + subnet_01_secondary_svc_1_range = "10.5.0.0/20" + subnet_01_secondary_svc_2_name = "subnet1-service2" + subnet_01_secondary_svc_2_range = "10.5.16.0/20" + subnet_01_secondary_pod_name = "subnet1-pod" + subnet_01_secondary_pod_range = "10.0.0.0/14" + subnet_02_secondary_svc_1_name = "subnet2-service1" + subnet_02_secondary_svc_1_range = "10.13.0.0/20" + subnet_02_secondary_svc_2_name = "subnet2-service2" + subnet_02_secondary_svc_2_range = "10.13.16.0/20" + subnet_02_secondary_pod_name = "subnet2-pod" + subnet_02_secondary_pod_range = "10.8.0.0/14" + +} +``` + +## Workflow + +This module is called from [multi-tenant platform repo][muti-tenant-platform-repo] that stands up multi-tenant infrastructure for [dev][dev-multi-tenant], [staging][staging-multi-tenant] and [prod][prod-multi-tenant] environments to create a VPC network. Additionally, this module can be called by [infrastructure repo][infra-repo] if the application needs its own VPC networks inside its projects. + +## Contributing + +* [Contributing guidelines][contributing-guidelines] +* [Code of conduct][code-of-conduct] + + + +[contributing-guidelines]: CONTRIBUTING.md +[code-of-conduct]: code-of-conduct.md + + +[muti-tenant-platform-repo]: ../../platform-template +[dev-multi-tenant]: ../../platform-template/env/dev/main.tf?plain=1#L50 +[staging-multi-tenant]: ../../platform-template/env/staging/main.tf?plain=1#L50 +[prod-multi-tenant]: ../../platform-template/env/prod/main.tf?plain=1#L50 +[infra-repo]: ../../app-factory-template/README.md?plain=1#L64 diff --git a/best-practices/ml-platform/terraform/modules/network/outputs.tf b/best-practices/ml-platform/terraform/modules/network/outputs.tf new file mode 100644 index 000000000..d05cb77d0 --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/network/outputs.tf @@ -0,0 +1,28 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "subnet-1" { + description = "subnet1." + value = google_compute_subnetwork.subnet-1.id +} + +output "subnet-2" { + description = "subnet2." + value = google_compute_subnetwork.subnet-2.id +} + +output "vpc" { + description = "VPC." + value = google_compute_network.vpc-network.id +} diff --git a/best-practices/ml-platform/terraform/modules/network/variables.tf b/best-practices/ml-platform/terraform/modules/network/variables.tf new file mode 100644 index 000000000..44a83c9bb --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/network/variables.tf @@ -0,0 +1,59 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + type = string + description = "Id of the GCP project where VPC is to be created." +} + +variable "network_name" { + type = string + description = "Name of the VPC network." +} + +variable "routing_mode" { + default = "GLOBAL" + description = "The network routing mode." + type = string +} + +variable "subnet_01_ip" { + type = string + description = "IP range of first subnet." +} + +variable "subnet_01_name" { + type = string + description = "Name of first subnet." +} + +variable "subnet_01_region" { + type = string + description = "Region of first subnet." +} + +variable "subnet_02_ip" { + type = string + description = "IP range of second subnet." +} + +variable "subnet_02_name" { + type = string + description = "Name of the second subnet." +} + +variable "subnet_02_region" { + type = string + description = "Region of second subnet." +} diff --git a/best-practices/ml-platform/terraform/modules/network/versions.tf b/best-practices/ml-platform/terraform/modules/network/versions.tf new file mode 100644 index 000000000..466fd04d7 --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/network/versions.tf @@ -0,0 +1,22 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "5.19.0" + } + } +} diff --git a/best-practices/ml-platform/terraform/modules/network/vpc.tf b/best-practices/ml-platform/terraform/modules/network/vpc.tf new file mode 100644 index 000000000..a573374b4 --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/network/vpc.tf @@ -0,0 +1,38 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "google_compute_network" "vpc-network" { + auto_create_subnetworks = false + name = var.network_name + project = var.project_id + routing_mode = var.routing_mode +} + +resource "google_compute_subnetwork" "subnet-1" { + ip_cidr_range = var.subnet_01_ip + name = var.subnet_01_name + network = google_compute_network.vpc-network.id + private_ip_google_access = true + project = var.project_id + region = var.subnet_01_region +} + +resource "google_compute_subnetwork" "subnet-2" { + ip_cidr_range = var.subnet_02_ip + name = var.subnet_02_name + network = google_compute_network.vpc-network.id + private_ip_google_access = true + project = var.project_id + region = var.subnet_02_region +} diff --git a/best-practices/ml-platform/terraform/modules/node-pools/nodepools.tf b/best-practices/ml-platform/terraform/modules/node-pools/nodepools.tf new file mode 100644 index 000000000..79fd15029 --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/node-pools/nodepools.tf @@ -0,0 +1,85 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "google_container_node_pool" "node-pool" { + cluster = var.cluster_name + location = var.region + name = format("%s-%s", var.cluster_name, var.node_pool_name) + project = var.project_id + + autoscaling { + location_policy = var.autoscaling["location_policy"] + total_max_node_count = var.autoscaling["total_max_node_count"] + total_min_node_count = var.autoscaling["total_min_node_count"] + } + + network_config { + enable_private_nodes = true + } + + node_config { + machine_type = var.machine_type + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform" + ] + + labels = { + "resource-type" : var.resource_type + } + + gcfs_config { + enabled = true + } + + guest_accelerator { + count = var.accelerator_count + type = var.accelerator + } + + dynamic "reservation_affinity" { + for_each = var.reservation_name != "" ? [1] : [] + content { + consume_reservation_type = "SPECIFIC_RESERVATION" + key = "compute.googleapis.com/reservation-name" + values = [var.reservation_name] + } + } + + shielded_instance_config { + enable_integrity_monitoring = true + enable_secure_boot = true + } + + dynamic "taint" { + for_each = var.taints + content { + effect = taint.value.effect + key = taint.value.key + value = taint.value.value + } + } + } + + lifecycle { + ignore_changes = [ + node_config[0].labels, + node_config[0].taint, + ] + } + + timeouts { + create = "30m" + update = "20m" + } +} diff --git a/best-practices/ml-platform/terraform/modules/node-pools/variables.tf b/best-practices/ml-platform/terraform/modules/node-pools/variables.tf new file mode 100644 index 000000000..f298cf5eb --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/node-pools/variables.tf @@ -0,0 +1,90 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "accelerator" { + default = "nvidia-l4" + description = "The GPU accelerator to use." + type = string +} + +variable "accelerator_count" { + default = 2 + description = "The number of accelerators per machine." + type = number +} + +variable "autoscaling" { + default = { + "total_min_node_count" : 0, + "total_max_node_count" : 24, + "location_policy" : "ANY" + } + type = map(any) +} + +variable "cluster_name" { + default = "" + description = "GKE cluster name" + type = string +} + +variable "machine_reservation_count" { + default = 4 + description = "Number of machines reserved instances with GPUs" + type = number +} + +variable "machine_type" { + default = "g2-standard-24" + description = "The machine type to use." + type = string +} + +variable "node_pool_name" { + description = "Name of the node pool" + type = string +} + +variable "project_id" { + default = "" + description = "The GCP project where the resources will be created" + type = string +} + +variable "region" { + default = "us-central1-a" + description = "The GCP zone where the reservation will be created" + type = string +} + +variable "reservation_name" { + default = "" + description = "reservation name to which the nodepool will be associated" + type = string +} + +variable "resource_type" { + default = "ondemand" + description = "ondemand/spot/reserved." + type = string +} + +variable "taints" { + description = "Taints to be applied to the on-demand node pool." + type = list(object({ + effect = string + key = string + value = any + })) +} diff --git a/best-practices/ml-platform/terraform/modules/node-pools/versions.tf b/best-practices/ml-platform/terraform/modules/node-pools/versions.tf new file mode 100644 index 000000000..b19f861ad --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/node-pools/versions.tf @@ -0,0 +1,26 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "5.19.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "5.19.0" + } + } +} diff --git a/best-practices/ml-platform/terraform/modules/vm-reservations/outputs.tf b/best-practices/ml-platform/terraform/modules/vm-reservations/outputs.tf new file mode 100644 index 000000000..11ffcc6d8 --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/vm-reservations/outputs.tf @@ -0,0 +1,17 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "reservation_name" { + value = split("/", google_compute_reservation.machine_reservation.id)[5] +} diff --git a/best-practices/ml-platform/terraform/modules/vm-reservations/reservations.tf b/best-practices/ml-platform/terraform/modules/vm-reservations/reservations.tf new file mode 100644 index 000000000..d7a0c1ad3 --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/vm-reservations/reservations.tf @@ -0,0 +1,33 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "google_compute_reservation" "machine_reservation" { + name = format("%s-%s", var.cluster_name, "reservation") + project = var.project_id + specific_reservation_required = true + zone = var.zone + + specific_reservation { + count = var.machine_reservation_count + + instance_properties { + machine_type = var.machine_type + + guest_accelerators { + accelerator_count = var.accelerator_count + accelerator_type = var.accelerator + } + } + } +} diff --git a/best-practices/ml-platform/terraform/modules/vm-reservations/variables.tf b/best-practices/ml-platform/terraform/modules/vm-reservations/variables.tf new file mode 100644 index 000000000..c534c75a6 --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/vm-reservations/variables.tf @@ -0,0 +1,55 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "accelerator" { + default = "nvidia-l4" + description = "The GPU accelerator to use." + type = string +} + +variable "accelerator_count" { + default = 2 + description = "The number of accelerators per machine." + type = number +} + +variable "cluster_name" { + default = "" + description = "GKE cluster name" + type = string +} + +variable "machine_reservation_count" { + default = 2 + description = "Number of machines reserved instances with GPUs" + type = number +} + +variable "machine_type" { + default = "g2-standard-24" + description = "The machine type to use." + type = string +} + +variable "project_id" { + default = "" + description = "The GCP project where the resources will be created" + type = string +} + +variable "zone" { + default = "us-central1-a" + description = "The GCP zone where the reservation will be created" + type = string +} diff --git a/best-practices/ml-platform/terraform/modules/vm-reservations/versions.tf b/best-practices/ml-platform/terraform/modules/vm-reservations/versions.tf new file mode 100644 index 000000000..b19f861ad --- /dev/null +++ b/best-practices/ml-platform/terraform/modules/vm-reservations/versions.tf @@ -0,0 +1,26 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "5.19.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "5.19.0" + } + } +}