From 08e2f9d77e19a6317d268466149f7df5ad6ad1c2 Mon Sep 17 00:00:00 2001 From: SonOfLope Date: Mon, 26 Feb 2024 13:25:15 -0500 Subject: [PATCH 01/20] Issue #56: Adds vault-ui ingress --- kubernetes/aks/system/vault/ingress.yml | 31 +++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 kubernetes/aks/system/vault/ingress.yml diff --git a/kubernetes/aks/system/vault/ingress.yml b/kubernetes/aks/system/vault/ingress.yml new file mode 100644 index 00000000..9841d140 --- /dev/null +++ b/kubernetes/aks/system/vault/ingress.yml @@ -0,0 +1,31 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: vault-ui + namespace: vault + labels: + app.kubernetes.io/name: vault-ui + app.kubernetes.io/instance: vault + annotations: + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + cert-manager.io/issuer: "letsencrypt-prod" + ingress.kubernetes.io/force-ssl-redirect: "true" + kubernetes.io/tls-acme: "true" +spec: + ingressClassName: nginx + tls: + - hosts: + - "vault.inspection.alpha.canada.ca" + secretName: vault-ui + rules: + - host: "vault.inspection.alpha.canada.ca" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: vault-ui + port: + name: https From a1dd62e309eb5f016497173d90c1eff187f3f758 Mon Sep 17 00:00:00 2001 From: SonOfLope Date: Mon, 26 Feb 2024 13:36:02 -0500 Subject: [PATCH 02/20] Issue #56: Adds external-dns annotations --- kubernetes/aks/system/vault/ingress.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kubernetes/aks/system/vault/ingress.yml b/kubernetes/aks/system/vault/ingress.yml index 9841d140..0f15567d 100644 --- a/kubernetes/aks/system/vault/ingress.yml +++ b/kubernetes/aks/system/vault/ingress.yml @@ -8,8 +8,10 @@ metadata: app.kubernetes.io/name: vault-ui app.kubernetes.io/instance: vault annotations: + external-dns.alpha.kubernetes.io/target: inspection.alpha.canada.ca nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" - cert-manager.io/issuer: "letsencrypt-prod" + cert-manager.io/issuer: letsencrypt-prod + nginx.ingress.kubernetes.io/rewrite-target: /$2 ingress.kubernetes.io/force-ssl-redirect: "true" kubernetes.io/tls-acme: "true" spec: From f2477b6906fd1c581c8703a601fdf3d35d99b016 Mon Sep 17 00:00:00 2001 From: SonOfLope Date: Mon, 26 Feb 2024 13:37:37 -0500 Subject: [PATCH 03/20] Issue #56: Fix external-dns annotation --- kubernetes/aks/system/vault/ingress.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/aks/system/vault/ingress.yml b/kubernetes/aks/system/vault/ingress.yml index 0f15567d..7ef957e6 100644 --- a/kubernetes/aks/system/vault/ingress.yml +++ b/kubernetes/aks/system/vault/ingress.yml @@ -11,7 +11,7 @@ metadata: external-dns.alpha.kubernetes.io/target: inspection.alpha.canada.ca nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" cert-manager.io/issuer: letsencrypt-prod - nginx.ingress.kubernetes.io/rewrite-target: /$2 + nginx.ingress.kubernetes.io/rewrite-target: / ingress.kubernetes.io/force-ssl-redirect: "true" kubernetes.io/tls-acme: "true" spec: From 924538ce738f9720df76f5aeef709f44e7414002 Mon Sep 17 00:00:00 2001 From: SonOfLope Date: Mon, 26 Feb 2024 13:59:13 -0500 Subject: [PATCH 04/20] Issue #56: Remove https annotation --- kubernetes/aks/system/vault/ingress.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/kubernetes/aks/system/vault/ingress.yml b/kubernetes/aks/system/vault/ingress.yml index 7ef957e6..b827c2f4 100644 --- a/kubernetes/aks/system/vault/ingress.yml +++ b/kubernetes/aks/system/vault/ingress.yml @@ -9,7 +9,6 @@ metadata: app.kubernetes.io/instance: vault annotations: external-dns.alpha.kubernetes.io/target: inspection.alpha.canada.ca - nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" cert-manager.io/issuer: letsencrypt-prod nginx.ingress.kubernetes.io/rewrite-target: / ingress.kubernetes.io/force-ssl-redirect: "true" From 658ec4c3f664bbc1c278bf0edad53033c71fcc51 Mon Sep 17 00:00:00 2001 From: Jonathan Lopez Date: Mon, 26 Feb 2024 14:08:19 -0500 Subject: [PATCH 05/20] Issue #56 : Force https --- kubernetes/aks/system/vault/ingress.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/aks/system/vault/ingress.yml b/kubernetes/aks/system/vault/ingress.yml index b827c2f4..1665c289 100644 --- a/kubernetes/aks/system/vault/ingress.yml +++ b/kubernetes/aks/system/vault/ingress.yml @@ -11,7 +11,7 @@ metadata: external-dns.alpha.kubernetes.io/target: inspection.alpha.canada.ca cert-manager.io/issuer: letsencrypt-prod nginx.ingress.kubernetes.io/rewrite-target: / - ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" kubernetes.io/tls-acme: "true" spec: ingressClassName: nginx From 063460f17e080c9a07f07cd8766cffb080c35042 Mon Sep 17 00:00:00 2001 From: SonOfLope Date: Mon, 26 Feb 2024 15:11:14 -0500 Subject: [PATCH 06/20] Issue #56: cluster-issuer annotation fix --- kubernetes/aks/system/vault/ingress.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/aks/system/vault/ingress.yml b/kubernetes/aks/system/vault/ingress.yml index 1665c289..089c0335 100644 --- a/kubernetes/aks/system/vault/ingress.yml +++ b/kubernetes/aks/system/vault/ingress.yml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/instance: vault annotations: external-dns.alpha.kubernetes.io/target: inspection.alpha.canada.ca - cert-manager.io/issuer: letsencrypt-prod + cert-manager.io/cluster-issuer: letsencrypt-prod nginx.ingress.kubernetes.io/rewrite-target: / nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" kubernetes.io/tls-acme: "true" From af7fa64f2ca70c79bc6b4a403966f8d605ca1883 Mon Sep 17 00:00:00 2001 From: SonOfLope Date: Mon, 26 Feb 2024 15:22:24 -0500 Subject: [PATCH 07/20] Issue #56: change port binding and annotations --- kubernetes/aks/system/vault/ingress.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/aks/system/vault/ingress.yml b/kubernetes/aks/system/vault/ingress.yml index 089c0335..e92d5cec 100644 --- a/kubernetes/aks/system/vault/ingress.yml +++ b/kubernetes/aks/system/vault/ingress.yml @@ -11,7 +11,7 @@ metadata: external-dns.alpha.kubernetes.io/target: inspection.alpha.canada.ca cert-manager.io/cluster-issuer: letsencrypt-prod nginx.ingress.kubernetes.io/rewrite-target: / - nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" kubernetes.io/tls-acme: "true" spec: ingressClassName: nginx @@ -29,4 +29,4 @@ spec: service: name: vault-ui port: - name: https + number: 8200 From d94ca0525baf4a9b54cf4f241ebab8753f9c4212 Mon Sep 17 00:00:00 2001 From: SonOfLope Date: Mon, 26 Feb 2024 15:42:45 -0500 Subject: [PATCH 08/20] Issue #56: revert changes to working fix --- kubernetes/aks/system/vault/ingress.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/aks/system/vault/ingress.yml b/kubernetes/aks/system/vault/ingress.yml index e92d5cec..089c0335 100644 --- a/kubernetes/aks/system/vault/ingress.yml +++ b/kubernetes/aks/system/vault/ingress.yml @@ -11,7 +11,7 @@ metadata: external-dns.alpha.kubernetes.io/target: inspection.alpha.canada.ca cert-manager.io/cluster-issuer: letsencrypt-prod nginx.ingress.kubernetes.io/rewrite-target: / - nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" kubernetes.io/tls-acme: "true" spec: ingressClassName: nginx @@ -29,4 +29,4 @@ spec: service: name: vault-ui port: - number: 8200 + name: https From c558fac57aae4a7c4b7be32c3a5ef0c023a710f1 Mon Sep 17 00:00:00 2001 From: Jonathan Lopez Date: Mon, 26 Feb 2024 15:43:54 -0500 Subject: [PATCH 09/20] Issue #56: enable tls communication --- kubernetes/aks/system/vault/helm/values.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/aks/system/vault/helm/values.yml b/kubernetes/aks/system/vault/helm/values.yml index 6244d890..24d16863 100644 --- a/kubernetes/aks/system/vault/helm/values.yml +++ b/kubernetes/aks/system/vault/helm/values.yml @@ -788,7 +788,7 @@ server: plugin_directory = "/vault/plugins" listener "tcp" { - tls_disable = true + tls_disable = false address = "[::]:8200" cluster_address = "[::]:8201" tls_cert_file = "/vault/userconfig/vault-ha-tls/vault.crt" From aa7ff6097cbd4de4b9cdef5b8f2186fa220083a1 Mon Sep 17 00:00:00 2001 From: Jonathan Lopez Date: Tue, 5 Mar 2024 12:47:35 -0500 Subject: [PATCH 10/20] Issue #56: Add vault documentation --- .github/workflows/test.yml | 23 +++++++ docs/generic-achitecture.md | 2 +- docs/img/Vault-argoCD-workflow.svg | 21 ++++++ docs/img/create-new-secret.PNG | Bin 0 -> 11562 bytes docs/{png => img}/ha-redundancy-lb.png | Bin docs/img/pat-token-scope.png | Bin 0 -> 20865 bytes docs/img/pv-secret-engine.png | Bin 0 -> 25804 bytes docs/secret-management.md | 89 +++++++++++++++++++++++++ 8 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test.yml create mode 100644 docs/img/Vault-argoCD-workflow.svg create mode 100644 docs/img/create-new-secret.PNG rename docs/{png => img}/ha-redundancy-lb.png (100%) create mode 100644 docs/img/pat-token-scope.png create mode 100644 docs/img/pv-secret-engine.png create mode 100644 docs/secret-management.md diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..e229876f --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,23 @@ +name: Python api-test workflows + +on: + pull_request: + types: + - opened + - closed + - synchronize + +jobs: + lint-test: + uses: ai-cfia/github-workflows/.github/workflows/workflow-lint-test-python.yml@main + secrets: inherit + + markdown-check: + uses: ai-cfia/github-workflows/.github/workflows/workflow-markdown-check.yml@main + with: + config-file-path: '.mlc_config.json' + secrets: inherit + + repo-standard: + uses: ai-cfia/github-workflows/.github/workflows/workflow-repo-standards-validation.yml@main + secrets: inherit diff --git a/docs/generic-achitecture.md b/docs/generic-achitecture.md index d0264749..e74632d1 100644 --- a/docs/generic-achitecture.md +++ b/docs/generic-achitecture.md @@ -158,7 +158,7 @@ flowchart 3. Representation of the two diagrams above in one -![Diagram](png/ha-redundancy-lb.png) +![Diagram](img/ha-redundancy-lb.png) ## References diff --git a/docs/img/Vault-argoCD-workflow.svg b/docs/img/Vault-argoCD-workflow.svg new file mode 100644 index 00000000..273f9d18 --- /dev/null +++ b/docs/img/Vault-argoCD-workflow.svg @@ -0,0 +1,21 @@ + + + + + + + + Sync deployment version from the podFinesseRepositoryDeveloperCommits codeGets new versionthrough webhookbuild and push a new semantic versionArgoCDFinesse namespaceFinessePodGithub Container RegistryGithubAzure Kubernetes ClusterTriggers new pipelineAccess Vault UI to update secretsVault agentInject secrets as a newversionCopy secrets to shared volume mount Reads new secrets as env variableGithub actions \ No newline at end of file diff --git a/docs/img/create-new-secret.PNG b/docs/img/create-new-secret.PNG new file mode 100644 index 0000000000000000000000000000000000000000..c0a721b345e68711a38ff69a8bd5702b7f1cee45 GIT binary patch literal 11562 zcmdVAWl$W=7ynCuKyU~aAVBcowm@)q_u#=TKybI104Uj_i_PmE-Wt$15+K1@?iM(4#MVy%Y<&1jWj~_df4|Wgar}lA%|XqPdv`17O7J204HkmA z@ZA{qjY7I)Gzr#=F)te~&x7wjqhVkYVz6J2Lc|XLJ_ZJ+SXANl2;&?%ij+u|T74jp z$+@@?R?8H?xjldom5WBv)+s$yvNZ?yFP|U2FolDNslw&Pl)ir~(mcxO9d#+?I`%<< zl6o`tMF&nK;;5}7^-s9LKLKH2ULJqUtdDS3SdG=SADT|7-vl+Wy{5D^{b;9nJyV+R za{D5^0sPan7?q}R`WX4`Es zwMLd+HoqvZ_l8(;Y>2^2NQ$?X#06b+_3WC&(58#YTy452Z1Hk4cV-7%RK3gUl(19h z8(G`?_w~#Ds2RDklpL&TZ|b_Fs411irvpQ~Ed8s44%rm$PL#8w+|BhZ7>M4d1os>u z#R{)(?$}P`^ZNj4y;t&CS4YX;H^m4gO0i?3!#^vIXn?I*%J;Tt<~}Hsa6o_XVuZR+#{EY zBMGl|BZ+&-tGnM9D)zFc*wC|cGDd8@_m>D7{7LG{SQr7Mp-2)NR1QT6hL0m05V1pEap-lBuS_BwiVMn3wPIGB*ZIz=;$Z3lYb5uy z>Y>Pp;~4Yd2Kz88bPn_=snc<{|Kwclg;SMPw{3pyAW!4U^f#ND%fTS7@ZD?UK6S9J zJ`2=$#FyJOESC)hE=-ope7x0&^9WCOPjKf+!$3ud$7B95H|(c1(=-US&uh|%`*B1~ z7Ca!~m1t6am4#23X`XTHJ*slL5mT*TjOf zWnWSC$o4t%Vjd~aaH|KyZ^}gOlCA}J(BRUWE6@=f9qVa`1@;q z@ig*pd3*ifaG;Db_fc9w022!^D4dEoqG%H}zV*utU9}TcF!f_CvlA8+I?1r-a@_}4 zYy13{NUF7D-TYP(*gp|v@IObBrL6F2&Im#P3&9c|MW?T{8bQUEQWS@m&^M#Mt3D9P zQ!NOBB=-HSC&@D_e>4j|$Jt-ra1dehL5_$R1JTmbAeha;@h2lNyR75KPa}f-emdJh zX81x;wXrf&D>KY4bCl9>2yRso{^u}oC+KKCw7QL{qti;|1++Bv>@;z-_3!K)8de}pEQR2Ocok&ON8Jtl=CgKk9*fC z%}hf

9)UZi@`_WbAi)TW4d#6zj6>x*lIIynW!pN%&XKb#d-5bdSs5sD^}Pd$e4t zFi{W8wd)ol1P>nS?@z~;=tS%C%=G4loHLlWGzSd+zq1PL|A)UHLV8WgLI3Ybm_KAR zuQ&X7wqBbCj8aI(e-H2fzYiM!KPaT-KSksdeS^!F6*nN1*_b+xH8!}_kmNca%OwGR zaO{#68tddbI6pDN=Eyan_ zo1Czd=C4HOXwheypph804Nhn+R+5+OsERHa>!bawmCPha7{LphZ2sDdz*%GK;Yt@D zy8|MfP;j(xI~w5Y;SFBPu(Kq;R#k)M^?cmb4Vie53@^1Ev|IHqS!#VcuDlriCw>;< z7_nkZHsv!a%|hm|HmO5bh?j-P({{3j*0nvGBbhYBT3f?~RbHh(nSi^*+)~OdW1~!& zY?QN~%V>;7DXxJA;ASI2VqoJwCo1XLF6c3$bg?lyZ0$;kb1_D^{Ny}3Qt3ML^@`zL zpAr52d&>83*Mv+>b?GSDi4nP|D3uD{HloO$jL*5und*w5P2|2Iq* zYdSiLXy6T>2+cjDM>vu9W(1JsaJ>*z`UsEDUa()Z({Z16HbquSK2daTGx8o_wVbmJOC(?3A$$u37m>F7yHp9>AhU_dbHy;<$Hf6SjyY&+W2vLQxIScUW-E&33l*iVamIHw(P&F^yQb}b1RakPrCqYXSJ?Ly_34g)h=uOGzbxnteV&-Lh``ju?n zy()XdLnl4aAAia6d6$iRrGZ9!&G-Pl#l^_@>pc&mcD)DRM?ar9X-gx9=;PfdQ{X-Q zfC!M?_;$KI1d7f3d`lLIj4q+Ni`$GNS6<_SYO&OJ7SIm+?q87{gOxRIq%Sr^=MJIq zK=lL{@-w%NFsPtRyy6b#0r1|OVqQG!%k^kKd8{5O3Y?0%6J+OT?a}Q)N_S(JeS!KB zs?7>jBU*I&lSoyl$JqFBwKATm!XSLj_3rurt{vg)UeZcvH2AK!1A9;*OCVcss@9Gx zk5kNr&1r8tK}C2xx9Sn`Ty7cqp|rDVBPG$`M0#$IwY(L}yQ$|gz>wEx)$tbJ;YkJ`&!G zYR|^_AJQb@PzL(6G8MG^iv@DtzVyYabSVw3xJ1J~Lt1W+NIOdPEg_avD9s&PjIO>^ zU}x+zG}iJ7`rchIFh3<;K0E%;Gld@5nR{K!WC_Cr`XSCcuZVApG%R zPzNuIzv)g*xbo&ffKUROEWsq;V=&>2h2?NFv2=vs!KAgnBiJp&eBuE1o!KqyFCw2C zSee6bWzgkw{0)iR8eOJP5%-n3$}k5#dF+8``7ce(F^&tvJvyNEih}GSpkVju>qZbj z&`j1XCE08DJ8ktoE$7^Aev;kTRr+)|(1OFXWNrTBEI^_r@Pl-6HHv`4@k+zx;Zq93 zz0brUI&5-ylCD_M4|NcU<38&~lh6iG*|<^{EW?iF0)7uNtOjG`#R9vNFGE$I?h+C(J$#o=BlX+WUF6mm zOoqKL&e#XQDXH4r?a&5}*up|wqO`OR`&hr{$NT=|^Rr)xUlz{e6wQxC?D6D1-_emZ zyR)hMV5;|X+U0VrsjW{<1ov8dVH}gu$|jebVC)-N8hz*3JrV$z4Vk!$O$_ykCA%x# z{~eIH=OH9j9i_yQ>vWk)3#8|nQ!Qkrfw%TjdAu$>N)gSOA5ZbzEEk~g4sN%d?eDMX z^hMtDK7@Wx(IR{ZQsd$_JQ>ht3dofJC!-Yz5d_Oa!DAzTmo*1_mB@KWS zc(o}@p<-r%{@WoDYe;v9)w7ow02r3@(EMK}85>-6w^7pg`Eh!GemmWVu1(S>=yr^G zaaaQ^iudcIhcYwB7ZdsFenLyXI0+s#@Rt|!<~}>zVFAjKh{c!g#~AdbTwP% ziysKjGu7g|v-Hs=v?LpR@tPbU-J@t3xGOk^hU8jYKsD(~6O9koCp2?pH6Ir>fnQo9 zpUHiVW!{#Q3h+pnd3Cm|;}^Q5gi0e)h~NZ`B+BV4>tlu?*)mv3IF_&{JDo?g-e0bpgSY7W(F0QtDcS>BgUb_(pisfH+*HLhE_7}SQla7ye6oRTbX7t3?DTu*TE^NV-y5Yo|#0ct*fKl zn#)}BO2k>CFBm-L7?Ug@AZc_I!@G>vI5kH%pgJeNcP~LVu17*ZJ+^d%rfpg6FLeHi zXM9ma=z@cb=A)iuhoDHDO3G4NuO_*F>S}a_uw9FC{>n4UTejFS$EaRxF!SqCxDK0-;P8iCA=Fm{ul&NaP zF{xbPYcW?;l&GM;+-*YZ$e5#IIkKMrefz+X-)txG$sHo0aS!Q#G$<9riG}87y5U5AXk<9PP%?;^ z{?nW{f)#Iyr59SM=WMwzsdeUbAgL>j$qExlVWpGet|97*Uvm zc*#Y>qrn7s@UTBu1|%J`-*w?KUTT?SPO@3^4*FVCmV)MB?^C}qQoUkW^h9nJz>kos zVL}yH4t{bB8-p>$pBY&h9qm?2dSC0ZxsF23v}tqeVx}YOK;;3&x84l*?9`8!YZ@Hp zYjqjCYb{J8tu>v!kSJcY<5~KEpBV{mr>R|4h7~1iZgGX;kBW#!;X6Beu9bDp(Tv@E z-1_uMy|1r3b}bpfyX`>&gDrss+yn_X)!}@q{F}HxEI~);V@NG(j;P?X!ou#p*}`a@ zXB+dB2WQT;*SJKOf5!1=u}ma6`XTkk;rqdLP9!V7lm5+l_7UijR45>8TtUM023Oy+(Jh(6)!0;W5MyHZ?Xh2M$hwUTPYM033*T@ zr2bV;lHVD;-;Tm-Bo*ZG`w1w3{SkzD{Suz$y#4flhnQZt|3#nwUxk*yu{_D>TO(q; z+S{Is&%X4p7{xDHSlyPyF*|-VvDwPhmT7H#mHn;D+rpghmZ+IgNg(f{s995B&Qf(@ zQ0qL=WVF`xgQuaR$kru_*Q@pV?+{#1_VMv1e=5(RwR^t3%u2FEu`y3>WP((-olk-? zR%gxw=50pD^m8Fl&W7dbxnbv9n2tX`6h1d;8d)tzRsItahL2Ib**)6ff^KG-e!)}Q z=WzacdH&8O!QiN|+61{pH?M%IF@csIWtK6ro1fR4VAixSEm_ilL^J1R+1lsuAb(f& z;JUHs?c>z(iv`)c{T5h}ur`Y<&jru)bhPXz-$0z~|3&bB2{t{z0E`4u-c=#Z^aHt> zs192}PePccRly2@G6k33Qe!xLnq7WIO?|l?{Q2%6#CD(SOK<-Et*(6dG;9m)Cd~(! z|8PMT6>v~DHc8+9uxUW8>n#iu#w1jO2q7)l6AIv-|59Rv;?XW zmF;qSMb3Q2b&hhX{sUM6`)WCo0o}FfffwJ^&1up|0n~_Tcot?kydlTr9nvT)|FH{2 zJ!ej(bi3ZSaBvu|W_?2G?W_36@+iwGqATDoQv_gFJpgUsm1_Up%PskPV#B-JzTfob zM*PdsN=F?9!dQ1CCtn1&zJZyA)wfq0x<-$-GarDNjJL=_qs2rRF=*QF;zpxflc~EQiotgfqI!UDXQwL{Xa9%xZl)? zGm-(oh(YEhtdhu^|C$vu?-sm8$4K}dh8I3L8J~fL_Fo02BGZ+p>)!CvJbj*d0-ZC$}06?bZ zi-9&l;c***emHexH(@I1@nfM`98_^knP)b}JJJ!a$i?N^-`7`69Qo;n^s~do$?C{( zWqJ9=yi#u7OksO&S5KQLKB~t{?ejUOxWdZ|L1PQB3nza*Egc_mmO}{F-=224lDP^~ za&lhIxcP`+4`Pqt_!jAp{F$FQn(Lqb(}BAysnM0-ST)3eVe9 zLl+lL14}xOuY29z3CTBIH^vp2*)iTU+1FQ$2Hpi7EZ^>|5LbBagmG6g1l}2xPzbI= zXOTuQX6$00-q1Jc$WxFJc-~5EH-t2UutRJQNKYS!VLL`Xhw8OcH>}dY!T3o?Y1#BV zRGZs!MyWnDQgWQq7~paAiZ$KuIT>rC&S0r^Kzt~?H6;y_FJ%45CL)kCb}dGINX&wy zal9^2WcVJu?Kisn(!COg6i9zHr1QP_B~Fd;yU>^_1G}?&YPyJ}df}dA9dK{n1BNq) zC8coe`AUl6YIRSl4%LFzPUmrYoJRX|R>T7z8d2`Vn&9VOAaG2KD89}{^++ER@qLG+hmU$NV@YD0M!<$!l1>9}wQgHyqyJ@TWGc_d$tTsR!QRTdE0s4-|tH9Odd9OL3wp?wW=V>hv#0s?R zu<0Di&S;46dDVO&V}hV-#%Ifh;wgHlf@TRTTg}B5V;9YoD^!UW4f)f zMGb9_><#)cAq*zhy;8&uQfqU*b)QtcC=FxMzGE~~UHr%&hCc7f1=oe-B%@h}aRV_py$P+emS?7A6-jQoB17tX>zD>ugQ2;l7g$E-6w`{bAP2$$*IeDCHKu2(@K zJHT|cYG`20Rq_(S{_A9xgak$Z*hQ;~KDZK$8@41E=7=oV);lpKL3y|VfG!Eq@cS&F zj2@2-yG%_LEdYZabdB=eYEZDTx6DP%(Q4Ka)l;5%s{$;;1GwaH*hQ@h;utR^ex)kY zo&^I$#t~71W`2XKa)*8)=Nz!)sbj3%z1-AlZ`RGmHql>nr@TSb^z~r3G*||6XGOtwC`ip8&+S7) zsQ0f{LB$Cbo8@DNPw_?Hm5x~_x|-5K2s7W!#j)RS(E*USCxE#P(i|?REEbA&%#LDx z`ssJdzOsN47;38tnERFVkXHE4ZU97-j zJhAQdRaIqkSO~NP3!Wu@gaoo}dGr9k{{$z#60!-X+8u5?3)UN8o!yggYBbBoUsx!4 z3jRn{Q_D3rRGjK{ZnAo=2k_*+vt#I3rg7lKZ|f@`Z6a}*Lszy+f-Jo|9x{nJy@e?2%*KJ6h{liT#!JmWqM3vD z2yE?4GO?IBhsCpHshN6=SQy!qQ3_uq*AgF`mho(-l_hrjgn|^mC+TM0+cfH7&!5}< z%Fqbyr8@6dh^s<+`gtdPZq%&1u@tQ{(1Lvgz22z_Q?BXRIMmxTd1L%D{#IVPhoiel zI_^NQCG*;#k~6mxVA^ZGQ|P)yVT?Pdj@7sisOby^}|D>9tCxI5pS-|e6?(p)GkSDhg{uLUNM0D z{Ne0=&OQ)+U^M)Z*m9y{oVuRM3>nsUxT&M}XnDig-su^4fHBi)`RPSGeXrY!RmS?B9M zznpkn@^3DGy!6l9cCvW%i>H>dv-={oRf8M4G5~Yw0RPY?*tcrgQ^>>ZC>~p`VjtcT z3=%M_799$N~2m^r?vXAG$%1RoR)Lx7l-KTjb0BHxOOTE~2Q3QtD zrUaTfhM{KbnO<-5Juph9S+$IpUX~ljl-2hsfdrPHye=K!UBj>y!A9R4>2Sb+8H#}Z zQH0G=jW=q#Sg}w)4R>QUvyy9|#&on$F7-VoCMGU!fBhtE#q>bVCqgi3K)P*TK@bg< z1v7Q$j^**)IaKm}BNLVB-4GMtn=1i!99e4?P0@4JP~1phMFDUHy=({ zhQ%Y6`k=R>lkDJ4i~~l$;2U=L^%QwFap=@pV0lw4M{b)es^}dpH_wcv%eToW^#=hw z6fqiI2Cx}Gsh#ND?m^8pJ*7PELK;6RJF=CO`uU|OLM6OFSs6kzu)ZA*Q%5|WY=Q>1 zWJWzokz9pF*H|z@va$FOEUc-STl1zxPn>xnH zw{G${D5)gVxArG&t+FiNAG}&8!R*N7g829XE2ArEdKLQrDjSXc>hs}zUlaIhW4Jd_ zPv+!tK~BKL4sd^7!Ix_-HIi7d3Ezou>2e-jxsu0$Us@;}n-v_u)LeOP-{L6X5zXL% z&g&XP5MP!Ra`7RZ^O402l)~-k6;*5DuVG2i*(7WO&dB)+G-TPS{@$M_=6OveU>oz} zI!otaH6CBm?HAB4>o4J&@l+PYnI#AxnXtsBr_t(@@(<{QEqD5QGZFSmf52e;%)AQa z&d^`~USb+?bKMpv){&cRh59Jrn=^gYygWr&)%QL38j=e|E=>pPdwb0$!{*Cl3k#_Y zYMQjNql^6O5}OKQFB8+smx%>aCS$}B(R5G!nw?&Ite%>#%nWSgr-GZ&ny0o4ox-}` zzvFaj1)MGrd~D>Y={ICKEVNInMsFGO#&Ia{*a@Ufa-X!;%}L2!|hgk z;{)d>)G`kc=@mkwjkt^|`caZwq1 z_t)Y38uAuP^Xe|UDTNoz`yCwDog;H>kojIveiYiXY;0!K(gy#rJUg(OY;5n)3Ioc2 zi7%U%pvhC?oWX=_&`uS`z443J9xH7-@*NMa`#8pBDtY91F##hscXPDtJLJXqlHyP2 z5#FuWJaM%Yxjeo)DwEMvo_cv~4{z0BOXE-IHj%d{ z-hgA_TgX;2DlKAsSbLrPh%XfFeMC&gxDh(1V!En+5Ev!f%R#h@YvQeYjCnk~P7w?= zyT=9Y_h8LP-@sT1i>GFg$%kELbI>sV82i4MH~V0oQgjyG5;PQqvtrtT@#3>Ja?3;A zduz&@Vzg8#R^J%_G`<=r7PveMefEfqfo(8WO+R8GJ-e^uq8!G>by^D5)8rfEp6Yb{ zxqdRn%+myLYb==9og=}1SAC884yQb8|LzW$@V&*A4V$8O!{n-+!)gtfohr(p_zRi$ z5YUD(GlZI3j&ZT(IrOY&n*(7@n^~A8uO2m-B2|fH8$}9cd}$_so9`--f;f8x!YGpV z?Zj(hxy98#JE6^>Eu7>znDBbq-kQdKD=QiT_1Rxn@^jf;o!o$hDc^YFzQH?^#P%;y z`fRZrhV0?t(dr`p4Tj5@Ae^i5>zGG&) z?BZOYK@T?S^VZN@_l{R<`a7GDy8%pD_6-0;va&C_1Ka^({(hztl-T)3PAb9xq%J?S#O|efN3E2)o=Z2RGdTduMi4 zfLt+9g}_te5JX&cyQzFz;sN7FEvsd-(UQR!=Ft?HdUR7V@dF}Qcg~)h?bRl*Us_(a zdAawDbV(|ag6-~^9mDKGUXZH11`l5m-8%dog5PA+@>R%|}{th)`$gIV-Z_YKN^8}0k!FJ~8W1l2BS`_LmR(rPT_421s%rP)WG!g7Kk~JCG+lg9NPu@z_LoI)+-{a^B+b^-d;u~SaDz#PUp_824_@V zjYMZ1a@EkKWMFI0aDb#&~q>bcp zYax}R>`$WP@ij4-+sIa1K=2=l$ikoawu;sEwQ28Aw|OPF{@#uYQk1r?$NN@%Wqkmyg2aHwQ3P{GY-U-@x|H2vZ0{`kh+J%4qPF z0k}VClII2F?^yq%Wi>CftVr7UTTT;5h_@ysm&BAffnP zS>DC2$Pa4ytP44|EIH78c}dv{lYqtlLBt9qF0QbHj~DXw_M527DZMCoB)hGJ(tg|i zGS?lpbblus_;!-&Ai14ou$=b8(64!7 zv}_zsm$&gJ;c@#1x2A(>i_6PZod12xT{jh?w%dnZITfofd}^wZXaMm^PbP&bl+=RD zETEE}ZT^1IHqSlIJhbR7AY54_+15YNf1~+L z8|?Df^K;*UBC+!vS<}tJY#g^K$)Q%0j)#S@uB-kBy;|k7S9Kjr?SbW(CgJ#hNdDAe odi@#Y-|Jy=0{^!HtRCz986xC*POnKR_%A_PTtN&d^6lsU1uAq8W&i*H literal 0 HcmV?d00001 diff --git a/docs/png/ha-redundancy-lb.png b/docs/img/ha-redundancy-lb.png similarity index 100% rename from docs/png/ha-redundancy-lb.png rename to docs/img/ha-redundancy-lb.png diff --git a/docs/img/pat-token-scope.png b/docs/img/pat-token-scope.png new file mode 100644 index 0000000000000000000000000000000000000000..08a33c9c9f59d696743b1b2d72dcbbe02d90b24f GIT binary patch literal 20865 zcmdqI^;=s_*9KapE!sk{VlCR@?ozzC2Zy4?f(7?NDNw9H(ICYML5c(^?k)j>1a}B7 z!2^fyectCi=WjSaWU|+F%{6;wuQhwGb>H_ysH@83VZXtC@ZbTS!Y3Kc2M?YI-(P=u zj&c8>Za8H=cZvRFQn}pduFM#_Z|6{lfW^KIp*%{Eq+oJ?wER zvV8DBgG50_QrpX9Z{G8l(UkwvrPF#k^sn&d6JA%W=bT?O2tV{CF(}pa*pD0yr&m=i zJ6u?Jj-+Gk?~qlS4rC+4yVE=t<$L6Y0Bc6c;0KaxQCPy8dd5P9O$Y8F^zRj=Qy<6# zwZxu&+b`BVSnymd)w~OEUu^nb?tgNP_VPiN5G|JQV4aw;HCp@?qZgJuN> zEB(O1!=Ag-7U<@QHwyjtHt7G;v31WSVZci0Y6D8cAnb7vSSK#RBMUP?(EP8@w>lbJ zTzvS%SpTyFn(Oy)u4;c%e;-$6LaKjn_pPqnVWBePtc+SU%kQ8=CQReb^q+~`&nEm? z5WVn!X7zu~QR`2cu>TEBO%}iPKcn-p^#5l_ug{-yr2o5|Z!ZaO72N)3tbBbJf&ahp zMLFr=|1-j$egFU0&5jt%ImtM|LxLHqjp7{H>fztNW97h$ZmXttx|W@{u5GTd|6Tk! zMaSrwKYF{5s~CammR^yxf;P+zaF<|sq^N^IGTmn?#5rKGH+vaYW8roHhN z#GafBfC4Ec|9!d_<}-~}B74SS>Ef5v6^T|DE(JHgH^dhd0$98gJMR7sS>(}jVi2x6 z+n>fFp78m36$>r8uOC0E1&~byx?WP+SMY6+l8{kQthZm{7&ZAM%Fw>^|HqNxzMtfD zBd_P1qOuv_NBYwlDHO=97v6^xW5wBk6hj3~=EcChT>p;-tR!zJC}wH{Y1G7zo``tt z=V>#?sTL}968X?d?|jR4eqP=aIdU)_I2BLHad%@Swp`KJsGO}=cR(cmPe}U3+2;N! z<)`pycnn1p2ME{)PB@9ObdrKeI3|mU#THoCPEd<0VtqVwvoc|bi1}5fUH8udP=PO{ z%I^|siPhpD+Vz<4&xzy`hyn$JdaSz4OElQl`q;|j-zR>6i0$i#&m&{+DDH&rPR2!M z^o2*D5||f(x48s*QC0#Tb%8btF`1dvCSJ==&uFG&js0y@m6e@uI$t<0)b$~eZ@SEn zI{abY!Zo-+vx8gz({r>2d$RY=P_5U=kN`P3Rkr&3t$+QxpN{5uUwZNQVSC*BZ0E<~ zEi~Yv>=pV{fW{n+9io_;r$F-sKsQOx*km%09g=e57+@RVL_}3OyCrnW+H!Zn_!r}$ z+-21r+tLBgYU=6>eXylW-zj$LfHgB^IU@O3>h&)Y*B^Lzs<@Bk z{}XrNBA=98ryV@rfDLVI_ga_T?w^OYv)Sg3QbqHff|p_SdXd>Tu)mtrkCT(E{;Y1^zfq@Tbd+fUTzpZ)yj3tx&QH{ zs8bV~q-Q~_dt&03xTZA7-dwX(T2)mQ)kgq4)&LL5DJkK^N*oEQiJ^Z5fCokbjZPGLuqFD~5k7Dm>hHK$c<}r7`ooBQY&p#e4_r8jf4L|r@6fv`UBox(Jf_LhFkV`u zEWmI;*K{dA*(9g56tBvI;CIgaaL(Q}5?a0ua(nVvl3MuYcBjIwfbSv!zk((WeqtmV92x#Y61(i?Z3Eot+&MAh zF7E-K2Sc(e*0b!L@6I@SkmgIRlcv|+R=j%gy_ zFOXB27J*Q_s$%)jH|zPf1&dy>JWv;Ny+6L~G6uq1yEeL#-Z*SS(J~zYe)vRazc^Qz z9NY9KJw9_ad%7V>(tT$h6*gcstY&oX_2ecF7XaubrBYq(Il8bQWe*~XekU+#5UNI0;d-kX|u z$+*<3Rr*W43uFFhON*DE&t+|yP84%G9u*aH+&{paVf})3`1o$`FbC9m5-0l zcGOpGrhviM&X%#0H1U0TnbV_$v|?iOSLxJMJUna1-sFh?dS-LYwI1!24=W8xTxT$^ z(7|{lj-1N3yfpo6FeVKy@35|2J%4sOTfx0O;d9sAL5mlC@kkQGPY$I2wLwiUoMu}x z$dy_a&ne5t?h^#f#yC!we_8Ytb$( zJ72#$lB;%WZe$ohSZ&0jrK|S#FzfJo(Jzzf47iysALG0s=7j?&{lia530I?2exVB!WR?kg~i$6F6U0h5NKT=f`xj+hO z0=zLYv^k%_i)z?tLqjY;paf5i5NxQ`=I8F;P8Cv|$sl!*&E&Pw^~Kvn&0apcFe1XCT9*c%n;U?_Z5C3#hdQPINb{fF30TSh{DRF zi{7a*>;GQ)F)HZwlTK~VVk#40#2#q^jP4tk1XYKa=w`I{b1PLma@YMeoFOlGJe?pq zPs;da%dAIwqGAB-p;+(LQtl{d-$vNO-hLc7*SC3x42t&ik ztExcv*QO~UvS2T5itMAdXEe??P3L;X#+Zq07bxnnr>#pw+TR$1_Kt10{!*5IQi!>S z9>h2GM%3J?QQ}ty&C0I{2umgG2L|ovM5+y|clm+Xk-kJq3%Rf#{mP5!t5+~$!tMvr zv*(pUZwlHLqzvsPX0JmoSd`{2KBv}4h^`;YwE`936t8rlx9-T637;)Bqp~^?7E-rR-)m>}LM%{NS=S@a({Q z9YZmwLlJ)FXg0F&TtnwKWS;D1uX=W)ki zYuOUjbZ6{RJs+?44m!JihtQo48iL41CuF6)O*=k)s zSp^~g*hX7w+Q*eiJv$7 z+*kz!oB{*<%zkWJzRINw8X|8mvJ&8*0O3TL3iXX3nmJl-nagk14lcH@N$dV-=j>yT zH8+75dd|i~rd20^-7*<}5z!(`9qXeg=$mO&JH=l1!1lYMvSJ0Q+O*iX^Z;;zt|NEF zFP%+rp{7#Y5UIBOsEI|-@*lI(&WL%zz6Kgr9$~}WYVp{uz<7_y2)GSxhJqF=G0Xrc zdh!ZLJ_DSsIf9k@M>_>x&z_>sw#jSM7`aYajk152&*cnFx!AXB{`>mR#!9z7kKK zOw!lHJ2GR};=?kQqOc~fHdP8*)?C$`yFC5xz-^PZI)y6vcZ+c!@=(I@OZkXVEym?>F z##P%Wr<3Tivz4eAti6J}zVhQr<@)2eH9))%qP1DinbswI_*(5Q_bxv0Y~+^bc5C+8 z41E0Cl;f1H=_PA*Nh<%TN`si}fjW&ypp zF-y@uSbG4;t+F2dtGC>mGp!F!J``E=&S*LUVLA@#v&DAS;vceS#k4`h3f!?#*_~?fDI2pht)&^X(>; z@590djj1}s6I0mo$jMA)~NE2{N{5iw)OtSreRAP~F!ySjw0Wz;^qAL36s zC^Hl}v&uEEy_uA!^RdNM z9F3XEsX6;~f1KaN9w)Q2;-t*xddc{j+Wa>pfxQJ!nyagQ;t8pobxKS=ByCSGir;mW zD9XlN1d2GIO=dS)8Ovu;rq{~Tsm~IBI~3iu(s7_lNFy@bAaSd7d-bgR@c2?^Z1Q;q z*pq8|T5-YwE+2YRn5U^gLQ2X>h9^Uj>(yl9uif3FU$T=cXvCg;3sO?55^i~EJ+Q}c z@i@TWTx)1>katE|?*BzFn*Z$E=7ii}N7UQ@vcVDCe>8LXfxWS%H+rg!%f&qg5igt-Gf(NIhuCfnINK9$ zeWx=;7v}M&ii)y^8)?9uWzxdg*87m!_jX70e9O_pP2d%}VZwfJr?z1N_cclm8?`y& zx|zvuu(~k>wye_U2O?#B4JiP#@(Cq&!bp+nzwRtRln^j=rs?{u!pa86q+G2ND3o=7g^(rO(dP^hvuJmxadD1u)~}u!^x(15u%{DHu@_5U9?+p8+BzqB7tt zd3uJDVCFO-I0EIZfQ$rPLi$2^FQw9^C+LZiyZgQz@05?7HNkWSI0GZ>nbe$D=2!=K z$L~4?ytW%!dd{2A?^8a{mVsJRzw{o@>*u~Or81i3j=MQku}YV@S)15-AIH^f!)t&d zpOK2OgYT_uGrDPU{rqkTDq79=%^Uq@N?dTbp-@*($Ug?>AfWYVhf6cl@vQMeC1B@V z8`FFz)_gwibW&O_+MXRi@G#8Ip)}Zs>XcTzJ9<{2_e`4OXlU`4dQrC=lTj|#oeO?k z=ZX`RA=Gg=fjh8%%u$bC_o&c|g6fuYb8{Cn3D+DX|rp{Qe;8@8D3MEH*?KAxm^C$JD*@x}!(+Xu(i`TrKJStGs#^zb&Eeyd^xGc1k&9g6!TcdE_}^+>mX zn@lR>3C2Wu+tylM)%n;pKJm!}%7fC8V40uP@!E$w2hbm9ocj4@?7H#RQ(|Kx@r8N) z4`CC^6fm2Q>=U54UzF4ys8a6l<+lQ1GLvvo9{Hkre8W2Gx{aAas_ z_XJPKQQXW&3!@ktsSqV%lvgD@?uY2kLGOxaTooc?v6CHjR7G86HICy}0u^4dl)}JLZuiHB1L(+Rk(ZF-wC0B_qr@Ow| zi_$MXe{{x;Xi^yx8m<}cq=hn65mUK?!^$oKt^~;IUef3H0F8=xGJ=K&7r3aKl}R}s zG9%J&-m`H`=sf#JlQvE{p>IRyC9u=RZ3V~M8L6#a#vHF;)vHsjy4AXOvUH}$sr%!# zxr^pHwpA|wZa-w?!}6b0&iOYa+T|K-r0>@d0pzeOMDE$qQdRMn0sTora_oF;cPJ6K zM`_ftO>-zteckGq!48W@t}ym@Pt3R;+zI_@V_<^Pq@7*o&L_gIwCqf)oCkdRiX3f_ zs>v?RpTM9yM#x@F%_!ngh!Z>ebtng+R-XPhs5l^B^wKTftvR@Fy<`1&iBq?T#Jp3> z>W3BZXBcs0ajw82{Bpaz9I~1mxipiBe%!zJ5CgcQ|A|} z$xd3&@9di|*Gzn(pq&NsW;37<%T1oC)_kwq~!FF(_DwcA1_?`nzeh$)ke#ham7){;3){9U$G zHYd2oVSVk$v!0^<*`EKz0-|F(;oy~_{v=I zuZ`tt*>Ydf<|9iVG%G{uZqu|G<;wtuSw_`mPj8+c>!Z^wHCz16{b#4kf1i!}IMvH2 z%*s;5I1i z#0kXEYN7a~CjlRgov4{zASsTLrqEaos4Xv?F{!I;>*Q3}P-xhKx8CtW9`B0E#K%>-!Ch9(c^BEG zr6$(5+!2OWP*%$l9)jn}r>3Ng#xni0UF&%EyTkEwIf9*LB{PwGxzq!3Y-BF86RQGF ze_rakgk#5@j<(yhfA(BOdmVaJ3$dryrF-m>g#0M2zUYraY_#t+VZ6Z_vM-6*NGew* zvOueoS>qc$oY1#`E^!pv8j2S-3yoLMR+@9Y@Z>D>&Frw4D zHq=`h{^+Y*C0Et50kUm+fb#)Ahvrr4#;3?7uCX~l5-Pvo0;U%(?#hF-i)_asN;RPQPpz3|-OLj$}MD5(^7^{RylYN`0s-IF# zaJ5IPoE}We>rN={x63*%q#wXBWKBciC77!=66caHMEb-$2jfK1pcPkRY)%b9<3Ev)qr z^2FPhM+6J1y>JJ*y(x7mE1xX-+`bjPv^O)ntfKM8m;|jzOrH$;wUXQDf6N$ZFOIyv zB16KZrfmTuQ(P3TosKcCt8=WN?{fS!)W1vN(b7-H7=gTg9&r{2SytxnBFbUOU)e;q z2t+E#bfSRlam_;QYuhZiy3uY_ekr$;a*auUvE}r=z$S1Hngdf2 zUe>E~lm5Ip7Nn&|7U{m2^2^9vE)vaX(MyrV<DM}a;dA4nj zUY)?#?6Uy*+91!~8w#UXZ@{K}IUPZghS^n+ER~RA6@%a>I=*(E4{fZ7S3$~iWE-fV z*AcE9*IE080tN-@?-R;|p4aaRmdf=*gu27ZL6-)h>D#uq7lCtRaF??;5-oqaKTga2 z=Iq|CnyTk-R%~18nt?@zd3tbrVnhYXjC6V$wQA)tKGY?zs8(>Yjo5c0qU$e(xVLdV z!uVb36xGyXGatYK1~ZlJI2a7(p6_B##HOPY@W!C0y!Q}Or*V`>3aP9AvX7pV88q-^ zMoOxs>@d$!{lA0t3CXD+1fQSF*hZAN*t&&w!3gDHW zHV_K3$cbXBi|&ZsHCr6EE|Gg|T@_lk0FYw7;4eeIa1iQURdXu%u&e&CK{rF5)Uc-3 zf;GAd;5k~2;f$q04&W2#QNU~;2!oFx)1BNjUm*TLOwG5fzF;XY$m(Ie?Ve4mD7S*! zNx+$SK#+%NVx(?&c%fq!-C^!4Dz*!S1eF4XsmB)-Vbe7r7Gn*%&`=H43@|P6{P03y z)=ET&AvwyHc6xL;2)pjttkHDj3q>m&y!%QNjscU~M2}8P%AOYhcadW%4J<>)D{nsJ zxb6A<+B8wW-rFmz;!?EG!Wp8Xdh$rpFNU`7gtT7hMoG>sWjCd1@07;pM`4!Fw=hri z_~N8Yo`zVB6@_%u(6>EefA#?xTY(Y3P1{Uwx1Y_!09uvOYM6VKRgY@TsQ@8oo&pj!I94QzB$!2-f@e!NVDS#Z^mv$ZJO(@BSvKM zYKvV1Q2-I0@I5=j@Y{G+n>n9}tvW)UiHS*HPw#D_a@@!!;@Qp{-EyP#&IqE$Go*dv z6$+Xz;b{^WeTwr>Il3PslDXyV@oI0j{LwiUE}fI><%$pqF1t}_2(ApBvT>TF3s+3N z1%)Av|M!Dk*94Js-J2z+z(E32Kt+2rE0%7|k(-Dw^=#pzO=f#ilg-7OE&{Jhw5MKQ z(x*+SX+M9;+tO0Ewc&eG5fjvMwM@>gf~X(>ZhY(!{a`9_o0+lur(N<9$ELSzF0FO)iI_RW)yLe;~K)DHkkjWl5q z8{UmSd8ED5m=vOfIYG~~st#z$FF)QIzOjWhb1T}1nI1I~=;s9JYlxch+Tf8^Iz>bR z{YNFG2Q5{dY0L-7jKb+==}iPDh8$wuanvk`(4BM%_cS8XLVfnXhkP*jq-Bq6XdgLE zQ-h)HfiqN?tU9k`7-VtcJQHDf(a`Uu{P zdnbC2i|;eU(Z}g34Ziltpd{d%@AU!^Y?Wb=nE(<%Bsog|9aKk!{hM6BKCMp$#nH{GZPUQunU(z-CL#$o2js%;73dz!Ro{hRyR|- zmjvRr)O9ue^lhjV8xm!~PSO-z*$krAU%sFb+SOSWsK)?b)meE$)Bk?d)k>-%y(a`wKgL-K+lH+K05ks;VaQiOP#$$D{YE{{W`28ex|z#oGz{n~P2rxm=9H)nu*LQEk@djPbiS9axH8 z^iUR-lCu2wfz>`8g+=kERMu;|Bi@(OPs>=>Aii|pa*;MqnE2g?n}ehg<7!%q{4t`} z#LeiHlg7II2oHBhq&b0{Y0CFo<6YlVq2^B^n@4QMy8);jzs>W2Sc#hsiP72v{~gW|n$xQMA*EKc zym%A(3Q;8oHaW$oHx>27g!V#;dVi3{PofyziQClss^V2SDPyZj%0|*_go|o^fB&%% zO@h7wYBc^h7_&EO;_aKT!!y)+h9`pHTR36(eR_dQD7B%q%T`U0y!* zH<1Tg${QN8sHjBil>&&FhFvgMoX`rywX3+=zfjV>xM_GowR2oDtn{Pnn*Z=(-QaqJ zOLv*c(s8O*BY!;o6kJ8#FbFFW+s7U&%9?{elDJ9%%sn2iAqyvg%9`?3>hXQ%GCb59 zN)gzw<4hgW)BXiyPC6y#CxR*s4k_~parJe=piz|m8CiF0z+vOM!8LSMx=IRK38s%GZwaEfZiQAGzu zv7!)d=r5=hb7Hg##zfn__jUiTVuSehuZZOdrxz7^z-da45`G;u9l82+CP+WRksSh^ zTDRAUFE(Nm(rv6zLU&>q+o>u901IR~eIeDj*1wk3wd}@qHX?f0%j%LB@ z2>O{En&gJW{e=e=1{8Zk8a_r=QAZ4Nx)UeN>W+?1xqXYWxqWK7(8Qa=XN{LWroj5ubqqAqFA^9%LfH22oW(JELF zG!Qsu=ZY&MRyxKfq(i18MN{kdJaC43l9SOJOsN#&1CW97s8ZxubJZO zW@(@J>2MEdMd1>FK$gL&>%22kX)=?NLz5%T#!YVZ@q;Eb8&_M{eZ>jtP@fF8o|Q{z z$;N=*-`Q|1D#2A3MIE}NPXxx`vr3SBvRy1Q1VmczVVizYto7N(cj6Mx!QWA z5@m&F#^$9WM?U52X{>`e8)&P6$gX`cvFe5?Jt~^@CVOiGPuOU|Vmx>MB{adZpIP-_ zacH3#|F23_XF*Rr?}F4_reSSa5r#ebrv{gQDez}ZjP+kyvo{$eln}ER%*|`k9&>u; z+IRsPI7vo$4^iHjt$&{(@GAHofIK>bd)9j$5zg#h0pDgN5B^VYXG-YF2+xl8Ia~^~ zkq{xM8O;Q*wzDMH*Zh(-ESHHr#s;u*afFCm+XD40@ zg*29?y7+-aST;#c6NgkCLRHkXWBK@Kl7FmP_zZeXJ@6>PFAcfGBRXf*<^(HpMcTN^Zaa>=$`TGMJu$-< zzoHL`wq`e_xFd#^l`sjD$Z>$ocsq~|SdoQa)#aIh8l=&a`h-?2)}ARp7qk1v|4K7Y77`e@Ir-WmGV{@m}?#^4|@ow`Fp9@ZVXd>@e@)N9#ypfvFy#il42`$ z!#%CA^-X5(iS03%V4`aZ@J@ja3=Ec+RaRCucTIBV#TNs$JkktkWf1n@+#xsq+R%zzLvY= za=?O~=Q0|k_oY(rx%=t(TCG-WG#8&whCPpg@d>>hiVK<$G!owR^ULY<^Ao5LHvcn^ z*sRZjQqsK%saK2a|I?j0E2f0W%BiooyGmG^-UK|?CF*f z(HX026$yCma{YIf~cqF z;~$FgnaX}YE1Vzn(XrExv?WyfK)5X&)BXHi78Tg1(u7@ea$}Nn_7AyYp7neam-;_en#@?w1|0@q@ z(5Yi}ykrKfDz$ro?exrjEYC`+MIFzD!28V$_p|ZYan)9{uEwLokby{nIqA)hG<&a} zsk*oOU>W^aH5@=y#(8@yM(K^RHSr5zl7=%wAQVQ8yo1X8Y8rdTRyV5XGDcx#5{mt9 zd|?97uCMZ0tMD+I&BHi(?@yTP(W;TkzylNhT7JSK8#-}*lT)y@hdKV~Sc3%(_{t#M zhfPo1iG@5WrRkObmTv2&=4LUzjXJ!vAv)=+u>*2K+I}we-K2iG#Z%h@~b=r&3FH)4+xkqmDFxRyhR0( z1~&HiORQ`>zS~a@n*Gx}hs)vz{k#hc>wFX9(tutDK4d;?LT7H%y`}-NJWefr5n4<_ zp4)_^0OdLZtgW46!s^lr!hyIcMuSTgXkw8SseU?{wVi9sx=fro`ea_yyju8h*Dt1{ z!PnO97owi={VzrJs8?AOR(9s`p?XPvmr6c6ibdebr03BaxFCUhGU}KHEVwQC`rX*1 zoK{t8e>M3a_p=qBf=P>{+JC$u`u<)i+~%AjLwj_YoJ?%0 z8p=B}12z%?#h)*D$66G!ySlr^dDVPc$9ZY=1{W(`RUq5>Tm8(l&j@0RqJ%ALAUUN( z7t5DAk0b8Y$brr==Ar@XUPdBIJn^~Zmu9tb7rWTo4tTlq*&~+rFr-US;NBHyn6P^; zdDVQxz_L-`TtPss9v4IfQ-m_j1?a{?re=(h;`qDB01q6-+^;^R1u^EWHlX635*$YY3VP4puRe)24Q5efb6(noI*R!NEIqAi9q1lXP`cl4ru~$q;S$W1Qvk@ z)gNH9x3ZIO&?& zu_*$e19V#wxBhITW*O0%@)Y^lTtlvG5+3Qm^n(S4d-ISwh*|V9T75J7jB5+6Jz`kx ziV%WqmR!LAR;d#^cm|jWZ^%?>ZK97!i`ClrMz{t>CL=^eWGedh2*;h@eyjn5%PTc#X7XTQ(EXOl|N6a#Bqy4&|D+@9lba2wGo?bmz?&qnera#u$^l}#TAqFO=x#m$<+%9tFcnO9h;P# z`uI42xSD*2xtG^cFEzt*WrN&;}@JT7mFiyrP;lz6ZQaggEQ1kd=OS}YHLLZP00 zj*}YG@D2UI(IZ)13e3MF(A&?gclS|XVrPR51e#VC6*brA-d^>BM zW@9mvkxJ)nVJ^^1H7ypEfM-a^q-JhZ)V(w_CEo~CwA2v2um|LQqT_AASd^asl4kv? zmlEp2k$r}5@HaSx`{}L;DSq)CPRP44-6VL_A863G#|K} z?a`Qjc{|dDgQGVAa}gRWj*hfkRf;mEF#8Y(0Olh3MwN?TVM8k7MkKk|$agLL9b=t% zjJlC)?ckZYWxAEhZFi3}=GRHXJc(|f&tT&VyIr<`uk}7=;vR`J+gt_7imze%X--^% zE)ZAYnp60e4lsMgFBhVCwCaw3K03J|)3B&qTvXOF{zD0^tHq~c=bpyWK}4^~fT$}r zHW0Ddv!%;9(-4ccn~bT2kq1_Py6GekbaA0ZbW*$^#bHzmN@4?(A@5#3R&jJFc8ZBo z;qqT+9glqBS_VK&`qMO%;udiC)!S}3MsX#x7uoNOU#t82YNO?Km~;XY76iVhY98eC zxz8JC+yw>v`IW!%eC1%KctP5v5ujsik!s|ZI)gKnqGUDN9Ci`xSMZ-xHh#-6Pml3& zfui!Iippx4?4K>GlD}WJD6l7k)#-z0*H>09d03&->eo3ps#n#0pAN>t@x!n?ueSGp zti)t3_xG|YrK(+SG=AWKiNvJ@HDnqrvsn^jLZDXD|*F7Fx5w9!yQFS3X3jtC0fS z`hdTCFNzWiGJe{D-+G?ooL z5-#uQL?_wt<|N^y6OW4b9L;_DVSVyMq4*X*XK0VtTW3_CC1IpMd$h6Dt}(H!>3aL1 z+AL+Dvlee^jLnYZNamd}Y05Y;VX~J~9^L<*WN6VnW9ISN z=q+{s&JuB4VG-NCt+y*WXjychRNNejQV10EaZweEAJ6Z?#2I>qdu=q0qmZOo++xt6 zof?9|6pE`?!%54+7X+k=14)UX^aw6Lk<454wPbrxeMM}W^fNN;1GU($D2A;Lb6ucl z{;)t{|K!L&$HDwatkLJq(ocihK7YbA>HgMZRm_nV<&2V}yZ!i}e540y5S8EMO&mCX z8NN)t@RyLjWjnIj@(U4+Ft_%_TnDpr;az? z^c(s)2mDYRF!^3n^q|r!BZ1>C{e0mrDAJxp`Ii5Rj@fH4ibu4jH?8VnGMl9L&HJ83 zk%5^id+kST-*T5&)p%+R3Kr@T7h?GR>Rs-o58rBIA(*6+i?qKs9#fo*NqT)Yi$%4| z?+0H}H1)3?a!!>St@Y>Q%u#Y^^3G^KXd3L?_N?b>QdCS_u4Iyc8QEV6>v9-b^^Yo! z<~)M$^mwoCB^ZGR<5>8{_@`AzA&=%mWdVw%xMC+{;77J3!z~>YPK{luC&~4d8*j(v zl_Ffh?%wp^ zPvBQ5$6Zt|ld7|$f^k)lU5^~cWxf?*na5w@*%!o2DH>DfrJlpi-h--MrbI zd2Gh=7~-j7oJran+=(AhR!i7U{pHlbGh30#Nw(ej-M6<31C+X(ey--<`}DBPxhmHy zo81!s!y;WcR6EtMk>IEdm0r<=I;cjeFkhe(Bch`?y>*Q9N`&uz09v4|LyUdGMo&b5dV5G5 z#S&Fq9v+VnFLBw5%E7}4S{!B@fwa9kSpDnM$AMBRe1Ky{I*Z7;ln+%&tX#Q6vGJ*2 zJd|fA-*;C|$R&ptY8#7@q|cieH3dA9BMvG@LdbHSNRjN8@v_*_=+?bbYmHPOPEljy z@jl|kfwHU-g7N^Hh!HMxqU4m}k_v#@%aN{$9p;phfI(|8Kg&~X%@VdKmzyRQR(3|j zGr#CH*ZBHAG6nmf@F6ABUbT=W)%>4tSC5t)!U@ZWIjwPCmx!lPuz69URCFRL{#N&2 zSVkC`*zZ=Nl*s#afb>5-?0=!O+H{VO1S>+B*VW`UXS_A5lIwZnv&~N-0)_le%85D1 z5xbVAVMwAadwQ0IJD(aGS3h#>ISBFwOP6kA^5G*%5g3fv`g?Mp6-Hg(W6{w+TF}j@ah33CD1LGwwI{ z=L|Jm*071$4*$w4G&cO&>ix-4*q4N>m+^`f(Bt#@LczM$he+g)#ye90}_n&jbFS=;9bRR{WAa?A+-Kr)8IgC#sKFrPLtj=TOm3-qS9Z z<4eAkKHff9s(5nlGi;E)@ zS_F7d1;iI-%_*4b(&3{SZ!8zJ4`g>uXz--NrcZi7qGuE1qSf{|7!3kdD4?}xr{vL{ za$F^UMTy_)HxI&%z%vCtuEZNrcC4NI^UtH83ecSA$+F~=4;BhC9=-D*g?taYJR*gBl(#rsS z_Yl*nXYx)6l*+TY(Tl~;bup?dSTa;GzSq49DD_GfSNRWsNv(}s%hgBM-kq*JiHroH zoa8Cs-Hc+(*?8wuOk-0_$;m1;9JdA0ONYgRiaRaHi6&30iEkK!NEo@tqf_N&%DSK#o!m6k;0V-Qif z+TyT!87}_5X@?(n9WR^AfSRM;j9-!jl6FsD?(Gz|ndZC!TR0-r#y6r3b*(q`J_vHk zV$D=KvEX&dCY705Sz5bW2pQ49Z>di4c<<8I?H+pN3fJEE1tjU0-SZLW>c*KC-j4rI zIcNS21=~LGiaa7JvR5jMLd0Zj5y`&C%rGX~U>M7gUA8Fd$x=in%Ls!RjCJf|DUTuh zScXBQERi*a#?se0J>Smz{tfT%_kI0v-Pbwyxz6>uKA3!t(e20TQc+t0+&ql8H`@i$ zZG8vp@Gf}~KpDyI*A~Zv{a-$YC^32z{8IeHtzI@{v%(6e8>r0~t;p=Tad%e%-lYI; znN<)|tt4LnnU1x+Vp-!*2^y+Gkhx~NI9&8zbDLqtsJFW-6!|8Ro#J1_kD!foj4O&? z=5aaG0TkicH7v!@=X~tO8pPxnsycPR5AZ8b)i+XmkSLy)H&pUA_g3vaw@POlL0fyf znp~Wyr{jr)QvV>e&O~E#e?1?zlJu%78IRg^ap2cV1D*y1>~kggwEVC5VO zu6~(d7P&u~nzXAL=fyC+x1Ny;bkb2vDlp*O6g#3o&oWL0eipF<6 zFRi?*3dP;G|23}_Yx}z&1knZjBvv>loh%L@6ekcOXl$b>YH)4m3vC{-Rcx{c1&uei zW|(b53%e7>ov3?TRO$&%zba;OWl-Qo%EZ(MTj&7P$r>WbS6e+eSo^#tc7}-(pr_~p zS=HResoF+$rY`Qu^cECRMnj>`o|uezEd|9~3{p1Ee?Yb|ltVFEloZAfYm*kQ^k}ydDKWO6+pkbkCiER}k-6;V#x{OT!fR$$ z?r7TP7&^`hN+vaTOrW!7HsevXPA1!w{#Q{xtmS{5xb>ETsn4PRNGuYfcxQROtet*Fjx30us&(w-nBJqi*tD5RGWitw3c=IxR%gmDExbt2#8Z4Ds z`3X%*1$&>icM02K^W|%_D&7G_j7;L@TxcpH^luuQVD7C(130;JY+ZRHnj>CDDtRUR z8Cji+byODka86EpY&8>mDJsZaNY12lPf)kULJtcta zB=2|VW4cfacB60N%THsQRWa zNukF4oi(xSjN%K$fnV(r4YnNTBMs{1M6m?}Kf&3P#ORf;*B0d9$N=}JR<6LtmeA#} zB;N?S!Gu_UWo`f6Kn6fSHsO^Nq16g4eTISWhO20N^mmC8i!8Tm5n02|eDqI~F&$$B zG&kJ&8@<6H+H`O{Wn;_BPt#hY!ChnD#O^!%%f~%R@UQpcHri*A3t~;X&d*3jlJ&t$ z!rdw>69?+Iw;hGP$S8ECAsGV}Dt*1hyvDK++C&bKewC!FI-#@E%<_E_k>|tEYig-* zM!9vL1Fg^vAAP4>xTDy;B{dexsB)jyzRjOLp9)Qs_Z;t7Wr+(eDK$n}J=i&7cJiZd zG0qGE1zQz@GJ_KBksTFJZ#)XBl0LDqW2R6kELqxTRIFW}E`hNK7~eW!Uth|_LEx&X z{8ho-)O>nK32FY|>>5LIY_^}LB_u6nK~LY=yZ2BeCxqfyv$~8Edz5JeePg4)`+RH7 zxx3Fa;1K$=QIuR|w=fmsSu3~RW;Q+BF)Iu+<3OvwA3fL`vr?sqpap!mzB?}ic<0*# zOtDx%w$!;xsyzdJUe$sEn25cykcFB#mpjzDUpj$a?zne1d+Iq6GCi{Q*TyKW+jI?N zT15&HlYrlerq|j@TYQ9ezK+=npIi3aHhCq~(z)FwETIiNNK)@(#Ijm0kps7M+4e>k zA`c>NzPd8+hN#^G@TQzfJQ&fY8!ol{0P2icHoA1+J&9&kq7 zP*94io1W@E_i4u`WISNRDSK;E#)x#etw5HIx%W*}Fu$xNso9mY-UC@5$~+-<)GOo& z6%}6wPV%>U$ERdwxLTaVqvn)8L|AT4b`NAzZSWKD#qjUQMx@{8Sy6Z<#4|l z_Ua>-gAo)hNN)QfWlybrE~#UyqkZq2hb#QZ5Lwgl*?(wM9kiJ!OE_7?E8y{Gzc(2D^Vc)> z?=_At*DAEMq8f~KL`2TI;YQH;+1Ac>C;!1a{j9Nt#s?}T`f&Erm*jv|GQ@&ww9<6= zIhfc>n{-5odqUVhPX;gKYbRxXX=tDQ;hTQlLIjNR(}!7v#;3$)jZn6~z6qIcFS<+Q+F@a)i_)pL4(^Ji zuh)i*0ZGIova>m%|`-zF)^r(HN)r4{v9E|!K@2XRxUXa?eoZB{+^ zO?rJ1o_T$Ap5oM_K{~LVqyio;_Bvz>@YcxuDFi5_q+<*%O6oT}Jo1YphNhMD%OSbh zNhz`bn+#z`@4QltpWHG|Eg^J`$0gDhkm=9)f5}D&1KOWhJx5T#`+olgLQVgc;|drk3|o$@_fJL4rpiwWZo0yOaJnoXs^T*OQg>io$&nrCMqODc{d1Fhkk`9?2`jKyPDf2kE#UA1< zV^s0oVu2qZ_|OqQcfA8?4X^sl&gL5y-0TvL<1dwMdwJ#!*rUv>CiPTImsx%|IRdjQ zjBg^mNLyS7Q3cBpviZ--I>77j`WwL@dgd7TScya^btC;S;SnDvppc_LjNhz!0VK!h+|I~F*_sZ6q!!q&UJH4uTCB*HoEG~X2Ag{6U zSN(-*G*QX!@}FCBxIMAKbF2AY6IU3bA(>)H8t>4L8Ya_`Wj^1?mhtVDZb7s9*?&xJ zyAK=XydTKZ)s^9{pt1 znMz!PlhlqQEuUCTWyu}a;BOO==4S1*q7f96<={PAYk>sz!i`K^68fGP-bRsqXr96(d`XkES-9$<+W@U!{ zA|GufXiw~n6iQNxK3@0_7huvK7hqHBV$?6ul>9uEO>SB1BW>a_10B-l)-m)~s5!t6JLHv1)|cBSMW>F*=mCYSpSu?L?wRV#KOZ zMU5gtA`~@a6S2ZK`uRMcKc4UL9N*vba~vf1o%_D8^SZ7x=k+>Y=j-y`Ku?Q>nVT5^ z0I)oH{16BLFxCM849RCt)6bYvj+p2#3_d`u2LM#>)dl*&NoRFkbpW6|_S}K(Df%(f zi^t|Z0KkRT<6njj&teAvz`^#(Lv`Z->tzyCh8}LZW4#P|QKMp2yX#ry!*@1sZd#np zy)K^osmb9s7~`OAeDkK^lWp1kTe%Js5&t;wzxjOoBPDEQGsFB&>oX_V?B)tIy>bke ztrWYw+p9nwb8zmgwK2q>}6o)vlHA0fW{VDClsi@*L4b>n4ZNGxY z@hzC|sO}U=@}KwjPaVvZ(f0j_!=InH<9T>Bk8QokZznjPEN1=yVOCaw4$lP^bP+HR zEb`jz+eWBQ6*E7O5X<4)YxCDYb-tb3ZK%ZCMk26YWf4PoeGI=+X*3x>l9jIQj)zJA z?P*)&GxLvb4 zgUPV;6shTOPkXSa?<;y|b2`?@3jaF4v$g}5ddM@8@!zgh)sio~QKICx)Zf9+>I<3v z9@stkX{*=IE^U#9$tOySvWsY#?!&>+K)eEyv>;M@s;`L!y;vFEK&}Vq_-J& zvE8ko@AZM=OCt!uJg6S6&mdClO5^&a0a42+M0a zIwTI$?|ng)Ta0=jF1!C3dE)+wD+qP`uQD4oyuJ1mp*@O|@=J@+orUhuK}QF|;$t_( zCxkLt(9ul&>w!SbFU;k07mjZ9Ob8%9jJ7EN9)Gn{ndejVea0rNSAi~e?ELh|1fjY@ z=t>qRyNO!Wya-4)@=KJok*HsOb>=RX@MT)ZM>X`I2FW1bm95rPsFPZ1-w{*8_r^y) z-6w`W>_B~YW2Vyi4B#e*-d9phA4tN;;OX3qgs`mM$i!ZCq8vJqpLs+R@HpRYMxhSh zogiE-=Q*A~RMIn%;r07qcQc4SZJCPDE{s5l8=*d0NdM!W)n?v@zGHq*caTtkUX1|*~@Ua0#EeSh{ZCUkd0{Q!h z+h*{epu2|OdPS9AtS=$FUhAh#cI|34@2?k{#@atC*zLdFE2$UPU%l4AE|-ay;q$cb zj$r{}3^DG$A(o2GJJX;cidaHMSL8RrA;W`KuU)R;?13TqQClB4Q6Q& zh9~Nik>3;9w!nu=W~g=H_l`y>qwOx!^6A?A9>uB7|y0qbDso-2AI244Ec zb9GqB*7L(_nSBbG2oTtAf6@;<&!_y@vw?lr^Wdzg>4Dwe8En_?CTD+&P0Y}b_&k}g z!(cN@`ao^aof|!h!ihZlCJ@ z7|u_L9;QCM77x=7%Z}sgy{(eiEvMNLK7!Zz4B_GM&45s%7A|)$r`EN)GbvBNC3(xu__;vwSAc z!eTu0OIJ7m@7T&!p$b2Rv(NiPS82|)y=Swi-H^v)Y~O6kSHZ3!zRDFXJ*)Nz2_SYR z7{(5tK63`Vvvvt{`k|z5n#>;6X0vQhWus5+=oYB13d7+KH={z|j;V(1!QF6fizDR; zV4tqut~vgYA)CfV{gDO+fNjRUA!}CR-J%rc@$YQ7!wWoU^ff#aYV+staDBb&FH z-_e+1?;uC!FEf1O1-7%9aXho#L73ln_}{3?(ImAN!$!CIqGUe77&i%L+@**R`fc}f zcEvcZ&vY$~+b#7Qqu*cg<~8RpD4!qT+jnd?bQcD{%J@I@p3T8YLG-X-pTgTpH_B$a8Q30flH96B6g_%pPU0`e_s-}f zW$}}44C{VX@p#}n0@j!B7>^O1-DkB7CaZ3$-qfF!m>@8t!^+d29$dNP4f2)&B1~<- zXwlD}LyJhd5-)2woz?N{3q`Ha?Ii`vD1*|^vE=aw&fw93196BlWyoj}f(EP2 zxcRKSIuC~JOh#Hk;xm;4fi}TX1H&(g)wZpQltr9T=co1^o4o+D`T!}~uJe}KqJQDt zlv{>l^w?<2oc=OdCC_&!ze=r;Y^G@IpTU&UJ9ts|GxRNp&5oS54Z)}cR}Q}4Ol@S; zdl~)8T~ENT@UBf5s(AI#+ouH;q9PnVp*I434JlXt#Rq7sy~k3gcac|NjY_mt+3Zk!K zB3j?W)DSa_w0v8waDG`)+lBF9qu!yw>KHNI_1c{ky&SQM?e1h zTX6g!Z6c86C5}DAGc{LGi&p|)UoQF5|H^*>flMtP~k?^TCWtZ3U zTaV{zBc8kk^~A*6%sFMi{94~2Yh!?z03~2i;?r6GST|3l7d)|MU4xJF%(O1~b*mNV z*>fyJ(f!Iu7gi<#Xe+1p;=RuDEM?U~>PY3Xxl7&hUZWV}_N8$`d|$ zn$ac=K`gtw;%2=W3K0DkG?XltB*9D5;wkd%PDWx*1K#~)s_nh;_G<5q2oLTJ?FEnsuPob{H7`G_d6DZjR6H5fxCu8AH9=Xm zWcgBd=duyNf~P3+i+CSGM&qR$&vIA*U)ZMOoKHB6R5?1Iv$R#viF_Q=%fa!QSMPrG z7$=8jroG46LVSbX9~!T1*|Z`Z2d;AN%6g`3x)fYjCFtw2D3VpsJNe*tHXYhkZY##LGg2W+ zp1yd-GBY#Nv0t=1xMfDHWqUr4eP*?3kTSNACJT1i-{wVO4D%YRU zh2>xLN4=bTnylJDHL;BGDKRP@c14WOUK49YdFgMbLnvEnzF;#`z#f;D#70?3g0jW zxo1F?C$C42sU1F>W_HJW>cN74e3r`NkhCt&RPu+LA_-ktD$YB3HD8DsAz49-uFE8> znRc?+T?p6GD7|ZdKDO0v@!PEaS-<`a$Qk|Tkfv6F{!AV$ ziWO%l2QLn}qEgTNwQY?O@&I9qY|KmgE@@p~JP~SHu!swR9A_xM1E*Bx6@~P(mMl~h z3Nzwwt2MFH!&_>lb6;&C!-ObxBCLs=QI0goeV}*@AW4mM;8^lF{yIHc(DR_1-WGimd7|g94kP{pdk|Y6ABnCMj?yssLpOrtLPr)zxLh4RNubA{li~h|^ zZkjULEqZ?*Z`daQw^R?}l9IhV@5PR|4*V(j2Wb(E5Z8*aLis%6{F@=RWu9dBo$Jj+ zAT6s$3DL7ZQxOm%hR98~2>_6PYM_q4L~`q*=O|Jk)qLxJGr*e=CW(0oi>ha2d|5U6 zY;TKF7>$yGWRRLxel6_Sm5AX8%K*RYoPPPA0o>{Ico5IMmwdkrU)i||rDJ3E7EE6? z;R4{k@qd3nqQk-<%m(cUs405zpm*dl;5X`?QA+kgnF$8&WCMm^KVdHK_eNR|jc(@F zc?&KCyX#~rrw$aX1G&qLF{kd^wfrR+JVZ*~T19Ts6er|gw!Oau0+dJ4|GYPU-mgZO zYQhhXJ>9(m+%BbRjrvR5c96;g_%S8hR54%ad#(+*%=R}Uzg$^~kooievFKg1 z0K*BfT}{gMvH@k6B#9AHYeoj~+JF>gE4pbJ0ARg;%s#wpp{3p}4lapAjb>ox_dGCf zdHq*G{qVjC{7@wZJ_H}|sj-F)m`U!B2zB(4tH>4N*l?F_!rHp+?YT|(L7X__%QJro zj)&Kv@I=&0?m!>H7czE~!*8s?&z6?LeOJY2wR9aDurM!Pqx9=xf>Sg0fTGqs$u4SOyn1UecI zH^T6~?D$KF+|+c&FLXCfkl!KU&E1Q5cs-ElhGIsuUxLQE>=x26?k6^ep&x$K3R0{J zPGEx1@P$5m$nN4GuXtyvO}k7 zrH&zh`Iwx!e?`KoK0iz^(yCGc@{2-pKc_^$dbm!CMxhy6>Y6=L(X2A-^u?_Fumts) zVI1x8i{gr0rq*i2zL`Ou75fN>2it@tAQeU9Lc-_R%(sZ`8 z%PtC_%!-g;d?#C4cV0kByB{IJc`N6{K2_BH<8U>#kK1^T!7}Q%*gcAG)7QMLfgDHL zpLE-l#=yqZ#;R+h)f0DFf);0~7bm=Nj?Q}(N3D~2Wxa{hu0fg*xPy;ACtszxu$4+6 zZtcx!)Yn>{;TMo@3kzXWk7RgbeneW)FVQz3Sc!`+z80kr8w19D=KsSfM;cj~ev#Q- zR4C(90t=S|+Eitb(}ZN*XJCWP-SF=FQ*Fntd*9Az!%&6Wb!*tOMV}BvtLXa9XaerW zCl@@jm|Lm}i%O1+K8c9W+H5$ZzLK`k9bYIO`L^@xLJOt9HGZX*B!thg4P^tj=EJyg&8muMfYj zd}r*75Z1!^SBfvyz=Vt-=!IS59a;_HD0(jhO^MJtd)vs-q|CPa`w|lC>rHlh+jCcs z_N9!EMXi(cP9{WQqiQC{NYTs9r|pD+G;7*NE(b0Df-jTm(&^q#s>QDM2QB+3sFjw9 z1qTbSREcHWSjo=;46gjx7Xh&Vfc6+A>my7 z>k%#M!{d^D;f8`sh_cu^F5}u+Ba9emq)N}HgwL?L)qutyGnY}I0E4tW$6V|?VjMp^N|5mVmKX%!N{iiO1>9cdWlD%hVcItgZtl?k^hjuZQlJorHjiEC0YreFe}NG9;}p=q5BFDmG9p2{aD(BDVLdG zrHITbGNlVE@^{s%q2HoXz@l^_X(M-Iqg-4F3IC|?bPBSos|D%Ke1Z@gtxxJYCK2g0 zV%y%(%pl^hKX-_AG4~8aW2Ixupz!{#=XvA8r{cJUR@+5V*yo^n>WAQg8b5CQuSZgZcIy?%rBT5u9S%(;6Cf*TeX zoKXYWY{B#!U|Yjwb6xPDq@Sn|kL9l%=V=eY`r7+Wsnfy-9>ik=3U~s#Dy9k576a0x4w$jI{-m$ovH-7%9KTfXJV6K`hZ4W@h0` zUSAx!%))!mq_UcZ_$k>j&Taxe>4d^k}weYtXMaTgAtfP$=C)gBy zH*7K$eL5R7>PIA-B-KptB|H$4m2*GSsVrrrk>1Fei)H`(q6Y-}pWjc0qmvg;elF9< z55|Z$)}Y$+$4*V>Z(zi&twzg6-w1cEK46d~_v_5u9odG7y%?R_3n(|PHd0ba#mRlg zMspt>Dno8lK1MWIeaA37zC|B>I~H}$m%^iphVXU75ISin_ci3>QmITx>DGeZf}|1B z+=_doH{4DsY~`okpuBpT-vdPt!g%PTMUeRQs?bs6J=+g|Xc=2Q(z91Lb&mQ51`McI z7RG0-WPQ@{^Vb@nKST656MSRS!GygxTZbY(jZw3*gX#Dc`|a4}g-W8(0_n-VNvW4J zS|t6u-1z*T!ql{?7e(7gKAp12-B;IRi_Hg%Oazb%=l9I{S!F%%NAGR0SbSNqJjAsR zrEcwrmX%DlE}&=jL%EyUL-;s_pA9R{sur4RLy_^VGKUCSccI(Ch6b~odkB5R)Os23kVyjpY(bfEu;%bFPN!77yVx?{^Iq_S zKJD29%O0}w<}0SQ0#e7p+WQ4hj|WxRo$p2SnP<8E(=qzmY501x%?Quych;(Q!-Uv> z@`U_Qac)`b?<8IUu1TRG6TsDB$>$Q@nGvz(-ns_o58CyyA>0dZFtLbd@WDOEM&Dh} z_rcaKMHmxZZ?m9Xj9XEoo?vTWZA52(p*-5kwiiQ`oqWM^V0$g+=Un#Xf>l$d9xH#37MJwg2~ktHRk>G*VJT+&E$&=VzE zAya6Lv(f_jSRE+kP9!hmYxle7(cc?IK8Z#$8@s@_p*mA)6;GP@zhb#u`PWSecl!7;~hK{xNSnv+0aWLSc(TO)VFrUXZG`x^&X?}0amB@e&hk6X$!lN5bE+R z18WxkP6yFh{O_=yq5PINK>`{=GbL+PaHYH|t=QXl0=Wx%US!u^zkv(ckfX_4C1YII z2L^@CnwcOJXO^L8-_ZFHGT5Lnn2{N8EqvCK<<60)n2K+L zzO|kUR6z(L)d2yqd!gK0$6lGIfM8<+d1Y12C#MUH^6Mc5)r|>LQ?v0uxiaI2!=M|9 z*w8$C#aW%o-gL{rF&2q&s!~TLzVo?1X);q`V6pSI$Kz=EOE_@8p`xT^J+EFI=21s` z0$Bz&@GM06`MWwle#EX&K*vaD9s_H5ZCp5RDc#vCw98n2a46)ZN%sfuGRj@W?N3CQPKm89(VMtcp4LmeBbu?%=Kc)G2PYA%6D5< z_nhYY_GoqtvJesw1d}l$5zWt%UuAT8gn~}6uEfgdj{cc2nuY0SJsIWy$^x)E^%u~d zo{haT>6OJ44<-?dgp7OTv-7iVgB7Emdgja1jPlj=>!_?_Vj-T{%9}gWLT9t%6wga{ ztjNCde)|-ocp`(u0?FF8FAk z5BtP#f1P3zv}$jnZrcPQ(`_lde7-YwjaMj zAuYWyjrKF$FehAOJb^uNv)GqubM_2!Fh7-Cb@YP|nXO=QE@s4|gE25k7|ArklkC#D zVy{>QfeY;_*zOBGcyJ}S{dT@d*U%i)Ei*gWM*2(o(~=DR0WGRa_bVu&VuHk-8_p-1 zt+`?2+Gyhg5@vO!R(Q&|k^~FHkBZ5WTmU=Pur@k4u@l&*4OA2W-$<=Mo+vJ)_X0uA zZ3ErmBYXqI!sRy%!1$N}^0_f8Uh(d597p~28KvhjvNUL;T&~O>CEa*KwlYj`lRH7z z>h2u;@cmXwAnogDk03Z5e>Z77FCRtx+3lR1oaUogX6YPZjLzE1Qc#s}2rDh9!vt9s zcpo50O$UWf62gXKMK3G>zv~EHe>!er0e7B0ljMi@= zQ$o_|>2e1CCS=P7|FJ?+LxC($KgayJO``sS7t@KY2cwdFg9l#rvQsHlLG{44h8n}5 zoKK6&$xl-J_Fd;+ddi6Azv}|A{nRcVmR9%^;$8GTtoEVOmw+7a_&sqP#*`z|!X5=) zljp9Ts$w2?%STc2p4BoptF+U(uQqNz$&-M0oMziDgyzn@<;ISoN?L+l@}0P?f8`8q zJz<5K5<9xs*o@&$M*0_VOo!&DDPmf1`5NcrFH-{Z+qgs|5iK}?ar-3$MNHf8d)joA z%qoj^AYzbOewB?Q(}#RTz8Dkw)U;)uclGsXeHwjry>>79z`Ti9zpR71dI&s|$sWUS z^O2X_7oG~I*W>QEN1oeR5=Y%{haCE;X+s3_*B9xx*PIK?GA-f*$p#zf6VIjD=x?0E%trGap{|4 zLiL01Y2@O)Iu2!`eu#}oX@7QPv2_n#b}O#BVuW;TF<@PE{I$nhFK@OuXEY*5{y zm6EP_RC+`rs$;5Fp-jGg!o;^X;ai5YW;GLmXD8ChZyy8$sT|w*a)(1rU0&l{gM~uW zgX&V;KSmIRipsQ|ldd|`&J>r_1@De0HyqxjsSvfZ#T4_!WG!6(Sh3v)8KvdL^@=VU z;;w9iKa1wpJ&&4{NJkEwWm$TrYYmQ=Utn&he3QJk(s(OCL7y)+IP1LY^%GLZEecq- zgZgzdP1r?nCT~xN$a)Zt=&BF^?Ot(H#=V8i;kLFoHUg~0(gdbf_+aJ6*oh)CF-sCL zx&^om;bTWQ5Iu#Aq#F9`UNN(~=c<;eeL0^SS>=kVav#0$Hb3NHS&W-x_L3@`K(Nvp zH`qN=u<81wKF)0NBbS$1e8gHZF-F2r`t%l$Y!z$K#am;{KNO;u?e0a5-aTZ^{9$f1ZK`jStrO`xLK4I6-3|U$G*jj< zIDRS*2QFX6rVf7F2x`PS^(>U`nd`ZzA8m7!tNNBpE=(U3Fe7Cg7i6N{jh=n0R2@vu zKCs;ra{2C8;=Au^K?z}bd&o|Ehx#5;C^;NnY|zk~CTm}_7`gO(@$Ru@H!pS$6j2-9 zx&bfr--c(D)7NN#z)BnOZ!Lr#Ax{54HD^+BCt7YgjE+~G3RhC?EfdKAP-de3ca_@zm4I^Oowx{o z_K}&c$*F8F_FEn=T2?w7=rk%i76%ZS1sd7AQhJhKD18#vNA0ini0EgHVQ`iI zX;G{$N6CKM?rlVt*|!scCKYY1820=1bwMU(D4{{-CU&|6j)1u&1*I1!PSGVFeOVz- zvsD{Da8k=vvZ8Y6lnou$-gGcL1#-UWPm4gpLHl7(6BLucSK5NDfE5lap#ZNw2dheu zlyMb}hL_g_?^ft# zoi%Q@IV3cBRzn(;j>pX0Sl$2o(%V_a=YB~dhC`GJx!!_Xgdvmd&dA_Zjd-3Xso0Z_`K38>!pNp3DD+I_+M^$MttE^$m2TOS$zf2_R5jN6zVPF?#7d)ptyMTo&zX*#GC)W8-ep=k)ju-@oLJ0-Xx5?Tqae zk~4Ts<}Y;qM+xPqfi2eNq{{g&coF$vF_OI8!et2pyf%fo8Y!v%yfyw8_>@bl$5(Qk zv%vZC7L9qFJo5Nr$&SUAx99v=15W%U@6SW_J&NU22cEn+!|De?qh*;Q?+9KbvMA}I;rVK!#qt2uGm&(rZd6Pbs(U# zW6l=3I&6MKqh^O@+$F69vI)w*lwX?Q=p=C7+a}vPRH@eB0_+2y!SxZ}=6H1aa8=$;H|LIH+P2DsgJs!vgA)h2{ z-4N0QWqvGHHrv`muP z2cNNOf3%!vQE$&b@L!cW`?G&UK(%ohMpoc-AwMqoYFvgsH($o1A z;hh^Yd-WXj`sn3oz6L^8E0^Q)bSR{9;#Y<~+*gkJgZyRs==O88E9=h^;TtqS^JE6h zb|Zt}zU>1t`a_P{PRr+|p&icfYA9Q!~UD$A3CSq+^Rkm^< z4Jxl*b;t&{-9m?8lkPfgVUU;fGVYb|D*3@RqlMkIWlkh7FK{v~SOl$2(AyHw1T zM5){l#bGt(u%b*)H7H!W#k)l}KQ*i2;Hxs0wTzFlfmp>_U-wS=v@ZRixF0%M_Xx== zI)4egm*I2&6%A{n|4~qfSjd?y8kZ#9b^K7Q22i{PnUcFrG+G?$jG0%o_5L!XT)MJU zSyFns3D{Ed=-t@LRcYV&3R(WrSrjQFr_P{O2?~sUob3Rj&Pya`d84tdzb6T{9S>h@ zrx(vI8cT0{`(=|tuJ3TO@sIA6)frm1$Xp7{!24u2TPF-QsXLyLPf=)vhT!QTpiNwG zXbfG3ZM>s35_qF_wxMn%V_W4@LsoGaSJnz$FHH{-pL!!B+(vP@)*nSiZGvqsbe`j`$j3E4TWZ44%0e&u*@#8lUv?7q1USXy}YT388$Wu?1FW}@cHOn z`NE;72GUQ(D{B_`eH&H_9(0xw~(_o2@4@XV^>n* zPgiEd?SloC3P#a=7sc;QZr$q$N3$9J3n#f?@eUBfGMkS)w1^(1-k&zVL3 z!^VEqtYMmJx>-d=m^1PpXgQYmD(l(5WJWo4dM*ngK{Dq27qC&62L9jsOcn9`*Gw?6HyJ#r7E8|h${$*|H7_it_S$T=LBq&Y0EJc4H!nupOK zK9rKTneJHW3Z9%M5W~p&M{6d2uJKZvqK7E0?T?KvCIpa}xw+(_mw};{-#t6EdDOn8 zmZX!xsF~7YkFfXX_dBUGU_xY=>SKG&G>8FAR@}4bQer8By0`6D9d;PMSHJhpxFIPb zeXrk|S9aj);9SyInkw^zXEPC(Bsp;9k1s0@_c<52l^=){Cf4(>Tktmt+2GH$Ka_2d zAG$h-O&Z@H;Zqy30*~mM{ge$}i*JaSTt#Sk`jAj~8S4cX0*u_SSm<6Bc7NZL+pWxg zXPs`;Ga6!nJo!*sYXp+GwkSMc`{n6TF=!FVN=NO-yj>bSf z+&jXwN9Aihjfh)a)p~4E+pAdC}rRD>w}Fc+~!E#?a)n|E_2aS@;`K> zek>UX>gEE0Do2Myh610C{Gf|VOWC}6uDUo8zaxAeVfQI|SBGAG2rO$zy6k^*`^`$P z>t67l5U*?)(!^!R`o?}^p>Sm#wnE0GFk*If3o>y5h!{^mZ(B+1s%|EJSTsb#O42?A z@up0$+RM0qv%PK9#xgzL89PsGTqU=zj_uDV)#}iQjb(>UO}_2m`GTUNg;q>hU1*Ef zcG<2Q&h=xm@-G9Zb`ZI(?8Z>n5)n?8-|)#^IgNU8l|v-+4zJK4O%eOks&6QhSub6i zJ-gU}`y3*W<|gvTitK-5D6C83Hm#?~$+`Y0gCj&$God>27N}8cE1w)nop0nU+4V_E zTpi%kU4o#9TOm7nWEnbgj==@|dC?~GTOxi%u*Xy*GtD_lrd@uC3oB?$4DbEA{l25( zky=n-t3rlri`3j$6Zat@y{o2ZUlRt>6IjVFiqWmlrCE#`=Nb7pT3+>9k#jNrtE{rg?jM{J0;QXPolZ1Qz?g<#a7n;6gxXX)?7Gq54h8cSA9+Lcd zAERel^ef$}LwU5ZntMj&ONm~ypQf*jwHE^dDwM20GD}T~siG(2c{qDcc;Fi>K8gs19tOUT zxUjF2(8!x-Rr`gdslDWA57tw(>>>x0j!IJBX(iSxF0cM z69NM&b(WrITjTRT{8QWTJf_wtUXN{txc!*2a+cT3)*(J8yaI9C|Jw8CKpFjYS!Q_)Tv}TCrpP!BO@f zL9bqqqUG!l5V`vsa~F*R9vR(N7|?(AGZGq`7HUygO|Icf5D%_qd30agO*kspB5>DHQbX7cVvt zj6LPx`7NEX;N{n&dp*9(2Tqve1NKs}86J4Y4=TkP!uDHF(I_GXJjM$4yn z4&-&EE@qH@@pt!od{>UdOBI(V@Sl4r#!{enppYef}%Dn!{yLAdS*zi);4Cd6n8eO67qeLrD>pKbZI*?qfa zy1qb}i=I5c*RUGTOE-QID`EGi({!0)ou_3YV`Hl_J`XYJypgF=kr4NHU)cJ~92jc` zlWs{cj(f3zYFQG99;>_ZRa5O?4Y5oYbzOh|YxySSVYceJLEZxc(r9tCEu(JRhj?tmb7`((RR?t%yzjmphZ;R&!_c zIVG%yHXKC*>ic1=g;V&UE$C^@^leJBWWm@A zF6DxO8&{sD-mUHQxZSEDp-vTYQ>J88IQq{9Fs|L~V2pJc{i1;JbEb12m+!uohzkfO zA;{sNx$(d==G0*;#z_gi;)X?L%*sMnuM<{z2y}*YyyXsvUhIKD?xT~Yr-(#rk_%^=kRw5*nPQ| zJ}5UGPJ#s1y(SP*N=a8vczlLk_T}yU;V~H*ymdc$X{_?x%?G_IwZi%*MSe>jVfi{) z{@oBmvk$3~uAaox37^M5Fe_ta_8s~JZa{MaoiI5jd+44srQyB*FQH}ZU-jVszrBn9 z7c2BXQd3_4_!7Zmgk?enG)B?st1e)rHnnoLcv)%rc(WVeanO{+JO%|HP|RhUV@5nk zAH!T~ZooYMri}I=X_By1WBR0^I&6pbCYc1bp{e_P=xbohrt z6S#YrZkE@ZLSH1=O@d@s?}YR#R7!_Xu0j!is%A~2Q9f0{c@S!Pp&xt3rKx|h;BV2A zG@xvTPrbBmH+~co?b8;nck@(&kgjo=wTelR2Pi}_I2>50;|inO0O;csR+n!@#1_x9 zhaoduM2te?!ybE>xB!2VN=|d%%YO@ZI@>?D+K38br0F{Kh|pHG>GZ1M_+bKWvKBA< zJl(a|t``>+ep)Ur8G|YSCSm70236fg^BS0Oi$#9W3YXRS2oaNtny0v6?d~BnlEwZl zUvo(lFG81cQQtD+M3%?!Pp-w$JZx|X0^D{sbdJ6d2`s#Olap?VJe<2xRux;ECk?Md zH;6dIF`I;Ts)W=6b(4ApA&2XZ(V*?fZM}sro}6#t4>Ik#co)k$J-6@`Az||!G14+> z#%?p<-Wii;x{cPQ;)pX?t%%jWFJHh1(4CBwr%h58343TW+){6+Jb2=g&ndJbcz`p+ zq)C8!`o=L-tV(Q zDrmL1<6q&r{RtJm_!?sJ+uG5l+zS3_ z7EYg0tlW(#)+SJDi{qq9})}kY!+gvj5JS zRu}1ZrL-81)GD?Ycb&!1ODG=}NAg}(+^S~A@#K0jTGZHxy{Kzk-mOT}WsMt{HI$mm z8jn+x*-XGwX20qx#@~%%PWE4A4ybVINke!}gj(DQYgDW!TBk^WDvk;Kmvd1wIhKaH z8E?PDTpjMZNc^+$=79BSEq|aUyUh3Ci`k_;LKmueRIMvjVd{^J&S)Fe6}R3CEAcf7 zQFN}?Y_u@2p1lg&r{aa1G7Pd#hoFPWc0G$5?;ae0&1ygPEHw$7J>FY1QU8Vixtr=Vto`-(6~j zXdf{BXWuei;QN<$rmtm@_X6wx)_4G3|MSM^C_{#KvSOO21j-cxMV>L7iJ8|~3&YG< zbbb;vLM^fWD0-}F#s!GGLywSund>$u(kw_3rfOVU=`5A~$=&`Y>_~NSFYN}{De0EQ z%dh7EOjnFj=w4Pj=zK`u7}#>kyX3_okGvLl*^nZK?^B}xGM*I8ang00{K)t>rKB*N z=tEs4eM?pdRNooa&JEr_sE2vdKDgVtW1Ai`I#NacE;0a7lu*|DzD&! z#z@bFE)xM%z1ew{!M%BOm*CGvLF3JK)eR*F?A32dGT6DO6|nan3}x`*89uEHC%T~k z)5B#|Vx-->R$`T^LO#;6x*CdO3s2Rc@PY*Jho{9CB>pL;W{WwaS{uS(`wiH!#fEcD zS+X0_6S13zDtkKecxup#SG$MZq<~%)MqSI?SA^fjEwK~DIL77mBulrc;Wg2B@%>On z%IiRWqJE|4#^}p;R6K=GV)$FPQ`5xWXKMRFkOy>iwgQ^C7=b?JIn?%b;>_Ewg}17g znj`>rf2ZT3Mp761U;?h|t>@c~xuGP4RC;}Vh>mo_+Jd^t%3gx0L}q@_k?z2psC&`$ zl4i&{&?mZf>w{*Qby?CIQP*lF5Wo1@WdCT*AYszAo;^Xah0|c#ms((^Q;_1(g=olj z*7w3N4Yr?NVR|b$^|>q0d|wch+z5`*AE=ha;)pO{_eUcN0i0Jws+E)-s<(bKhdZR6 z!Fg{zpe@Kh5kIFGKgc^`QGZZ!!E_PM6pBFmKD^=TBrgk5qo%>^SYsBE+0y>~@|(`! zHlJ_Y#2MXa&|Jo~J@5yMuNcpDQ-}|*)-q9&4x$UM;0)G!A~7i(H7z zRxI8Kor58fv4>)a73rT8+}ho_C8T*T>*+7$p+AZUYdd#(p54`nUbW1%hBOWP5&g9z z3*p*PWd)`QGBvh*Y`v_mey};3IBI1xllmJH*ym1WCR)c11MTPVK9Ab;CFFj}H%T~- z8!UhNhWa`Vh6z$AmSy6hHpM|7;@pz`5&@|iaY+hsTstb zlA}%{v?{4Z+gw`?I3eHlxl6NWrQyM-;(4PY?!Z5pd7ruc11#GaH9I|{6xu7E%u+s- zto~qdI|#6K?6J{|wh4P+uA4~NUj##LZNA#`3ovcpbjH1zwa^T<(pK;O?i%>#)u?6z zuEbVqrpHF7FKq3V#bR6|h|6_k)T>AKkubt7Yz-59&N+!n3;aVQoK$67#w`2WQIHeZ zJEIm=CYu|9iPZ-@ZdMaACyjxdtXa4>y%cmnZ+(~MDUt&~)iR28t9Lpu5xrLFcAzQ9 zW-ag5Cq)dDPT6+xEAy}bxxUo&mAHX-pBFd6g_)Z|`L}a}H(}}2-|T3&bS)lDi2=W| zr$Kd4K1g<53r!=kn+LszHIS9Vf5t1wWu(+^nhhv{vSslXe*67P-euW;ZCO<`N%SyB z>11tyXb&UCq5PehBtx#A6ajp{y$s%$O}#0kD3>># zTrdYlxXpN2xDf3(#^1iL;gc+=?F(g5y5-E5RM|fN@Gx{B|Kn~Cwxfxs_)wG?0gBY@ zN^c=?TzNh&x@jG{6+BYrdC#9`u&GV2TQ6y>z)k$lfVhW*#B7_=}5W>4@&w1bTd(Jt(_q^YiKa=de z*Is+Ad)@c7uIt*4l&JD7Z0EpOJMrX*_V>Q-wW@cYbyy5{={5EFjgXM$%EoK{;X%)< zR};NWvncOBI*kVGlsk@)?-#HgZ^KM;D*~1E8=3uO&zaxOwQOaOrtSOS%`APgx6A3S z^Nc9!VCaN3KN)qB7B%@Dronq46Itl zfKB4_GXBCl&xSIGm5e4^b9mCQ8kIGAaV4WUCbT`1t>akL$gL+w-fb#R4AuwfGIex< zjj`#%%FNPjk2K9BVv19x)Ix(G3!!fv8YHT%~_N< z&;RKS`8i3Yb*#Z*Bty;H)v)ObQI%EZf!6sENV$8ZI6QA^CA7LqXn8<{c6YMz7R~yS zfd)^%LQ3{)SiQMXSiVQo&=m~xnp|MbC79z@)u8W%n^_X&5|Gu4?s98&<~N0P!-^jX zM-DM5>4TZphr_JA_b!K+D%@E16IQJ7P%>VVzN#7S)swI&b>z;N^(3oZa`y8*Dt*S* zV@p*Z5hk3M4m~P0TOK=+@&OspIN#2iell`EqF zom_TlE{2POv7NH^$Z)U2p{+^OXoG-{D*C({TQ9|LxpV|1Hz|EV)y|e=AEdz@!hin_QQhj22v3r;rlJvcZw$vuPc%WL~m@0qf z{v=_vPk*~~3v(73_Pn~sKa8)ZSFa|$``i_*mv(o}CE|H%11a}X!^(q$LnUiXDIz0O7?tgDfbAdl11KMn_->ciS z4Bx6jHp05v#cQZ+?oTVNO%B*~UoVC_VXHoX`eS};Kqvz8w|w+bYjtalA0xET$EDp- z{mj7U>DQ)aFy7g zb0@FXIPZsL?IS4J0Q@D%@~wonc^Ii}1a8}xucRh(cAX>L^Lw!}Y~4f^vYnQ{NWU!H zx!+n|(~CgO=9}V{3QXJTF6^!HFf{^!NKJ8tTBOE_+YrwqFOeMWmH)iV{ySgHnX#b> zH$|Y$zMk{YGup`Fq(3PFr7>*azm8W5Xc63h?23MDib7`uCl~69a^QKO4JX_lO_{!{ z0_V(%NRJ50l>{3kSj0hGCk(nZfDO##EOa${wPv%TQF<#IT(KTyoMfl|CPwn|xwZ~1 zuBmIVtz0xB{M~({WCe3#2pyDG?N6WiaS(uO@_ve)?L~N?$HBD4NXa{UQCm%WBzqPO z_lHWvMdmYa5R-8=`HDwF(BJTitrk$~pfhFDxcXe)G8w_nTR0l)) zbF!CSVASUFJ~Zo2f0(Iir%ew zt!`E!iP_y#5X|NDMvo(%3T*KLECmpYfrDP81$N#x6$_)>)?atgIYaDl#ge|5;xR>Ih-IABxtGpd z8+k(a!`|?k8+R8~@xdD&rrS8j?Yh#CA`LxRsG$}pm7q4ksaQ(V^hpF~&yu<$Y_}exM2Y8%+wIBPxpTgh*&#-_{ zl}`=9op?8;Yhd1oJf~d1!25GNZ%tLy^B8e+fdkjP+I6^DGw_l6TD?q2IcmUJ^b-9; z@=QW&FCrVk$HDB4zug=a*_D}XDHhsV8cn^V#ohD{LCZp;Xb zE5h+K@>|XFy6NIBf^1pWf>HO9zr%X#!NZq00Rh+}U``cKdrFE-W8%!35+pqMfYXhx zjY&*ff*RkXktQ0(S&PGcV0P8XjZlV<$P8Y#?V~DgST+e ztd+uP6%{Ree>g3t!-)vWyK@&MxtywAsdn{-68!r<`dlZpu8ySW_*?ik*AD8QuQn&DL#W}gFbv? z%s??xFL&r~aN4bp4=-|50^V(dhqlkF@NuCPF=f)P6~&uH@IC`N>Ml3rw5~YhhwS*_ zk6ax-5G{bzMLKU}-rFjNUOi-m+vKJ8ZHbrM{8qrACG`Lu0L=XK$obHcVTsAG{F-V~ zVc`XCNTI3r@!+)ntTRvYoSFm9pQee2_=6B@cbMj+-mL;mu=eEER45|!^vQ77@8@u{ ztqv)ErKEQ|1{qG&UnPo~jmgS^ZhW`3v(MG+X z92T+$cVF+rzne)XTZn#MGvv2%YwB_n8ptn=bV24TX{>$l_lpusuIMEGLBhm|J-=kq z8)52xH2xZKpbVuF7;JA5wqZV8jm+FO9@!2iC*1S_Nt}|>4^FW=onv_byq2!}RL~PL zX8M!z^YPmR`WCmy`0xQG`PG#hGd#f+fwkKZgovOdY0@)WfJnyRcBiS5g zUlq0Yh-9{G?{yxRx`I~}mI=c1ruShE@1LTYJ13FLW$U7#}kF1Osb|v-Ctif@r7NRhDxm4=?MGE;u}sGPlOjZHw#Wffyebr~JHtNLT$O*r3Ry>WaKC&{nWLrPJ&_9+0e?M;DomN zKd*;lnHD3BTs=i@R%`6>T0xGVJ{1xDiSG?5NwkfL^|Ge*&FLvg7x>~mO7NAqGJ!D5 z*pb51qJxEMjYf@$Xlnh1qH4Z%v${1R3O^wQBVCHS^wf6Z!2544@JU>5V6}QtG2xT6 z4?g5&+3x<5b-Ki4B!<)`GI^f7Jl!Ca zkK4T19Y$N*t%0CjzQ}P}cc)<=Gy4yR;JqV0@7uJQHLaR2BQ5T?P;@Gh*AmW+&Im8r zAI>9nlLArt1vIaS6LxOu*KO^Arz;Rf{K5!>KuykHQjQ&qzV zRKz2I?t6&>V^VR0p?D}In3!e~oME3U{q!GNx?r@Iy%l{IO_G>v;_~G8U1ew@ZO|QcGFYV$u1h85EUHuR?rEU$l9RlU& zj=&w@7x4jB5SIZ;m~kP9)9juL6HYU_(q- zhe^r!-gQ{o+lAL?$-@^{>ciK(HoR5)Dx9zhse5&NnAJz;;-TlqKFG8 z5L~UDx|G?9F7aVEk6xJc%nTf9!rF(lSelfZMjK(f4H02JM9bd2()6Wn^`<*r3wiNs zHtG%umOROuOI}5lxRt(W9=aI@%z~Bm^!ulq4^FGNaEcDUuvE|c)6j*jxW4>mP|)#N z^JbKt$71SU=lIQGyxvJZm)?fbSsw4$_;~r9lH+6A%Z86tpFrX6H z$XhvFq9#pCp{J{w9St$ZslS4*pC#fbL>Bz+El(2cpxa*m&n+x3?2NDqRL{dAs>Pr@ zJYn|jO@J*;hm+Ja%c_iYkZ81|1H4@iRhEbL4V{S#xJGkL^%*AHKJNm@wPfU+?L zuyVrO8>CDg_;kIWce7+#-6`l`%~d#`q4JWt>v~=ec?em+bzs&nf3C@aob$4K>SUe< z4Tnd_8d#aQ(Y#Xp)wMPG_X*SY&pzpL@UTWS`47!&7bHl)?YbH9dy{L017ZvMJRhDq zu`|%dEJUz!B88+`h7=s#2quEe+2xBzY2|9>ss)CfP93WJA=a@Sy(<|E86+WjOs4%FCX`353(12erCd^XOp4>ZKe~00P$PJUDIJ%2 z+@;h?k8iv1q3;8i+E-F3G*u%tn}nYF`lqA>4}z+$Q3T~sVJbePD6qV&i{Q1AW?%Za zLNJ$tL=;*&hZ|>N7Pa_5#R+X=#X4fq!L`6Ex{Jgo_$=T!%?2>$bOoYWwWq!F4w$>VW60=@%c|8X3c z&D&ag;NOcUJr_E6I4~y?caYn^>kIyqDEJpor_%?{nQ}{wzB`z&dBXcpi*3hLrMlz6b;k(5PkEeGQM)aWezIR%S59#p!5x{xkp5r#A6XDNl zsmI1c-d>gAZwnv)u0M{lX^b>+TgS2uUgvPfDCB7#Br@11Av_VeG5>Ps@JVq0eoEmR zFEt#~W%s}X)j;og1Z;WYyhka(21(0#B-~uc7!P*m;yA6gv)+=useTV{SL&9l9Gy7FGoB?d>2bd$a4odXYi)uFhwZW&T= zT{@V2mP7yXPN0?pp>5^VtV?WHEzgIWK&yoAe{FVBac!>U1Cx)y!!JW1z2Vt|)_M24 zDdycWa8T`tmWMU_rHykcrq50|kM=5WREG!-96f*%9(7?byE9en_Nt|T@iDkQC7jAR zq|QjOJx+Sd>NEQ7Z?fr_H^G$NZd9#1FUM))osRpNx_PYD$`;iwFcYmF$0iRWqRadR z^Gy_&zNqTSDbD{OJmz`&Vnl$6b*QafF7%1vv;EfJOokkT7A)P+^%Yz2!$O~Gvx46{ z@_VgBJuU#)Eb-TNM(EgwH)?|cmV>}{2^ReoOG`4vL$kV^zOJLO$(`Rc*ZfyV9{{}u z`*rMNDt5?Qb?AwiVmS7-2T*MR1gPzr-{(rKV$mvK;Xm||GPb33UEaopv?PP4yH@#Ny4@2Pcbdod z7TGalnijTTMG0=I zNBqZs$j!E$-cQ`U31kL+22Ih+Q)SXhM}II6_T@ZC<$pbHS}IUyVUnS71+q-O8!p3f z`B${K>R8_M+i>KXQHS&&A2MsU9x_BPW0EU>uci$Kb_`W5yGwgsS)Y?r1e)_t+L2yuFLi;`6|f_mdhy^gqSow7@}7`#@KOI&$Csd5tcrgL9Fg6(#LkyY z^Le?u=K}V#wvjeKa$Tm{ugsLsZ^9Kj-kp>NDQZWkFZ6$^?fsn{bame4>+s6u?N}Uf z?orLuwB7GrgNn=exuP>m8Xmg}11-r1)fRfbhd1t8xt&>q4Xr|{-hY+)T^MEAVB7U= zq^_&~({d|3wlm_a?Cjx!+5Ew* zbs+Y>8;xi(9cJEbUvbZE3o2fpnj_uHoAO{EM&;ccQ-`mzr!sDT1W_jsBMP;Q@`0cTeCBTgrpo(JHm*aj8(1q<3O2pVPDJI!(i(AB-1qQB zEaue(tfc6cnQH?K)@8uTHNT6j>0Vh##ojn`+=5)0Zh?NpnSb({R810cBxzLT%KPK2 z+{r)KZw;5+hP0deWP0C)ROOvC7h%o?AqU@N*DA%Dk0a-lSqq zV=)VQ;ijdg4~v`x>7ix1DNw+y2vS&Aj>MLqKy`B68bc&lXp zcqjCB32f=>owC(AMsiiVJ;DXw*M&4G$oc0Gw7bT3vrb$B#N$rlcRY^zrya`d;ZKHT zLIp9CU)o?}hjZm(Ge|j3`OPE4p;jivb2x&B6}8l-$PY;lBL0#KlW~2I7(&M0^c#g* z?E}@pYB(UL6}9Wo0U&n15l@jvJILvf;awp=QAA8j$6)s{5WVCe7U4+jPOcF2(L$I8 z6~Pu7L0^cFo?Y5T#YWN#bLO#+8Kf^s!6~m;3Ay+C)(Gx8F&!H_g>s*U`?azb}P)P+YRT zVtc8Uq*~7hk1-<)>pvp4Au5iCqIbpo^7gRqm_%qTTY?LEs}#SlYxtdVF^+|(86*D` zeUWxS1fOKM0?FKyNGot#`;kwOYdHzkw=%*p#ES3U4k?6k%hK2m6(MZLYOIE*r>Vc- zlq}daPr|Uk?M}-3z3KEM%EkmA6OBF0Ry@$zA$v*Uw3jhGN#1T;D7mz@F<^#o|FoQ@ zRNRF5OY`muCwh=S<^=c9lao6G=W4~Tw@BRWqryhB8!*f5ekfR1GlBEE)qguuU3oek zlK3p=dm-dLtmy?YnaYIYgsM(&(alZo?L2{x>bialH(ttySv5)?1J)+!-cCyJINiBj zPUYMD?iUKbUKH^^g55oI^LGFIKfM5BCoKS6EKjjDoczUg-mYJKxP8j&Iwse!TDKzq E3+VoZ8UO$Q literal 0 HcmV?d00001 diff --git a/docs/secret-management.md b/docs/secret-management.md new file mode 100644 index 00000000..930f7d1d --- /dev/null +++ b/docs/secret-management.md @@ -0,0 +1,89 @@ +# Secret management + +## Introduction + +Secrets are sensitive pieces of information that should be protected from +unauthorized access. In the context of a Kubernetes cluster, secrets are used to +store sensitive data such as passwords, tokens, and keys. To allow for secure +and efficient management of secrets, we are using HashiCorp Vault, a tool that +is designed to manage secrets and protect sensitive data. Vault provides a +centralized way to manage access to secrets and encryption keys, and it also has +the ability to generate dynamic secrets on demand. This document provides an +overview of the secret management process and the role of Vault in securing and +managing secrets in the Kubernetes cluster. + +## Vault architecture + +Vault is a highly available and distributed system that is designed to provide +secure storage and management of secrets. It is built on a client-server +architecture, with the server being the central component that stores and +manages secrets, and the clients being the applications and services that access +the secrets. The server is responsible for authenticating clients, authorizing +access to secrets, and providing encryption and decryption services. The server +is also responsible for generating dynamic secrets on demand, which are +short-lived and are automatically revoked after a certain period of time. + +Current configuration allows vault to inject secrets into pods using a sidecar +container that runs the Vault Agent Injector. The Vault Agent Injector is a +mutating webhook that intercepts requests to create or update pods and injects +secrets into the pod's file system. This allows clients (hosted applications) to +access secrets as files, which is a secure and efficient way to manage secrets +in a Kubernetes environment. + +The following diagram illustrates the workflow of the Vault Agent Injector and +how developers can manage secrets of hosted applications: +![Developer workflow diagram](img/Vault-argoCD-workflow.svg) + +## Secret management process + +The secret management process involves the following steps: + +1. **Secret creation**: Secrets are created and stored in Vault using the Vault + CLI or API. When a secret is created, it is encrypted and stored in the + central Vault server. + +2. **Secret retrieval**: Applications and services can retrieve secrets from + Vault using the Vault CLI or API. When a secret is retrieved, it is + decrypted and returned to the client in a secure manner. + +3. **Dynamic secret generation**: Vault has the ability to generate dynamic + secrets on demand. This means that instead of storing static secrets in + Vault, Vault can generate short-lived secrets that are automatically revoked + after a certain period of time. This provides an additional layer of + security and reduces the risk of unauthorized access to secrets. + +4. **Access control**: Vault provides fine-grained access control to secrets, + allowing administrators to define policies that specify which clients can + access which secrets. This ensures that only authorized clients can access + sensitive data. Currently, we are using the Kubernetes authentication method + to authenticate hosted applications and authorize access to secrets. As for + the human users, we are using the Github authentication method to + authenticate and authorize access to secrets. + +## Create, read, update, and delete secrets + +Vault provides a UI service to manage secrets. The UI service is a web-based +user interface that allows administrators to create, read, update, and delete +secrets. The service also provides a way to manage access control policies +and audit logs. The service is accessible through a web browser and is +protected by the same security mechanisms as the Vault server. + +### Steps + +1. In order to gain access to the Vault UI service, you need to have the + appropriate permissions and access to the Vault URL. It is currently + configured to give access to any member of the `ai-cfia` organization on + Github. +2. Generate a personal access token on Github and use it to authenticate to the + Vault UI service. The scope of the token should be : ![PAT token + scope](img/pat-token-scope.png) +3. Gain access to the Vault UI service by navigating to the Vault URL in a web + browser. You will be prompted to authenticate using your Github PAT token. +4. Once authenticated, you will be able to create, read, update, and delete + secrets using the UI service. Simply navigate to the PV secret engine and + follow the path to your applications secrets. The PV secret engine is a + key-value store that allows you to store and manage secrets for your + applications. ![PV secret engine](img/pv-secret-engine.png) +5. Once in the directory of your application secrets, simply click on 'create + new version' and you will be able to add, update, or delete secrets as + needed. ![Create mew secret](img/create-new-secret.PNG) From fb601e15902268af7c2cdee002c345178ce531d3 Mon Sep 17 00:00:00 2001 From: Jonathan Lopez Date: Tue, 5 Mar 2024 13:54:03 -0500 Subject: [PATCH 11/20] Issue #56: Rename Vault-argoCD-workflow.svg to vault-argocd-workflow.svg --- .../{Vault-argoCD-workflow.svg => vault-argocd-workflow.svg} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename docs/img/{Vault-argoCD-workflow.svg => vault-argocd-workflow.svg} (99%) diff --git a/docs/img/Vault-argoCD-workflow.svg b/docs/img/vault-argocd-workflow.svg similarity index 99% rename from docs/img/Vault-argoCD-workflow.svg rename to docs/img/vault-argocd-workflow.svg index 273f9d18..2d37cd14 100644 --- a/docs/img/Vault-argoCD-workflow.svg +++ b/docs/img/vault-argocd-workflow.svg @@ -18,4 +18,4 @@ - Sync deployment version from the podFinesseRepositoryDeveloperCommits codeGets new versionthrough webhookbuild and push a new semantic versionArgoCDFinesse namespaceFinessePodGithub Container RegistryGithubAzure Kubernetes ClusterTriggers new pipelineAccess Vault UI to update secretsVault agentInject secrets as a newversionCopy secrets to shared volume mount Reads new secrets as env variableGithub actions \ No newline at end of file + Sync deployment version from the podFinesseRepositoryDeveloperCommits codeGets new versionthrough webhookbuild and push a new semantic versionArgoCDFinesse namespaceFinessePodGithub Container RegistryGithubAzure Kubernetes ClusterTriggers new pipelineAccess Vault UI to update secretsVault agentInject secrets as a newversionCopy secrets to shared volume mount Reads new secrets as env variableGithub actions From 8a0659497c51ccf252a14bc58bf96ff51c02a0e1 Mon Sep 17 00:00:00 2001 From: SonOfLope Date: Tue, 5 Mar 2024 15:44:44 -0500 Subject: [PATCH 12/20] Issue #56: Adds crds for vault config operator. Still needs a fix because it currently doesnt have enough credentials to access vault --- .../aks/system/vault/base/kustomization.yaml | 6 + .../system/vault/base/kv-secret-engine.yaml | 9 + .../aks/system/vault/base/policies.yaml | 43 + kubernetes/aks/system/vault/base/roles.yaml | 50 + .../helm/vault-config-operator.values.yaml | 14 + .../aks/system/vault/helm/vault.values.yml | 1178 +++++++++++++++++ 6 files changed, 1300 insertions(+) create mode 100644 kubernetes/aks/system/vault/base/kustomization.yaml create mode 100644 kubernetes/aks/system/vault/base/kv-secret-engine.yaml create mode 100644 kubernetes/aks/system/vault/base/policies.yaml create mode 100644 kubernetes/aks/system/vault/base/roles.yaml create mode 100644 kubernetes/aks/system/vault/helm/vault-config-operator.values.yaml create mode 100644 kubernetes/aks/system/vault/helm/vault.values.yml diff --git a/kubernetes/aks/system/vault/base/kustomization.yaml b/kubernetes/aks/system/vault/base/kustomization.yaml new file mode 100644 index 00000000..c23d953c --- /dev/null +++ b/kubernetes/aks/system/vault/base/kustomization.yaml @@ -0,0 +1,6 @@ +namespace: vault + +resources: + - policies.yaml + - roles.yaml + - kv-secret-engine.yaml diff --git a/kubernetes/aks/system/vault/base/kv-secret-engine.yaml b/kubernetes/aks/system/vault/base/kv-secret-engine.yaml new file mode 100644 index 00000000..847e2911 --- /dev/null +++ b/kubernetes/aks/system/vault/base/kv-secret-engine.yaml @@ -0,0 +1,9 @@ +apiVersion: redhatcop.redhat.io/v1alpha1 +kind: SecretEngineMount +metadata: + name: kv +spec: + authentication: + path: kubernetes + role: config-admin + type: kv diff --git a/kubernetes/aks/system/vault/base/policies.yaml b/kubernetes/aks/system/vault/base/policies.yaml new file mode 100644 index 00000000..52a442cf --- /dev/null +++ b/kubernetes/aks/system/vault/base/policies.yaml @@ -0,0 +1,43 @@ +apiVersion: redhatcop.redhat.io/v1alpha1 +kind: Policy +metadata: + name: secrets-writer +spec: + authentication: + path: kubernetes + role: config-admin + policy: | + # create secrets + path "kv/data/{{identity.entity.aliases.${auth/kubernetes/@accessor}.metadata.service_account_namespace}}" { + capabilities = [ "create", "update", "delete" ] + } +--- +apiVersion: redhatcop.redhat.io/v1alpha1 +kind: Policy +metadata: + name: secrets-reader +spec: + authentication: + path: kubernetes + role: config-admin + policy: | + path "kv/data/{{identity.entity.aliases.${auth/kubernetes/@accessor}.metadata.service_account_namespace}}" { + capabilities = [ "read" ] + } +--- +apiVersion: redhatcop.redhat.io/v1alpha1 +kind: Policy +metadata: + name: config-admin +spec: + authentication: + path: kubernetes + role: config-admin + policy: | + path "sys/*" { + capabilities = ["create", "read", "update", "delete", "list", "sudo"] + } + + path "auth/*" { + capabilities = ["create", "read", "update", "delete", "list", "sudo"] + } diff --git a/kubernetes/aks/system/vault/base/roles.yaml b/kubernetes/aks/system/vault/base/roles.yaml new file mode 100644 index 00000000..ec12215e --- /dev/null +++ b/kubernetes/aks/system/vault/base/roles.yaml @@ -0,0 +1,50 @@ +apiVersion: redhatcop.redhat.io/v1alpha1 +kind: KubernetesAuthEngineRole +metadata: + name: config-admin +spec: + authentication: + path: kubernetes + role: config-admin + path: kubernetes + policies: + - config-admin + targetServiceAccounts: + - default + targetNamespaces: + targetNamespaces: + - vault +--- +apiVersion: redhatcop.redhat.io/v1alpha1 +kind: KubernetesAuthEngineRole +metadata: + name: secrets-writer +spec: + authentication: + path: kubernetes + role: config-admin + path: kubernetes + policies: + - secrets-writer + targetServiceAccounts: + - "*" + targetNamespaces: + targetNamespaces: + - "*" +--- +apiVersion: redhatcop.redhat.io/v1alpha1 +kind: KubernetesAuthEngineRole +metadata: + name: secrets-reader +spec: + authentication: + path: kubernetes + role: config-admin + path: kubernetes + policies: + - secrets-reader + targetServiceAccounts: + - "*" + targetNamespaces: + targetNamespaces: + - "*" diff --git a/kubernetes/aks/system/vault/helm/vault-config-operator.values.yaml b/kubernetes/aks/system/vault/helm/vault-config-operator.values.yaml new file mode 100644 index 00000000..f509eaff --- /dev/null +++ b/kubernetes/aks/system/vault/helm/vault-config-operator.values.yaml @@ -0,0 +1,14 @@ +enableMonitoring: false +enableCertManager: true +env: + - name: VAULT_ADDR + value: https://vault.vault:8200 + - name: VAULT_CACERT + value: /vault-certs/vault.ca +volumes: + - name: vault-certs + secret: + secretName: vault-ha-tls +volumeMounts: + - mountPath: /vault-certs + name: vault-certs diff --git a/kubernetes/aks/system/vault/helm/vault.values.yml b/kubernetes/aks/system/vault/helm/vault.values.yml new file mode 100644 index 00000000..2acfc80f --- /dev/null +++ b/kubernetes/aks/system/vault/helm/vault.values.yml @@ -0,0 +1,1178 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# Available parameters and their default values for the Vault chart. +--- +global: + # enabled is the master enabled switch. Setting this to true or false + # will enable or disable all the components within this chart by default. + enabled: true + + # The namespace to deploy to. Defaults to the `helm` installation namespace. + namespace: "vault" + + # Image pull secret to use for registry authentication. + # Alternatively, the value may be specified as an array of strings. + imagePullSecrets: [] + # imagePullSecrets: + # - name: image-pull-secret + + # TLS for end-to-end encrypted transport + tlsDisable: false + + # External vault server address for the injector and CSI provider to use. + # Setting this will disable deployment of a vault server. + externalVaultAddr: "" + + # If deploying to OpenShift + openshift: false + + # Create PodSecurityPolicy for pods + psp: + enable: false + # Annotation for PodSecurityPolicy. + # This is a multi-line templated string map, and can also be set as YAML. + annotations: | + seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default,runtime/default + apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default + seccomp.security.alpha.kubernetes.io/defaultProfileName: runtime/default + apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default + + serverTelemetry: + # Enable integration with the Prometheus Operator + # See the top level serverTelemetry section below before enabling this feature. + prometheusOperator: false + +injector: + # True if you want to enable vault agent injection. + # @default: global.enabled + enabled: true + + replicas: 1 + + # Configures the port the injector should listen on + port: 8080 + + # If multiple replicas are specified, by default a leader will be determined + # so that only one injector attempts to create TLS certificates. + leaderElector: + enabled: true + + # If true, will enable a node exporter metrics endpoint at /metrics. + metrics: + enabled: true + + # Deprecated: Please use global.externalVaultAddr instead. + externalVaultAddr: "" + + # image sets the repo and tag of the vault-k8s image to use for the injector. + image: + repository: "hashicorp/vault-k8s" + tag: "1.3.1" + pullPolicy: IfNotPresent + + # agentImage sets the repo and tag of the Vault image to use for the Vault Agent + # containers. This should be set to the official Vault image. Vault 1.3.1+ is + # required. + agentImage: + repository: "hashicorp/vault" + tag: "1.15.2" + + # The default values for the injected Vault Agent containers. + agentDefaults: + # For more information on configuring resources, see the K8s documentation: + # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + cpuLimit: "500m" + cpuRequest: "250m" + memLimit: "128Mi" + memRequest: "64Mi" + # ephemeralLimit: "128Mi" + # ephemeralRequest: "64Mi" + + # Default template type for secrets when no custom template is specified. + # Possible values include: "json" and "map". + template: "map" + + # Default values within Agent's template_config stanza. + templateConfig: + exitOnRetryFailure: true + staticSecretRenderInterval: "" + + # Used to define custom livenessProbe settings + livenessProbe: + failureThreshold: 2 + initialDelaySeconds: 5 + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 5 + + # Used to define custom readinessProbe settings + readinessProbe: + failureThreshold: 2 + initialDelaySeconds: 5 + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 5 + + # Used to define custom startupProbe settings + startupProbe: + failureThreshold: 12 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 5 + + # Mount Path of the Vault Kubernetes Auth Method. + authPath: "auth/kubernetes" + + # Configures the log verbosity of the injector. + # Supported log levels include: trace, debug, info, warn, error + logLevel: "info" + + # Configures the log format of the injector. Supported log formats: "standard", "json". + logFormat: "standard" + + # Configures all Vault Agent sidecars to revoke their token when shutting down + revokeOnShutdown: false + + webhook: + # Configures failurePolicy of the webhook. The "unspecified" default behaviour depends on the + # API Version of the WebHook. + # To block pod creation while the webhook is unavailable, set the policy to `Fail` below. + # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy + # + failurePolicy: Ignore + + # matchPolicy specifies the approach to accepting changes based on the rules of + # the MutatingWebhookConfiguration. + # + matchPolicy: Exact + + # timeoutSeconds is the amount of seconds before the webhook request will be ignored + # or fails. + # If it is ignored or fails depends on the failurePolicy + # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#timeouts + # for more details. + # + timeoutSeconds: 30 + + # namespaceSelector is the selector for restricting the webhook to only + # specific namespaces. + # Example: + # namespaceSelector: + # matchLabels: + # sidecar-injector: enabled + namespaceSelector: {} + + # objectSelector is the selector for restricting the webhook to only + # specific labels. + # Example: + # objectSelector: + # matchLabels: + # vault-sidecar-injector: enabled + objectSelector: | + matchExpressions: + - key: app.kubernetes.io/name + operator: NotIn + values: + - {{ template "vault.name" . }}-agent-injector + + # Extra annotations to attach to the webhook + annotations: {} + + # Deprecated: please use 'webhook.failurePolicy' instead + # Configures failurePolicy of the webhook. The "unspecified" default behaviour depends on the + # API Version of the WebHook. + # To block pod creation while webhook is unavailable, set the policy to `Fail` below. + # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy + # + failurePolicy: Ignore + + # Deprecated: please use 'webhook.namespaceSelector' instead + # namespaceSelector is the selector for restricting the webhook to only + # specific namespaces. + # for more details. + # Example: + # namespaceSelector: + # matchLabels: + # sidecar-injector: enabled + namespaceSelector: {} + + # Deprecated: please use 'webhook.objectSelector' instead + # objectSelector is the selector for restricting the webhook to only + # specific labels. + # for more details. + # Example: + # objectSelector: + # matchLabels: + # vault-sidecar-injector: enabled + objectSelector: {} + + # Deprecated: please use 'webhook.annotations' instead + # Extra annotations to attach to the webhook + webhookAnnotations: {} + + certs: + # secretName is the name of the secret that has the TLS certificate and + # private key to serve the injector webhook. If this is null, then the + # injector will default to its automatic management mode that will assign + # a service account to the injector to generate its own certificates. + secretName: null + + # caBundle is a base64-encoded PEM-encoded certificate bundle for the CA + # that signed the TLS certificate that the webhook serves. This must be set + # if secretName is non-null unless an external service like cert-manager is + # keeping the caBundle updated. + caBundle: "" + + # certName and keyName are the names of the files within the secret for + # the TLS cert and private key, respectively. These have reasonable + # defaults but can be customized if necessary. + certName: tls.crt + keyName: tls.key + + # Security context for the pod template and the injector container + # The default pod securityContext is: + # runAsNonRoot: true + # runAsGroup: {{ .Values.injector.gid | default 1000 }} + # runAsUser: {{ .Values.injector.uid | default 100 }} + # fsGroup: {{ .Values.injector.gid | default 1000 }} + # and for container is + # allowPrivilegeEscalation: false + # capabilities: + # drop: + # - ALL + securityContext: + pod: {} + container: {} + + resources: {} + # resources: + # requests: + # memory: 256Mi + # cpu: 250m + # limits: + # memory: 256Mi + # cpu: 250m + + # extraEnvironmentVars is a list of extra environment variables to set in the + # injector deployment. + extraEnvironmentVars: + VAULT_ADDR: https://vault.vault:8200 + + # Affinity Settings for injector pods + # This can either be a multi-line string or YAML matching the PodSpec's affinity field. + # Commenting out or setting as empty the affinity variable, will allow + # deployment of multiple replicas to single node services such as Minikube. + affinity: | + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: {{ template "vault.name" . }}-agent-injector + app.kubernetes.io/instance: "{{ .Release.Name }}" + component: webhook + topologyKey: kubernetes.io/hostname + + # Topology settings for injector pods + # ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + # This should be either a multi-line string or YAML matching the topologySpreadConstraints array + # in a PodSpec. + topologySpreadConstraints: [] + + # Toleration Settings for injector pods + # This should be either a multi-line string or YAML matching the Toleration array + # in a PodSpec. + tolerations: [] + + # nodeSelector labels for server pod assignment, formatted as a multi-line string or YAML map. + # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + # Example: + # nodeSelector: + # beta.kubernetes.io/arch: amd64 + nodeSelector: {} + + # Priority class for injector pods + priorityClassName: "" + + # Extra annotations to attach to the injector pods + # This can either be YAML or a YAML-formatted multi-line templated string map + # of the annotations to apply to the injector pods + annotations: {} + + # Extra labels to attach to the agent-injector + # This should be a YAML map of the labels to apply to the injector + extraLabels: {} + + # Should the injector pods run on the host network (useful when using + # an alternate CNI in EKS) + hostNetwork: false + + # Injector service specific config + service: + # Extra annotations to attach to the injector service + annotations: {} + + # Injector serviceAccount specific config + serviceAccount: + # Extra annotations to attach to the injector serviceAccount + annotations: {} + + # A disruption budget limits the number of pods of a replicated application + # that are down simultaneously from voluntary disruptions + podDisruptionBudget: {} + # podDisruptionBudget: + # maxUnavailable: 1 + + # strategy for updating the deployment. This can be a multi-line string or a + # YAML map. + strategy: {} + # strategy: | + # rollingUpdate: + # maxSurge: 25% + # maxUnavailable: 25% + # type: RollingUpdate + +server: + # If true, or "-" with global.enabled true, Vault server will be installed. + # See vault.mode in _helpers.tpl for implementation details. + enabled: "-" + + # [Enterprise Only] This value refers to a Kubernetes secret that you have + # created that contains your enterprise license. If you are not using an + # enterprise image or if you plan to introduce the license key via another + # route, then leave secretName blank ("") or set it to null. + # Requires Vault Enterprise 1.8 or later. + enterpriseLicense: + # The name of the Kubernetes secret that holds the enterprise license. The + # secret must be in the same namespace that Vault is installed into. + secretName: "" + # The key within the Kubernetes secret that holds the enterprise license. + secretKey: "license" + + # Resource requests, limits, etc. for the server cluster placement. This + # should map directly to the value of the resources field for a PodSpec. + # By default no direct resource request is made. + + image: + repository: "hashicorp/vault" + tag: "1.15.2" + # Overrides the default Image Pull Policy + pullPolicy: IfNotPresent + + # Configure the Update Strategy Type for the StatefulSet + # See https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies + updateStrategyType: "OnDelete" + + # Configure the logging verbosity for the Vault server. + # Supported log levels include: trace, debug, info, warn, error + logLevel: "" + + # Configure the logging format for the Vault server. + # Supported log formats include: standard, json + logFormat: "" + + resources: {} + # resources: + # requests: + # memory: 256Mi + # cpu: 250m + # limits: + # memory: 256Mi + # cpu: 250m + + # Ingress allows ingress services to be created to allow external access + # from Kubernetes to access Vault pods. + # If deployment is on OpenShift, the following block is ignored. + # In order to expose the service, use the route section below + ingress: + enabled: false + labels: {} + # traffic: external + annotations: {} + # | + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + # or + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + + # Optionally use ingressClassName instead of deprecated annotation. + # See: https://kubernetes.io/docs/concepts/services-networking/ingress/#deprecated-annotation + ingressClassName: "nginx" + + # As of Kubernetes 1.19, all Ingress Paths must have a pathType configured. The default value below should be + # sufficient in most cases. + # See: https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types for other possible values. + pathType: ImplementationSpecific + + # When HA mode is enabled and K8s service registration is being used, + # configure the ingress to point to the Vault active service. + activeService: true + hosts: + - host: vault.azure.sonoflope.ca + paths: [] + ## Extra paths to prepend to the host configuration. This is useful when working with annotation based services. + extraPaths: [] + # - path: /* + # backend: + # service: + # name: ssl-redirect + # port: + # number: use-annotation + tls: + - secretName: vault-tls + hosts: + - vault.azure.sonoflope.ca + + # hostAliases is a list of aliases to be added to /etc/hosts. Specified as a YAML list. + hostAliases: [] + # - ip: 127.0.0.1 + # hostnames: + # - chart-example.local + + # OpenShift only - create a route to expose the service + # By default the created route will be of type passthrough + route: + enabled: false + + # When HA mode is enabled and K8s service registration is being used, + # configure the route to point to the Vault active service. + activeService: true + + labels: {} + annotations: {} + host: chart-example.local + # tls will be passed directly to the route's TLS config, which + # can be used to configure other termination methods that terminate + # TLS at the router + tls: + termination: passthrough + + # authDelegator enables a cluster role binding to be attached to the service + # account. This cluster role binding can be used to setup Kubernetes auth + # method. See https://developer.hashicorp.com/vault/docs/auth/kubernetes + authDelegator: + enabled: true + + # extraInitContainers is a list of init containers. Specified as a YAML list. + # This is useful if you need to run a script to provision TLS certificates or + # write out configuration files in a dynamic way. + extraInitContainers: null + + # extraContainers is a list of sidecar containers. Specified as a YAML list. + extraContainers: null + + # shareProcessNamespace enables process namespace sharing between Vault and the extraContainers + # This is useful if Vault must be signaled, e.g. to send a SIGHUP for a log rotation + shareProcessNamespace: false + + # extraArgs is a string containing additional Vault server arguments. + extraArgs: "" + + # extraPorts is a list of extra ports. Specified as a YAML list. + # This is useful if you need to add additional ports to the statefulset in dynamic way. + extraPorts: null + # - containerPort: 8300 + # name: http-monitoring + + # Used to define custom readinessProbe settings + readinessProbe: + enabled: true + path: /v1/sys/health?standbyok=true&sealedcode=204&uninitcode=204 + port: 8200 + failureThreshold: 2 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 3 + + # Used to enable a livenessProbe for the pods + livenessProbe: + enabled: true + execCommand: [] + path: "/v1/sys/health?standbyok=true" + port: 8200 + failureThreshold: 2 + initialDelaySeconds: 60 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 3 + + # Optional duration in seconds the pod needs to terminate gracefully. + # See: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ + terminationGracePeriodSeconds: 10 + + # Used to set the sleep time during the preStop step + preStopSleepSeconds: 5 + + # Used to define commands to run after the pod is ready. + # This can be used to automate processes such as initialization + # or boostrapping auth methods. + postStart: [] + # - /bin/sh + # - -c + # - /vault/userconfig/myscript/run.sh + + # extraEnvironmentVars is a list of extra environment variables to set with the stateful set. These could be + # used to include variables required for auto-unseal. + extraEnvironmentVars: + VAULT_CACERT: /vault/userconfig/vault-ha-tls/vault.ca + VAULT_TLSCERT: /vault/userconfig/vault-ha-tls/vault.crt + VAULT_TLSKEY: /vault/userconfig/vault-ha-tls/vault.key + + # extraSecretEnvironmentVars is a list of extra environment variables to set with the stateful set. + # These variables take value from existing Secret objects. + extraSecretEnvironmentVars: [] + # - envName: AWS_SECRET_ACCESS_KEY + # secretName: vault + # secretKey: AWS_SECRET_ACCESS_KEY + + # Deprecated: please use 'volumes' instead. + # extraVolumes is a list of extra volumes to mount. These will be exposed + # to Vault in the path `/vault/userconfig//`. The value below is + # an array of objects, examples are shown below. + extraVolumes: [] + # - type: secret (or "configMap") + # name: my-secret + # path: null # default is `/vault/userconfig` + + # volumes is a list of volumes made available to all containers. These are rendered + # via toYaml rather than pre-processed like the extraVolumes value. + # The purpose is to make it easy to share volumes between containers. + volumes: + - name: vault-ha-tls + secret: + secretName: vault-ha-tls + + # volumeMounts is a list of volumeMounts for the main server container. These are rendered + # via toYaml rather than pre-processed like the extraVolumes value. + # The purpose is to make it easy to share volumes between containers. + volumeMounts: + - name: vault-ha-tls + mountPath: /vault/userconfig/vault-ha-tls + + # Affinity Settings + # Commenting out or setting as empty the affinity variable, will allow + # deployment to single node services such as Minikube + # This should be either a multi-line string or YAML matching the PodSpec's affinity field. + affinity: | + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: {{ template "vault.name" . }} + app.kubernetes.io/instance: "{{ .Release.Name }}" + component: server + topologyKey: kubernetes.io/hostname + + # Topology settings for server pods + # ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + # This should be either a multi-line string or YAML matching the topologySpreadConstraints array + # in a PodSpec. + topologySpreadConstraints: [] + + # Toleration Settings for server pods + # This should be either a multi-line string or YAML matching the Toleration array + # in a PodSpec. + tolerations: [] + + # nodeSelector labels for server pod assignment, formatted as a multi-line string or YAML map. + # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + # Example: + # nodeSelector: + # beta.kubernetes.io/arch: amd64 + nodeSelector: {} + + # Enables network policy for server pods + networkPolicy: + enabled: false + egress: [] + # egress: + # - to: + # - ipBlock: + # cidr: 10.0.0.0/24 + # ports: + # - protocol: TCP + # port: 443 + ingress: + - from: + - namespaceSelector: {} + ports: + - port: 8200 + protocol: TCP + - port: 8201 + protocol: TCP + + # Priority class for server pods + priorityClassName: "" + + # Extra labels to attach to the server pods + # This should be a YAML map of the labels to apply to the server pods + extraLabels: {} + + # Extra annotations to attach to the server pods + # This can either be YAML or a YAML-formatted multi-line templated string map + # of the annotations to apply to the server pods + annotations: {} + + # Enables a headless service to be used by the Vault Statefulset + service: + enabled: true + # Enable or disable the vault-active service, which selects Vault pods that + # have labeled themselves as the cluster leader with `vault-active: "true"`. + active: + enabled: true + # Extra annotations for the service definition. This can either be YAML or a + # YAML-formatted multi-line templated string map of the annotations to apply + # to the active service. + annotations: {} + # Enable or disable the vault-standby service, which selects Vault pods that + # have labeled themselves as a cluster follower with `vault-active: "false"`. + standby: + enabled: true + # Extra annotations for the service definition. This can either be YAML or a + # YAML-formatted multi-line templated string map of the annotations to apply + # to the standby service. + annotations: {} + # If enabled, the service selectors will include `app.kubernetes.io/instance: {{ .Release.Name }}` + # When disabled, services may select Vault pods not deployed from the chart. + # Does not affect the headless vault-internal service with `ClusterIP: None` + instanceSelector: + enabled: true + # clusterIP controls whether a Cluster IP address is attached to the + # Vault service within Kubernetes. By default, the Vault service will + # be given a Cluster IP address, set to None to disable. When disabled + # Kubernetes will create a "headless" service. Headless services can be + # used to communicate with pods directly through DNS instead of a round-robin + # load balancer. + # clusterIP: None + + # Configures the service type for the main Vault service. Can be ClusterIP + # or NodePort. + type: ClusterIP + + # The IP family and IP families options are to set the behaviour in a dual-stack environment. + # Omitting these values will let the service fall back to whatever the CNI dictates the defaults + # should be. + # These are only supported for kubernetes versions >=1.23.0 + # + # Configures the service's supported IP family policy, can be either: + # SingleStack: Single-stack service. The control plane allocates a cluster IP for the Service, using the first + # configured service cluster IP range. + # PreferDualStack: Allocates IPv4 and IPv6 cluster IPs for the Service. + # RequireDualStack: Allocates Service .spec.ClusterIPs from both IPv4 and IPv6 address ranges. + ipFamilyPolicy: "" + + # Sets the families that should be supported and the order in which they should be applied to ClusterIP as well. + # Can be IPv4 and/or IPv6. + ipFamilies: [] + + # Do not wait for pods to be ready before including them in the services' + # targets. Does not apply to the headless service, which is used for + # cluster-internal communication. + publishNotReadyAddresses: true + + # The externalTrafficPolicy can be set to either Cluster or Local + # and is only valid for LoadBalancer and NodePort service types. + # The default value is Cluster. + # ref: https://kubernetes.io/docs/concepts/services-networking/service/#external-traffic-policy + externalTrafficPolicy: Cluster + + # If type is set to "NodePort", a specific nodePort value can be configured, + # will be random if left blank. + # nodePort: 30000 + + # When HA mode is enabled + # If type is set to "NodePort", a specific nodePort value can be configured, + # will be random if left blank. + # activeNodePort: 30001 + + # When HA mode is enabled + # If type is set to "NodePort", a specific nodePort value can be configured, + # will be random if left blank. + # standbyNodePort: 30002 + + # Port on which Vault server is listening + port: 8200 + # Target port to which the service should be mapped to + targetPort: 8200 + # Extra annotations for the service definition. This can either be YAML or a + # YAML-formatted multi-line templated string map of the annotations to apply + # to the service. + annotations: {} + + # This configures the Vault Statefulset to create a PVC for data + # storage when using the file or raft backend storage engines. + # See https://developer.hashicorp.com/vault/docs/configuration/storage to know more + dataStorage: + enabled: true + # Size of the PVC created + size: 10Gi + # Location where the PVC will be mounted. + mountPath: "/vault/data" + # Name of the storage class to use. If null it will use the + # configured default Storage Class. + storageClass: null + # Access Mode of the storage device being used for the PVC + accessMode: ReadWriteOnce + # Annotations to apply to the PVC + annotations: {} + # Labels to apply to the PVC + labels: {} + + # Persistent Volume Claim (PVC) retention policy + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#persistentvolumeclaim-retention + # Example: + # persistentVolumeClaimRetentionPolicy: + # whenDeleted: Retain + # whenScaled: Retain + persistentVolumeClaimRetentionPolicy: {} + + # This configures the Vault Statefulset to create a PVC for audit + # logs. Once Vault is deployed, initialized, and unsealed, Vault must + # be configured to use this for audit logs. This will be mounted to + # /vault/audit + # See https://developer.hashicorp.com/vault/docs/audit to know more + auditStorage: + enabled: true + # Size of the PVC created + size: 10Gi + # Location where the PVC will be mounted. + mountPath: "/vault/audit" + # Name of the storage class to use. If null it will use the + # configured default Storage Class. + storageClass: null + # Access Mode of the storage device being used for the PVC + accessMode: ReadWriteOnce + # Annotations to apply to the PVC + annotations: {} + # Labels to apply to the PVC + labels: {} + + # Run Vault in "HA" mode. There are no storage requirements unless the audit log + # persistence is required. In HA mode Vault will configure itself to use Consul + # for its storage backend. The default configuration provided will work the Consul + # Helm project by default. It is possible to manually configure Vault to use a + # different HA backend. + ha: + enabled: true + replicas: 2 + + # Set the api_addr configuration for Vault HA + # See https://developer.hashicorp.com/vault/docs/configuration#api_addr + # If set to null, this will be set to the Pod IP Address + apiAddr: null + + # Set the cluster_addr confuguration for Vault HA + # See https://developer.hashicorp.com/vault/docs/configuration#cluster_addr + # If set to null, this will be set to https://$(HOSTNAME).{{ template "vault.fullname" . }}-internal:8201 + clusterAddr: null + + # Enables Vault's integrated Raft storage. Unlike the typical HA modes where + # Vault's persistence is external (such as Consul), enabling Raft mode will create + # persistent volumes for Vault to store data according to the configuration under server.dataStorage. + # The Vault cluster will coordinate leader elections and failovers internally. + raft: + enabled: true + # Set the Node Raft ID to the name of the pod + setNodeId: true + + # Note: Configuration files are stored in ConfigMaps so sensitive data + # such as passwords should be either mounted through extraSecretEnvironmentVars + # or through a Kube secret. For more information see: + # https://developer.hashicorp.com/vault/docs/platform/k8s/helm/run#protecting-sensitive-vault-configurations + # https://developer.hashicorp.com/vault/docs/configuration/storage/raft + config: | + ui = true + plugin_directory = "/vault/plugins" + + listener "tcp" { + tls_disable = false + address = "[::]:8200" + cluster_address = "[::]:8201" + tls_cert_file = "/vault/userconfig/vault-ha-tls/vault.crt" + tls_key_file = "/vault/userconfig/vault-ha-tls/vault.key" + tls_client_ca_file = "/vault/userconfig/vault-ha-tls/vault.ca" + } + + storage "raft" { + path = "/vault/data" + retry_join { + leader_api_addr = "https://vault-0.vault-internal:8200" + leader_ca_cert_file = "/vault/userconfig/vault-ha-tls/vault.ca" + leader_client_cert_file = "/vault/userconfig/vault-ha-tls/vault.crt" + leader_client_key_file = "/vault/userconfig/vault-ha-tls/vault.key" + } + retry_join { + leader_api_addr = "https://vault-1.vault-internal:8200" + leader_ca_cert_file = "/vault/userconfig/vault-ha-tls/vault.ca" + leader_client_cert_file = "/vault/userconfig/vault-ha-tls/vault.crt" + leader_client_key_file = "/vault/userconfig/vault-ha-tls/vault.key" + } + retry_join { + leader_api_addr = "https://vault-2.vault-internal:8200" + leader_ca_cert_file = "/vault/userconfig/vault-ha-tls/vault.ca" + leader_client_cert_file = "/vault/userconfig/vault-ha-tls/vault.crt" + leader_client_key_file = "/vault/userconfig/vault-ha-tls/vault.key" + } + } + + + seal "azurekeyvault" { + tenant_id = "912b82dc-b892-42e5-ae31-c0ab5350455e" + client_id = "80bfe77e-97f0-46dd-9cc8-bd6c3b7fb050" + vault_name = "staging-vault-6a3fp" + key_name = "staging-vault-unseal-6a3fp" + } + + service_registration "kubernetes" {} + + # A disruption budget limits the number of pods of a replicated application + # that are down simultaneously from voluntary disruptions + disruptionBudget: + enabled: true + + # maxUnavailable will default to (n/2)-1 where n is the number of + # replicas. If you'd like a custom value, you can specify an override here. + maxUnavailable: null + + # Definition of the serviceAccount used to run Vault. + # These options are also used when using an external Vault server to validate + # Kubernetes tokens. + serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + # Create a Secret API object to store a non-expiring token for the service account. + # Prior to v1.24.0, Kubernetes used to generate this secret for each service account by default. + # Kubernetes now recommends using short-lived tokens from the TokenRequest API or projected volumes instead if + # possible. + # For more details, see https://kubernetes.io/docs/concepts/configuration/secret/#service-account-token-secrets + # serviceAccount.create must be equal to 'true' in order to use this feature. + createSecret: false + # Extra annotations for the serviceAccount definition. This can either be + # YAML or a YAML-formatted multi-line templated string map of the + # annotations to apply to the serviceAccount. + annotations: {} + # Extra labels to attach to the serviceAccount + # This should be a YAML map of the labels to apply to the serviceAccount + extraLabels: {} + # Enable or disable a service account role binding with the permissions required for + # Vault's Kubernetes service_registration config option. + # See https://developer.hashicorp.com/vault/docs/configuration/service-registration/kubernetes + serviceDiscovery: + enabled: true + + # Settings for the statefulSet used to run Vault. + statefulSet: + # Extra annotations for the statefulSet. This can either be YAML or a + # YAML-formatted multi-line templated string map of the annotations to apply + # to the statefulSet. + annotations: {} + + # Set the pod and container security contexts. + # If not set, these will default to, and for *not* OpenShift: + # pod: + # runAsNonRoot: true + # runAsGroup: {{ .Values.server.gid | default 1000 }} + # runAsUser: {{ .Values.server.uid | default 100 }} + # fsGroup: {{ .Values.server.gid | default 1000 }} + # container: + # allowPrivilegeEscalation: false + # + # If not set, these will default to, and for OpenShift: + # pod: {} + # container: {} + securityContext: + pod: {} + container: {} + + # Should the server pods run on the host network + hostNetwork: false + +# Vault UI +ui: + # True if you want to create a Service entry for the Vault UI. + # + # serviceType can be used to control the type of service created. For + # example, setting this to "LoadBalancer" will create an external load + # balancer (for supported K8S installations) to access the UI. + enabled: true + publishNotReadyAddresses: true + # The service should only contain selectors for active Vault pod + activeVaultPodOnly: false + serviceType: "ClusterIP" + serviceNodePort: null + externalPort: 8200 + targetPort: 8200 + + # The IP family and IP families options are to set the behaviour in a dual-stack environment. + # Omitting these values will let the service fall back to whatever the CNI dictates the defaults + # should be. + # These are only supported for kubernetes versions >=1.23.0 + # + # Configures the service's supported IP family, can be either: + # SingleStack: Single-stack service. The control plane allocates a cluster IP for the Service, using the first + # configured service cluster IP range. + # PreferDualStack: Allocates IPv4 and IPv6 cluster IPs for the Service. + # RequireDualStack: Allocates Service .spec.ClusterIPs from both IPv4 and IPv6 address ranges. + serviceIPFamilyPolicy: "" + + # Sets the families that should be supported and the order in which they should be applied to ClusterIP as well + # Can be IPv4 and/or IPv6. + serviceIPFamilies: [] + + # The externalTrafficPolicy can be set to either Cluster or Local + # and is only valid for LoadBalancer and NodePort service types. + # The default value is Cluster. + # ref: https://kubernetes.io/docs/concepts/services-networking/service/#external-traffic-policy + externalTrafficPolicy: Cluster + + # loadBalancerSourceRanges: + # - 10.0.0.0/16 + # - 1.78.23.3/32 + + # loadBalancerIP: + + # Extra annotations to attach to the ui service + # This can either be YAML or a YAML-formatted multi-line templated string map + # of the annotations to apply to the ui service + annotations: {} + +# secrets-store-csi-driver-provider-vault +csi: + # True if you want to install a secrets-store-csi-driver-provider-vault daemonset. + # + # Requires installing the secrets-store-csi-driver separately, see: + # https://github.com/kubernetes-sigs/secrets-store-csi-driver#install-the-secrets-store-csi-driver + # + # With the driver and provider installed, you can mount Vault secrets into volumes + # similar to the Vault Agent injector, and you can also sync those secrets into + # Kubernetes secrets. + enabled: true + + image: + repository: "hashicorp/vault-csi-provider" + tag: "1.4.1" + pullPolicy: IfNotPresent + + # volumes is a list of volumes made available to all containers. These are rendered + # via toYaml rather than pre-processed like the extraVolumes value. + # The purpose is to make it easy to share volumes between containers. + volumes: + - name: vault-ha-tls + secret: + secretName: vault-ha-tls + + # volumeMounts is a list of volumeMounts for the main server container. These are rendered + # via toYaml rather than pre-processed like the extraVolumes value. + # The purpose is to make it easy to share volumes between containers. + volumeMounts: + - name: vault-ha-tls + mountPath: "/vault/userconfig/vault-ha-tls" + readOnly: true + + resources: {} + # resources: + # requests: + # cpu: 50m + # memory: 128Mi + # limits: + # cpu: 50m + # memory: 128Mi + + # Override the default secret name for the CSI Provider's HMAC key used for + # generating secret versions. + hmacSecretName: "" + + # Settings for the daemonSet used to run the provider. + daemonSet: + updateStrategy: + type: RollingUpdate + maxUnavailable: "" + # Extra annotations for the daemonSet. This can either be YAML or a + # YAML-formatted multi-line templated string map of the annotations to apply + # to the daemonSet. + annotations: {} + # Provider host path (must match the CSI provider's path) + providersDir: "/etc/kubernetes/secrets-store-csi-providers" + # Kubelet host path + kubeletRootDir: "/var/lib/kubelet" + # Extra labels to attach to the vault-csi-provider daemonSet + # This should be a YAML map of the labels to apply to the csi provider daemonSet + extraLabels: {} + # security context for the pod template and container in the csi provider daemonSet + securityContext: + pod: {} + container: {} + + pod: + # Extra annotations for the provider pods. This can either be YAML or a + # YAML-formatted multi-line templated string map of the annotations to apply + # to the pod. + annotations: {} + + # Toleration Settings for provider pods + # This should be either a multi-line string or YAML matching the Toleration array + # in a PodSpec. + tolerations: [] + + # nodeSelector labels for csi pod assignment, formatted as a multi-line string or YAML map. + # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + # Example: + # nodeSelector: + # beta.kubernetes.io/arch: amd64 + nodeSelector: {} + + # Affinity Settings + # This should be either a multi-line string or YAML matching the PodSpec's affinity field. + affinity: {} + + # Extra labels to attach to the vault-csi-provider pod + # This should be a YAML map of the labels to apply to the csi provider pod + extraLabels: {} + + agent: + enabled: true + extraArgs: [] + + image: + repository: "hashicorp/vault" + tag: "1.15.2" + pullPolicy: IfNotPresent + + logFormat: standard + logLevel: info + + resources: {} + # resources: + # requests: + # memory: 256Mi + # cpu: 250m + # limits: + # memory: 256Mi + # cpu: 250m + + # Priority class for csi pods + priorityClassName: "" + + serviceAccount: + # Extra annotations for the serviceAccount definition. This can either be + # YAML or a YAML-formatted multi-line templated string map of the + # annotations to apply to the serviceAccount. + annotations: {} + + # Extra labels to attach to the vault-csi-provider serviceAccount + # This should be a YAML map of the labels to apply to the csi provider serviceAccount + extraLabels: {} + + # Used to configure readinessProbe for the pods. + readinessProbe: + failureThreshold: 2 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 3 + + # Used to configure livenessProbe for the pods. + livenessProbe: + failureThreshold: 2 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 3 + + # Enables debug logging. + debug: false + + # Pass arbitrary additional arguments to vault-csi-provider. + # See https://developer.hashicorp.com/vault/docs/platform/k8s/csi/configurations#command-line-arguments + # for the available command line flags. + extraArgs: + - "-vault-tls-ca-cert=/vault/userconfig/vault-ha-tls/vault.ca" + - "-vault-addr=https://vault.vault:8200" + +# Vault is able to collect and publish various runtime metrics. +# Enabling this feature requires setting adding `telemetry{}` stanza to +# the Vault configuration. There are a few examples included in the `config` sections above. +# +# For more information see: +# https://developer.hashicorp.com/vault/docs/configuration/telemetry +# https://developer.hashicorp.com/vault/docs/internals/telemetry +serverTelemetry: + # Enable support for the Prometheus Operator. Currently, this chart does not support + # authenticating to Vault's metrics endpoint, so the following `telemetry{}` must be included + # in the `listener "tcp"{}` stanza + # telemetry { + # unauthenticated_metrics_access = "true" + # } + # + # See the `standalone.config` for a more complete example of this. + # + # In addition, a top level `telemetry{}` stanza must also be included in the Vault configuration: + # + # example: + # telemetry { + # prometheus_retention_time = "30s" + # disable_hostname = true + # } + # + # Configuration for monitoring the Vault server. + serviceMonitor: + # The Prometheus operator *must* be installed before enabling this feature, + # if not the chart will fail to install due to missing CustomResourceDefinitions + # provided by the operator. + # + # Instructions on how to install the Helm chart can be found here: + # https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack + # More information can be found here: + # https://github.com/prometheus-operator/prometheus-operator + # https://github.com/prometheus-operator/kube-prometheus + + # Enable deployment of the Vault Server ServiceMonitor CustomResource. + enabled: false + + # Selector labels to add to the ServiceMonitor. + # When empty, defaults to: + # release: prometheus + selectors: {} + + # Interval at which Prometheus scrapes metrics + interval: 30s + + # Timeout for Prometheus scrapes + scrapeTimeout: 10s + + prometheusRules: + # The Prometheus operator *must* be installed before enabling this feature, + # if not the chart will fail to install due to missing CustomResourceDefinitions + # provided by the operator. + + # Deploy the PrometheusRule custom resource for AlertManager based alerts. + # Requires that AlertManager is properly deployed. + enabled: false + + # Selector labels to add to the PrometheusRules. + # When empty, defaults to: + # release: prometheus + selectors: {} + + # Some example rules. + rules: [] + # - alert: vault-HighResponseTime + # annotations: + # message: The response time of Vault is over 500ms on average over the last 5 minutes. + # expr: vault_core_handle_request{quantile="0.5", namespace="mynamespace"} > 500 + # for: 5m + # labels: + # severity: warning + # - alert: vault-HighResponseTime + # annotations: + # message: The response time of Vault is over 1s on average over the last 5 minutes. + # expr: vault_core_handle_request{quantile="0.5", namespace="mynamespace"} > 1000 + # for: 5m + # labels: + # severity: critical From cb9b90ec07f333743043e573cc25bd4a0585dc2d Mon Sep 17 00:00:00 2001 From: Jonathan Lopez Date: Tue, 5 Mar 2024 15:46:31 -0500 Subject: [PATCH 13/20] Issue #56: Delete .github/workflows/test.yml --- .github/workflows/test.yml | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index e229876f..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Python api-test workflows - -on: - pull_request: - types: - - opened - - closed - - synchronize - -jobs: - lint-test: - uses: ai-cfia/github-workflows/.github/workflows/workflow-lint-test-python.yml@main - secrets: inherit - - markdown-check: - uses: ai-cfia/github-workflows/.github/workflows/workflow-markdown-check.yml@main - with: - config-file-path: '.mlc_config.json' - secrets: inherit - - repo-standard: - uses: ai-cfia/github-workflows/.github/workflows/workflow-repo-standards-validation.yml@main - secrets: inherit From 135f811e73639d23240dc18739053be526c6cb22 Mon Sep 17 00:00:00 2001 From: SonOfLope Date: Tue, 5 Mar 2024 15:48:48 -0500 Subject: [PATCH 14/20] Issue #56: Fix doc deadlink --- docs/secret-management.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/secret-management.md b/docs/secret-management.md index 930f7d1d..6873b7ab 100644 --- a/docs/secret-management.md +++ b/docs/secret-management.md @@ -32,7 +32,7 @@ in a Kubernetes environment. The following diagram illustrates the workflow of the Vault Agent Injector and how developers can manage secrets of hosted applications: -![Developer workflow diagram](img/Vault-argoCD-workflow.svg) +![Developer workflow diagram](img/vault-argocd-workflow.svg) ## Secret management process From 474803d0a27c7aba0b438d157927bac2e89af100 Mon Sep 17 00:00:00 2001 From: SonOfLope Date: Wed, 6 Mar 2024 14:56:22 -0500 Subject: [PATCH 15/20] Issue #56: Fixing yaml lint and file name based on convention --- ...{create-new-secret.PNG => create-new-secret.png} | Bin .../aks/apps/finesse/base/finesse-ingress.yaml | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename docs/img/{create-new-secret.PNG => create-new-secret.png} (100%) diff --git a/docs/img/create-new-secret.PNG b/docs/img/create-new-secret.png similarity index 100% rename from docs/img/create-new-secret.PNG rename to docs/img/create-new-secret.png diff --git a/kubernetes/aks/apps/finesse/base/finesse-ingress.yaml b/kubernetes/aks/apps/finesse/base/finesse-ingress.yaml index 65b4dd0c..873918d0 100644 --- a/kubernetes/aks/apps/finesse/base/finesse-ingress.yaml +++ b/kubernetes/aks/apps/finesse/base/finesse-ingress.yaml @@ -8,7 +8,7 @@ metadata: external-dns.alpha.kubernetes.io/target: inspection.alpha.canada.ca cert-manager.io/cluster-issuer: letsencrypt-prod nginx.ingress.kubernetes.io/use-regex: "true" - nginx.ingress.kubernetes.io/rewrite-target: /$2 # https://kubernetes.github.io/ingress-nginx/examples/rewrite/ + nginx.ingress.kubernetes.io/rewrite-target: /$2 # https://kubernetes.github.io/ingress-nginx/examples/rewrite/ ingress.kubernetes.io/force-ssl-redirect: "true" kubernetes.io/tls-acme: "true" spec: @@ -41,7 +41,7 @@ metadata: cert-manager.io/cluster-issuer: letsencrypt-prod ingress.kubernetes.io/force-ssl-redirect: "true" kubernetes.io/tls-acme: "true" - nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/use-regex: "true" spec: ingressClassName: nginx tls: From c2254324fbb5fe28b1eaab99dd0865c548775d3b Mon Sep 17 00:00:00 2001 From: SonOfLope Date: Wed, 6 Mar 2024 14:57:59 -0500 Subject: [PATCH 16/20] Issue #56: Fix markdown png reference --- docs/secret-management.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/secret-management.md b/docs/secret-management.md index 6873b7ab..9284a1d8 100644 --- a/docs/secret-management.md +++ b/docs/secret-management.md @@ -86,4 +86,4 @@ protected by the same security mechanisms as the Vault server. applications. ![PV secret engine](img/pv-secret-engine.png) 5. Once in the directory of your application secrets, simply click on 'create new version' and you will be able to add, update, or delete secrets as - needed. ![Create mew secret](img/create-new-secret.PNG) + needed. ![Create mew secret](img/create-new-secret.png) From c6ca88b69722745af381a184f0becec773434944 Mon Sep 17 00:00:00 2001 From: SonOfLope Date: Fri, 8 Mar 2024 13:28:35 -0500 Subject: [PATCH 17/20] Issue #56: Updated in order to merge. config operator will be pushed since not a priority --- kubernetes/aks/system/vault/argo-app.yaml | 3 + .../aks/system/vault/{ => base}/ingress.yml | 1 + .../kustomization.yaml | 0 .../kv-secret-engine.yaml | 0 .../{ => vault-config-operator}/policies.yaml | 0 .../{ => vault-config-operator}/roles.yaml | 0 kubernetes/aks/system/vault/helm/values.yaml | 1178 ----------------- 7 files changed, 4 insertions(+), 1178 deletions(-) rename kubernetes/aks/system/vault/{ => base}/ingress.yml (92%) rename kubernetes/aks/system/vault/base/{ => vault-config-operator}/kustomization.yaml (100%) rename kubernetes/aks/system/vault/base/{ => vault-config-operator}/kv-secret-engine.yaml (100%) rename kubernetes/aks/system/vault/base/{ => vault-config-operator}/policies.yaml (100%) rename kubernetes/aks/system/vault/base/{ => vault-config-operator}/roles.yaml (100%) delete mode 100644 kubernetes/aks/system/vault/helm/values.yaml diff --git a/kubernetes/aks/system/vault/argo-app.yaml b/kubernetes/aks/system/vault/argo-app.yaml index dca2b57d..bac74d5c 100644 --- a/kubernetes/aks/system/vault/argo-app.yaml +++ b/kubernetes/aks/system/vault/argo-app.yaml @@ -22,3 +22,6 @@ spec: - repoURL: https://github.com/ai-cfia/howard.git targetRevision: HEAD ref: values + - repoURL: https://github.com/ai-cfia/howard.git + path: kubernetes/aks/system/vault/base + targetRevision: HEAD diff --git a/kubernetes/aks/system/vault/ingress.yml b/kubernetes/aks/system/vault/base/ingress.yml similarity index 92% rename from kubernetes/aks/system/vault/ingress.yml rename to kubernetes/aks/system/vault/base/ingress.yml index 089c0335..b038c86e 100644 --- a/kubernetes/aks/system/vault/ingress.yml +++ b/kubernetes/aks/system/vault/base/ingress.yml @@ -8,6 +8,7 @@ metadata: app.kubernetes.io/name: vault-ui app.kubernetes.io/instance: vault annotations: + nginx.ingress.kubernetes.io/whitelist-source-range: 200.194.32.0/24 external-dns.alpha.kubernetes.io/target: inspection.alpha.canada.ca cert-manager.io/cluster-issuer: letsencrypt-prod nginx.ingress.kubernetes.io/rewrite-target: / diff --git a/kubernetes/aks/system/vault/base/kustomization.yaml b/kubernetes/aks/system/vault/base/vault-config-operator/kustomization.yaml similarity index 100% rename from kubernetes/aks/system/vault/base/kustomization.yaml rename to kubernetes/aks/system/vault/base/vault-config-operator/kustomization.yaml diff --git a/kubernetes/aks/system/vault/base/kv-secret-engine.yaml b/kubernetes/aks/system/vault/base/vault-config-operator/kv-secret-engine.yaml similarity index 100% rename from kubernetes/aks/system/vault/base/kv-secret-engine.yaml rename to kubernetes/aks/system/vault/base/vault-config-operator/kv-secret-engine.yaml diff --git a/kubernetes/aks/system/vault/base/policies.yaml b/kubernetes/aks/system/vault/base/vault-config-operator/policies.yaml similarity index 100% rename from kubernetes/aks/system/vault/base/policies.yaml rename to kubernetes/aks/system/vault/base/vault-config-operator/policies.yaml diff --git a/kubernetes/aks/system/vault/base/roles.yaml b/kubernetes/aks/system/vault/base/vault-config-operator/roles.yaml similarity index 100% rename from kubernetes/aks/system/vault/base/roles.yaml rename to kubernetes/aks/system/vault/base/vault-config-operator/roles.yaml diff --git a/kubernetes/aks/system/vault/helm/values.yaml b/kubernetes/aks/system/vault/helm/values.yaml deleted file mode 100644 index 24d16863..00000000 --- a/kubernetes/aks/system/vault/helm/values.yaml +++ /dev/null @@ -1,1178 +0,0 @@ -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: MPL-2.0 - -# Available parameters and their default values for the Vault chart. ---- -global: - # enabled is the master enabled switch. Setting this to true or false - # will enable or disable all the components within this chart by default. - enabled: true - - # The namespace to deploy to. Defaults to the `helm` installation namespace. - namespace: "vault" - - # Image pull secret to use for registry authentication. - # Alternatively, the value may be specified as an array of strings. - imagePullSecrets: [] - # imagePullSecrets: - # - name: image-pull-secret - - # TLS for end-to-end encrypted transport - tlsDisable: false - - # External vault server address for the injector and CSI provider to use. - # Setting this will disable deployment of a vault server. - externalVaultAddr: "" - - # If deploying to OpenShift - openshift: false - - # Create PodSecurityPolicy for pods - psp: - enable: false - # Annotation for PodSecurityPolicy. - # This is a multi-line templated string map, and can also be set as YAML. - annotations: | - seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default,runtime/default - apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default - seccomp.security.alpha.kubernetes.io/defaultProfileName: runtime/default - apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default - - serverTelemetry: - # Enable integration with the Prometheus Operator - # See the top level serverTelemetry section below before enabling this feature. - prometheusOperator: false - -injector: - # True if you want to enable vault agent injection. - # @default: global.enabled - enabled: true - - replicas: 1 - - # Configures the port the injector should listen on - port: 8080 - - # If multiple replicas are specified, by default a leader will be determined - # so that only one injector attempts to create TLS certificates. - leaderElector: - enabled: true - - # If true, will enable a node exporter metrics endpoint at /metrics. - metrics: - enabled: true - - # Deprecated: Please use global.externalVaultAddr instead. - externalVaultAddr: "" - - # image sets the repo and tag of the vault-k8s image to use for the injector. - image: - repository: "hashicorp/vault-k8s" - tag: "1.3.1" - pullPolicy: IfNotPresent - - # agentImage sets the repo and tag of the Vault image to use for the Vault Agent - # containers. This should be set to the official Vault image. Vault 1.3.1+ is - # required. - agentImage: - repository: "hashicorp/vault" - tag: "1.15.2" - - # The default values for the injected Vault Agent containers. - agentDefaults: - # For more information on configuring resources, see the K8s documentation: - # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - cpuLimit: "500m" - cpuRequest: "250m" - memLimit: "128Mi" - memRequest: "64Mi" - # ephemeralLimit: "128Mi" - # ephemeralRequest: "64Mi" - - # Default template type for secrets when no custom template is specified. - # Possible values include: "json" and "map". - template: "map" - - # Default values within Agent's template_config stanza. - templateConfig: - exitOnRetryFailure: true - staticSecretRenderInterval: "" - - # Used to define custom livenessProbe settings - livenessProbe: - failureThreshold: 2 - initialDelaySeconds: 5 - periodSeconds: 2 - successThreshold: 1 - timeoutSeconds: 5 - - # Used to define custom readinessProbe settings - readinessProbe: - failureThreshold: 2 - initialDelaySeconds: 5 - periodSeconds: 2 - successThreshold: 1 - timeoutSeconds: 5 - - # Used to define custom startupProbe settings - startupProbe: - failureThreshold: 12 - initialDelaySeconds: 5 - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 5 - - # Mount Path of the Vault Kubernetes Auth Method. - authPath: "auth/kubernetes" - - # Configures the log verbosity of the injector. - # Supported log levels include: trace, debug, info, warn, error - logLevel: "info" - - # Configures the log format of the injector. Supported log formats: "standard", "json". - logFormat: "standard" - - # Configures all Vault Agent sidecars to revoke their token when shutting down - revokeOnShutdown: false - - webhook: - # Configures failurePolicy of the webhook. The "unspecified" default behaviour depends on the - # API Version of the WebHook. - # To block pod creation while the webhook is unavailable, set the policy to `Fail` below. - # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy - # - failurePolicy: Ignore - - # matchPolicy specifies the approach to accepting changes based on the rules of - # the MutatingWebhookConfiguration. - # - matchPolicy: Exact - - # timeoutSeconds is the amount of seconds before the webhook request will be ignored - # or fails. - # If it is ignored or fails depends on the failurePolicy - # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#timeouts - # for more details. - # - timeoutSeconds: 30 - - # namespaceSelector is the selector for restricting the webhook to only - # specific namespaces. - # Example: - # namespaceSelector: - # matchLabels: - # sidecar-injector: enabled - namespaceSelector: {} - - # objectSelector is the selector for restricting the webhook to only - # specific labels. - # Example: - # objectSelector: - # matchLabels: - # vault-sidecar-injector: enabled - objectSelector: | - matchExpressions: - - key: app.kubernetes.io/name - operator: NotIn - values: - - {{ template "vault.name" . }}-agent-injector - - # Extra annotations to attach to the webhook - annotations: {} - - # Deprecated: please use 'webhook.failurePolicy' instead - # Configures failurePolicy of the webhook. The "unspecified" default behaviour depends on the - # API Version of the WebHook. - # To block pod creation while webhook is unavailable, set the policy to `Fail` below. - # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy - # - failurePolicy: Ignore - - # Deprecated: please use 'webhook.namespaceSelector' instead - # namespaceSelector is the selector for restricting the webhook to only - # specific namespaces. - # for more details. - # Example: - # namespaceSelector: - # matchLabels: - # sidecar-injector: enabled - namespaceSelector: {} - - # Deprecated: please use 'webhook.objectSelector' instead - # objectSelector is the selector for restricting the webhook to only - # specific labels. - # for more details. - # Example: - # objectSelector: - # matchLabels: - # vault-sidecar-injector: enabled - objectSelector: {} - - # Deprecated: please use 'webhook.annotations' instead - # Extra annotations to attach to the webhook - webhookAnnotations: {} - - certs: - # secretName is the name of the secret that has the TLS certificate and - # private key to serve the injector webhook. If this is null, then the - # injector will default to its automatic management mode that will assign - # a service account to the injector to generate its own certificates. - secretName: null - - # caBundle is a base64-encoded PEM-encoded certificate bundle for the CA - # that signed the TLS certificate that the webhook serves. This must be set - # if secretName is non-null unless an external service like cert-manager is - # keeping the caBundle updated. - caBundle: "" - - # certName and keyName are the names of the files within the secret for - # the TLS cert and private key, respectively. These have reasonable - # defaults but can be customized if necessary. - certName: tls.crt - keyName: tls.key - - # Security context for the pod template and the injector container - # The default pod securityContext is: - # runAsNonRoot: true - # runAsGroup: {{ .Values.injector.gid | default 1000 }} - # runAsUser: {{ .Values.injector.uid | default 100 }} - # fsGroup: {{ .Values.injector.gid | default 1000 }} - # and for container is - # allowPrivilegeEscalation: false - # capabilities: - # drop: - # - ALL - securityContext: - pod: {} - container: {} - - resources: {} - # resources: - # requests: - # memory: 256Mi - # cpu: 250m - # limits: - # memory: 256Mi - # cpu: 250m - - # extraEnvironmentVars is a list of extra environment variables to set in the - # injector deployment. - extraEnvironmentVars: - VAULT_ADDR: https://vault.vault:8200 - - # Affinity Settings for injector pods - # This can either be a multi-line string or YAML matching the PodSpec's affinity field. - # Commenting out or setting as empty the affinity variable, will allow - # deployment of multiple replicas to single node services such as Minikube. - affinity: | - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app.kubernetes.io/name: {{ template "vault.name" . }}-agent-injector - app.kubernetes.io/instance: "{{ .Release.Name }}" - component: webhook - topologyKey: kubernetes.io/hostname - - # Topology settings for injector pods - # ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ - # This should be either a multi-line string or YAML matching the topologySpreadConstraints array - # in a PodSpec. - topologySpreadConstraints: [] - - # Toleration Settings for injector pods - # This should be either a multi-line string or YAML matching the Toleration array - # in a PodSpec. - tolerations: [] - - # nodeSelector labels for server pod assignment, formatted as a multi-line string or YAML map. - # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector - # Example: - # nodeSelector: - # beta.kubernetes.io/arch: amd64 - nodeSelector: {} - - # Priority class for injector pods - priorityClassName: "" - - # Extra annotations to attach to the injector pods - # This can either be YAML or a YAML-formatted multi-line templated string map - # of the annotations to apply to the injector pods - annotations: {} - - # Extra labels to attach to the agent-injector - # This should be a YAML map of the labels to apply to the injector - extraLabels: {} - - # Should the injector pods run on the host network (useful when using - # an alternate CNI in EKS) - hostNetwork: false - - # Injector service specific config - service: - # Extra annotations to attach to the injector service - annotations: {} - - # Injector serviceAccount specific config - serviceAccount: - # Extra annotations to attach to the injector serviceAccount - annotations: {} - - # A disruption budget limits the number of pods of a replicated application - # that are down simultaneously from voluntary disruptions - podDisruptionBudget: {} - # podDisruptionBudget: - # maxUnavailable: 1 - - # strategy for updating the deployment. This can be a multi-line string or a - # YAML map. - strategy: {} - # strategy: | - # rollingUpdate: - # maxSurge: 25% - # maxUnavailable: 25% - # type: RollingUpdate - -server: - # If true, or "-" with global.enabled true, Vault server will be installed. - # See vault.mode in _helpers.tpl for implementation details. - enabled: "-" - - # [Enterprise Only] This value refers to a Kubernetes secret that you have - # created that contains your enterprise license. If you are not using an - # enterprise image or if you plan to introduce the license key via another - # route, then leave secretName blank ("") or set it to null. - # Requires Vault Enterprise 1.8 or later. - enterpriseLicense: - # The name of the Kubernetes secret that holds the enterprise license. The - # secret must be in the same namespace that Vault is installed into. - secretName: "" - # The key within the Kubernetes secret that holds the enterprise license. - secretKey: "license" - - # Resource requests, limits, etc. for the server cluster placement. This - # should map directly to the value of the resources field for a PodSpec. - # By default no direct resource request is made. - - image: - repository: "hashicorp/vault" - tag: "1.15.2" - # Overrides the default Image Pull Policy - pullPolicy: IfNotPresent - - # Configure the Update Strategy Type for the StatefulSet - # See https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies - updateStrategyType: "OnDelete" - - # Configure the logging verbosity for the Vault server. - # Supported log levels include: trace, debug, info, warn, error - logLevel: "" - - # Configure the logging format for the Vault server. - # Supported log formats include: standard, json - logFormat: "" - - resources: {} - # resources: - # requests: - # memory: 256Mi - # cpu: 250m - # limits: - # memory: 256Mi - # cpu: 250m - - # Ingress allows ingress services to be created to allow external access - # from Kubernetes to access Vault pods. - # If deployment is on OpenShift, the following block is ignored. - # In order to expose the service, use the route section below - ingress: - enabled: false - labels: {} - # traffic: external - annotations: {} - # | - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - # or - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - - # Optionally use ingressClassName instead of deprecated annotation. - # See: https://kubernetes.io/docs/concepts/services-networking/ingress/#deprecated-annotation - ingressClassName: "nginx" - - # As of Kubernetes 1.19, all Ingress Paths must have a pathType configured. The default value below should be - # sufficient in most cases. - # See: https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types for other possible values. - pathType: Prefix - - # When HA mode is enabled and K8s service registration is being used, - # configure the ingress to point to the Vault active service. - activeService: true - hosts: - - host: vault.inspection.alpha.canada.ca # For future use. - paths: [] - ## Extra paths to prepend to the host configuration. This is useful when working with annotation based services. - extraPaths: [] - # - path: /* - # backend: - # service: - # name: ssl-redirect - # port: - # number: use-annotation - tls: - - secretName: vault-tls - hosts: - - vault.inspection.alpha.canada.ca - - # hostAliases is a list of aliases to be added to /etc/hosts. Specified as a YAML list. - hostAliases: [] - # - ip: 127.0.0.1 - # hostnames: - # - chart-example.local - - # OpenShift only - create a route to expose the service - # By default the created route will be of type passthrough - route: - enabled: false - - # When HA mode is enabled and K8s service registration is being used, - # configure the route to point to the Vault active service. - activeService: true - - labels: {} - annotations: {} - host: chart-example.local - # tls will be passed directly to the route's TLS config, which - # can be used to configure other termination methods that terminate - # TLS at the router - tls: - termination: passthrough - - # authDelegator enables a cluster role binding to be attached to the service - # account. This cluster role binding can be used to setup Kubernetes auth - # method. See https://developer.hashicorp.com/vault/docs/auth/kubernetes - authDelegator: - enabled: true - - # extraInitContainers is a list of init containers. Specified as a YAML list. - # This is useful if you need to run a script to provision TLS certificates or - # write out configuration files in a dynamic way. - extraInitContainers: null - - # extraContainers is a list of sidecar containers. Specified as a YAML list. - extraContainers: null - - # shareProcessNamespace enables process namespace sharing between Vault and the extraContainers - # This is useful if Vault must be signaled, e.g. to send a SIGHUP for a log rotation - shareProcessNamespace: false - - # extraArgs is a string containing additional Vault server arguments. - extraArgs: "" - - # extraPorts is a list of extra ports. Specified as a YAML list. - # This is useful if you need to add additional ports to the statefulset in dynamic way. - extraPorts: null - # - containerPort: 8300 - # name: http-monitoring - - # Used to define custom readinessProbe settings - readinessProbe: - enabled: true - path: /v1/sys/health?standbyok=true&sealedcode=204&uninitcode=204 - port: 8200 - failureThreshold: 2 - initialDelaySeconds: 5 - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 3 - - # Used to enable a livenessProbe for the pods - livenessProbe: - enabled: true - execCommand: [] - path: "/v1/sys/health?standbyok=true" - port: 8200 - failureThreshold: 2 - initialDelaySeconds: 60 - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 3 - - # Optional duration in seconds the pod needs to terminate gracefully. - # See: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ - terminationGracePeriodSeconds: 10 - - # Used to set the sleep time during the preStop step - preStopSleepSeconds: 5 - - # Used to define commands to run after the pod is ready. - # This can be used to automate processes such as initialization - # or boostrapping auth methods. - postStart: [] - # - /bin/sh - # - -c - # - /vault/userconfig/myscript/run.sh - - # extraEnvironmentVars is a list of extra environment variables to set with the stateful set. These could be - # used to include variables required for auto-unseal. - extraEnvironmentVars: - VAULT_CACERT: /vault/userconfig/vault-ha-tls/vault.ca - VAULT_TLSCERT: /vault/userconfig/vault-ha-tls/vault.crt - VAULT_TLSKEY: /vault/userconfig/vault-ha-tls/vault.key - - # extraSecretEnvironmentVars is a list of extra environment variables to set with the stateful set. - # These variables take value from existing Secret objects. - extraSecretEnvironmentVars: [] - # - envName: AWS_SECRET_ACCESS_KEY - # secretName: vault - # secretKey: AWS_SECRET_ACCESS_KEY - - # Deprecated: please use 'volumes' instead. - # extraVolumes is a list of extra volumes to mount. These will be exposed - # to Vault in the path `/vault/userconfig//`. The value below is - # an array of objects, examples are shown below. - extraVolumes: [] - # - type: secret (or "configMap") - # name: my-secret - # path: null # default is `/vault/userconfig` - - # volumes is a list of volumes made available to all containers. These are rendered - # via toYaml rather than pre-processed like the extraVolumes value. - # The purpose is to make it easy to share volumes between containers. - volumes: - - name: vault-ha-tls - secret: - secretName: vault-ha-tls - - # volumeMounts is a list of volumeMounts for the main server container. These are rendered - # via toYaml rather than pre-processed like the extraVolumes value. - # The purpose is to make it easy to share volumes between containers. - volumeMounts: - - name: vault-ha-tls - mountPath: /vault/userconfig/vault-ha-tls - - # Affinity Settings - # Commenting out or setting as empty the affinity variable, will allow - # deployment to single node services such as Minikube - # This should be either a multi-line string or YAML matching the PodSpec's affinity field. - affinity: | - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app.kubernetes.io/name: {{ template "vault.name" . }} - app.kubernetes.io/instance: "{{ .Release.Name }}" - component: server - topologyKey: kubernetes.io/hostname - - # Topology settings for server pods - # ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ - # This should be either a multi-line string or YAML matching the topologySpreadConstraints array - # in a PodSpec. - topologySpreadConstraints: [] - - # Toleration Settings for server pods - # This should be either a multi-line string or YAML matching the Toleration array - # in a PodSpec. - tolerations: [] - - # nodeSelector labels for server pod assignment, formatted as a multi-line string or YAML map. - # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector - # Example: - # nodeSelector: - # beta.kubernetes.io/arch: amd64 - nodeSelector: {} - - # Enables network policy for server pods - networkPolicy: - enabled: false - egress: [] - # egress: - # - to: - # - ipBlock: - # cidr: 10.0.0.0/24 - # ports: - # - protocol: TCP - # port: 443 - ingress: - - from: - - namespaceSelector: {} - ports: - - port: 8200 - protocol: TCP - - port: 8201 - protocol: TCP - - # Priority class for server pods - priorityClassName: "" - - # Extra labels to attach to the server pods - # This should be a YAML map of the labels to apply to the server pods - extraLabels: {} - - # Extra annotations to attach to the server pods - # This can either be YAML or a YAML-formatted multi-line templated string map - # of the annotations to apply to the server pods - annotations: {} - - # Enables a headless service to be used by the Vault Statefulset - service: - enabled: true - # Enable or disable the vault-active service, which selects Vault pods that - # have labeled themselves as the cluster leader with `vault-active: "true"`. - active: - enabled: true - # Extra annotations for the service definition. This can either be YAML or a - # YAML-formatted multi-line templated string map of the annotations to apply - # to the active service. - annotations: {} - # Enable or disable the vault-standby service, which selects Vault pods that - # have labeled themselves as a cluster follower with `vault-active: "false"`. - standby: - enabled: true - # Extra annotations for the service definition. This can either be YAML or a - # YAML-formatted multi-line templated string map of the annotations to apply - # to the standby service. - annotations: {} - # If enabled, the service selectors will include `app.kubernetes.io/instance: {{ .Release.Name }}` - # When disabled, services may select Vault pods not deployed from the chart. - # Does not affect the headless vault-internal service with `ClusterIP: None` - instanceSelector: - enabled: true - # clusterIP controls whether a Cluster IP address is attached to the - # Vault service within Kubernetes. By default, the Vault service will - # be given a Cluster IP address, set to None to disable. When disabled - # Kubernetes will create a "headless" service. Headless services can be - # used to communicate with pods directly through DNS instead of a round-robin - # load balancer. - # clusterIP: None - - # Configures the service type for the main Vault service. Can be ClusterIP - # or NodePort. - type: ClusterIP - - # The IP family and IP families options are to set the behaviour in a dual-stack environment. - # Omitting these values will let the service fall back to whatever the CNI dictates the defaults - # should be. - # These are only supported for kubernetes versions >=1.23.0 - # - # Configures the service's supported IP family policy, can be either: - # SingleStack: Single-stack service. The control plane allocates a cluster IP for the Service, using the first - # configured service cluster IP range. - # PreferDualStack: Allocates IPv4 and IPv6 cluster IPs for the Service. - # RequireDualStack: Allocates Service .spec.ClusterIPs from both IPv4 and IPv6 address ranges. - ipFamilyPolicy: "" - - # Sets the families that should be supported and the order in which they should be applied to ClusterIP as well. - # Can be IPv4 and/or IPv6. - ipFamilies: [] - - # Do not wait for pods to be ready before including them in the services' - # targets. Does not apply to the headless service, which is used for - # cluster-internal communication. - publishNotReadyAddresses: true - - # The externalTrafficPolicy can be set to either Cluster or Local - # and is only valid for LoadBalancer and NodePort service types. - # The default value is Cluster. - # ref: https://kubernetes.io/docs/concepts/services-networking/service/#external-traffic-policy - externalTrafficPolicy: Cluster - - # If type is set to "NodePort", a specific nodePort value can be configured, - # will be random if left blank. - # nodePort: 30000 - - # When HA mode is enabled - # If type is set to "NodePort", a specific nodePort value can be configured, - # will be random if left blank. - # activeNodePort: 30001 - - # When HA mode is enabled - # If type is set to "NodePort", a specific nodePort value can be configured, - # will be random if left blank. - # standbyNodePort: 30002 - - # Port on which Vault server is listening - port: 8200 - # Target port to which the service should be mapped to - targetPort: 8200 - # Extra annotations for the service definition. This can either be YAML or a - # YAML-formatted multi-line templated string map of the annotations to apply - # to the service. - annotations: {} - - # This configures the Vault Statefulset to create a PVC for data - # storage when using the file or raft backend storage engines. - # See https://developer.hashicorp.com/vault/docs/configuration/storage to know more - dataStorage: - enabled: true - # Size of the PVC created - size: 10Gi - # Location where the PVC will be mounted. - mountPath: "/vault/data" - # Name of the storage class to use. If null it will use the - # configured default Storage Class. - storageClass: null - # Access Mode of the storage device being used for the PVC - accessMode: ReadWriteOnce - # Annotations to apply to the PVC - annotations: {} - # Labels to apply to the PVC - labels: {} - - # Persistent Volume Claim (PVC) retention policy - # ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#persistentvolumeclaim-retention - # Example: - # persistentVolumeClaimRetentionPolicy: - # whenDeleted: Retain - # whenScaled: Retain - persistentVolumeClaimRetentionPolicy: {} - - # This configures the Vault Statefulset to create a PVC for audit - # logs. Once Vault is deployed, initialized, and unsealed, Vault must - # be configured to use this for audit logs. This will be mounted to - # /vault/audit - # See https://developer.hashicorp.com/vault/docs/audit to know more - auditStorage: - enabled: true - # Size of the PVC created - size: 10Gi - # Location where the PVC will be mounted. - mountPath: "/vault/audit" - # Name of the storage class to use. If null it will use the - # configured default Storage Class. - storageClass: null - # Access Mode of the storage device being used for the PVC - accessMode: ReadWriteOnce - # Annotations to apply to the PVC - annotations: {} - # Labels to apply to the PVC - labels: {} - - # Run Vault in "HA" mode. There are no storage requirements unless the audit log - # persistence is required. In HA mode Vault will configure itself to use Consul - # for its storage backend. The default configuration provided will work the Consul - # Helm project by default. It is possible to manually configure Vault to use a - # different HA backend. - ha: - enabled: true - replicas: 2 - - # Set the api_addr configuration for Vault HA - # See https://developer.hashicorp.com/vault/docs/configuration#api_addr - # If set to null, this will be set to the Pod IP Address - apiAddr: null - - # Set the cluster_addr confuguration for Vault HA - # See https://developer.hashicorp.com/vault/docs/configuration#cluster_addr - # If set to null, this will be set to https://$(HOSTNAME).{{ template "vault.fullname" . }}-internal:8201 - clusterAddr: null - - # Enables Vault's integrated Raft storage. Unlike the typical HA modes where - # Vault's persistence is external (such as Consul), enabling Raft mode will create - # persistent volumes for Vault to store data according to the configuration under server.dataStorage. - # The Vault cluster will coordinate leader elections and failovers internally. - raft: - enabled: true - # Set the Node Raft ID to the name of the pod - setNodeId: true - - # Note: Configuration files are stored in ConfigMaps so sensitive data - # such as passwords should be either mounted through extraSecretEnvironmentVars - # or through a Kube secret. For more information see: - # https://developer.hashicorp.com/vault/docs/platform/k8s/helm/run#protecting-sensitive-vault-configurations - # https://developer.hashicorp.com/vault/docs/configuration/storage/raft - config: | - ui = true - plugin_directory = "/vault/plugins" - - listener "tcp" { - tls_disable = false - address = "[::]:8200" - cluster_address = "[::]:8201" - tls_cert_file = "/vault/userconfig/vault-ha-tls/vault.crt" - tls_key_file = "/vault/userconfig/vault-ha-tls/vault.key" - tls_client_ca_file = "/vault/userconfig/vault-ha-tls/vault.ca" - } - - storage "raft" { - path = "/vault/data" - retry_join { - leader_api_addr = "https://vault-0.vault-internal:8200" - leader_ca_cert_file = "/vault/userconfig/vault-ha-tls/vault.ca" - leader_client_cert_file = "/vault/userconfig/vault-ha-tls/vault.crt" - leader_client_key_file = "/vault/userconfig/vault-ha-tls/vault.key" - } - retry_join { - leader_api_addr = "https://vault-1.vault-internal:8200" - leader_ca_cert_file = "/vault/userconfig/vault-ha-tls/vault.ca" - leader_client_cert_file = "/vault/userconfig/vault-ha-tls/vault.crt" - leader_client_key_file = "/vault/userconfig/vault-ha-tls/vault.key" - } - retry_join { - leader_api_addr = "https://vault-2.vault-internal:8200" - leader_ca_cert_file = "/vault/userconfig/vault-ha-tls/vault.ca" - leader_client_cert_file = "/vault/userconfig/vault-ha-tls/vault.crt" - leader_client_key_file = "/vault/userconfig/vault-ha-tls/vault.key" - } - } - - - seal "azurekeyvault" { - tenant_id = "18b5a5ed-1d86-41d3-94a0-bc27dae32ab2" - client_id = "fa9cd720-ddc4-4dc8-9be6-5d0fc8c4a3fb" - vault_name = "staging-vault-6biac" - key_name = "staging-vault-unseal-6biac" - } - - service_registration "kubernetes" {} - - # A disruption budget limits the number of pods of a replicated application - # that are down simultaneously from voluntary disruptions - disruptionBudget: - enabled: true - - # maxUnavailable will default to (n/2)-1 where n is the number of - # replicas. If you'd like a custom value, you can specify an override here. - maxUnavailable: null - - # Definition of the serviceAccount used to run Vault. - # These options are also used when using an external Vault server to validate - # Kubernetes tokens. - serviceAccount: - # Specifies whether a service account should be created - create: true - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - # Create a Secret API object to store a non-expiring token for the service account. - # Prior to v1.24.0, Kubernetes used to generate this secret for each service account by default. - # Kubernetes now recommends using short-lived tokens from the TokenRequest API or projected volumes instead if - # possible. - # For more details, see https://kubernetes.io/docs/concepts/configuration/secret/#service-account-token-secrets - # serviceAccount.create must be equal to 'true' in order to use this feature. - createSecret: false - # Extra annotations for the serviceAccount definition. This can either be - # YAML or a YAML-formatted multi-line templated string map of the - # annotations to apply to the serviceAccount. - annotations: {} - # Extra labels to attach to the serviceAccount - # This should be a YAML map of the labels to apply to the serviceAccount - extraLabels: {} - # Enable or disable a service account role binding with the permissions required for - # Vault's Kubernetes service_registration config option. - # See https://developer.hashicorp.com/vault/docs/configuration/service-registration/kubernetes - serviceDiscovery: - enabled: true - - # Settings for the statefulSet used to run Vault. - statefulSet: - # Extra annotations for the statefulSet. This can either be YAML or a - # YAML-formatted multi-line templated string map of the annotations to apply - # to the statefulSet. - annotations: {} - - # Set the pod and container security contexts. - # If not set, these will default to, and for *not* OpenShift: - # pod: - # runAsNonRoot: true - # runAsGroup: {{ .Values.server.gid | default 1000 }} - # runAsUser: {{ .Values.server.uid | default 100 }} - # fsGroup: {{ .Values.server.gid | default 1000 }} - # container: - # allowPrivilegeEscalation: false - # - # If not set, these will default to, and for OpenShift: - # pod: {} - # container: {} - securityContext: - pod: {} - container: {} - - # Should the server pods run on the host network - hostNetwork: false - -# Vault UI -ui: - # True if you want to create a Service entry for the Vault UI. - # - # serviceType can be used to control the type of service created. For - # example, setting this to "LoadBalancer" will create an external load - # balancer (for supported K8S installations) to access the UI. - enabled: true - publishNotReadyAddresses: true - # The service should only contain selectors for active Vault pod - activeVaultPodOnly: false - serviceType: "ClusterIP" - serviceNodePort: null - externalPort: 8200 - targetPort: 8200 - - # The IP family and IP families options are to set the behaviour in a dual-stack environment. - # Omitting these values will let the service fall back to whatever the CNI dictates the defaults - # should be. - # These are only supported for kubernetes versions >=1.23.0 - # - # Configures the service's supported IP family, can be either: - # SingleStack: Single-stack service. The control plane allocates a cluster IP for the Service, using the first - # configured service cluster IP range. - # PreferDualStack: Allocates IPv4 and IPv6 cluster IPs for the Service. - # RequireDualStack: Allocates Service .spec.ClusterIPs from both IPv4 and IPv6 address ranges. - serviceIPFamilyPolicy: "" - - # Sets the families that should be supported and the order in which they should be applied to ClusterIP as well - # Can be IPv4 and/or IPv6. - serviceIPFamilies: [] - - # The externalTrafficPolicy can be set to either Cluster or Local - # and is only valid for LoadBalancer and NodePort service types. - # The default value is Cluster. - # ref: https://kubernetes.io/docs/concepts/services-networking/service/#external-traffic-policy - externalTrafficPolicy: Cluster - - # loadBalancerSourceRanges: - # - 10.0.0.0/16 - # - 1.78.23.3/32 - - # loadBalancerIP: - - # Extra annotations to attach to the ui service - # This can either be YAML or a YAML-formatted multi-line templated string map - # of the annotations to apply to the ui service - annotations: {} - -# secrets-store-csi-driver-provider-vault -csi: - # True if you want to install a secrets-store-csi-driver-provider-vault daemonset. - # - # Requires installing the secrets-store-csi-driver separately, see: - # https://github.com/kubernetes-sigs/secrets-store-csi-driver#install-the-secrets-store-csi-driver - # - # With the driver and provider installed, you can mount Vault secrets into volumes - # similar to the Vault Agent injector, and you can also sync those secrets into - # Kubernetes secrets. - enabled: true - - image: - repository: "hashicorp/vault-csi-provider" - tag: "1.4.1" - pullPolicy: IfNotPresent - - # volumes is a list of volumes made available to all containers. These are rendered - # via toYaml rather than pre-processed like the extraVolumes value. - # The purpose is to make it easy to share volumes between containers. - volumes: - - name: vault-ha-tls - secret: - secretName: vault-ha-tls - - # volumeMounts is a list of volumeMounts for the main server container. These are rendered - # via toYaml rather than pre-processed like the extraVolumes value. - # The purpose is to make it easy to share volumes between containers. - volumeMounts: - - name: vault-ha-tls - mountPath: "/vault/userconfig/vault-ha-tls" - readOnly: true - - resources: {} - # resources: - # requests: - # cpu: 50m - # memory: 128Mi - # limits: - # cpu: 50m - # memory: 128Mi - - # Override the default secret name for the CSI Provider's HMAC key used for - # generating secret versions. - hmacSecretName: "" - - # Settings for the daemonSet used to run the provider. - daemonSet: - updateStrategy: - type: RollingUpdate - maxUnavailable: "" - # Extra annotations for the daemonSet. This can either be YAML or a - # YAML-formatted multi-line templated string map of the annotations to apply - # to the daemonSet. - annotations: {} - # Provider host path (must match the CSI provider's path) - providersDir: "/etc/kubernetes/secrets-store-csi-providers" - # Kubelet host path - kubeletRootDir: "/var/lib/kubelet" - # Extra labels to attach to the vault-csi-provider daemonSet - # This should be a YAML map of the labels to apply to the csi provider daemonSet - extraLabels: {} - # security context for the pod template and container in the csi provider daemonSet - securityContext: - pod: {} - container: {} - - pod: - # Extra annotations for the provider pods. This can either be YAML or a - # YAML-formatted multi-line templated string map of the annotations to apply - # to the pod. - annotations: {} - - # Toleration Settings for provider pods - # This should be either a multi-line string or YAML matching the Toleration array - # in a PodSpec. - tolerations: [] - - # nodeSelector labels for csi pod assignment, formatted as a multi-line string or YAML map. - # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector - # Example: - # nodeSelector: - # beta.kubernetes.io/arch: amd64 - nodeSelector: {} - - # Affinity Settings - # This should be either a multi-line string or YAML matching the PodSpec's affinity field. - affinity: {} - - # Extra labels to attach to the vault-csi-provider pod - # This should be a YAML map of the labels to apply to the csi provider pod - extraLabels: {} - - agent: - enabled: true - extraArgs: [] - - image: - repository: "hashicorp/vault" - tag: "1.15.2" - pullPolicy: IfNotPresent - - logFormat: standard - logLevel: info - - resources: {} - # resources: - # requests: - # memory: 256Mi - # cpu: 250m - # limits: - # memory: 256Mi - # cpu: 250m - - # Priority class for csi pods - priorityClassName: "" - - serviceAccount: - # Extra annotations for the serviceAccount definition. This can either be - # YAML or a YAML-formatted multi-line templated string map of the - # annotations to apply to the serviceAccount. - annotations: {} - - # Extra labels to attach to the vault-csi-provider serviceAccount - # This should be a YAML map of the labels to apply to the csi provider serviceAccount - extraLabels: {} - - # Used to configure readinessProbe for the pods. - readinessProbe: - failureThreshold: 2 - initialDelaySeconds: 5 - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 3 - - # Used to configure livenessProbe for the pods. - livenessProbe: - failureThreshold: 2 - initialDelaySeconds: 5 - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 3 - - # Enables debug logging. - debug: false - - # Pass arbitrary additional arguments to vault-csi-provider. - # See https://developer.hashicorp.com/vault/docs/platform/k8s/csi/configurations#command-line-arguments - # for the available command line flags. - extraArgs: - - "-vault-tls-ca-cert=/vault/userconfig/vault-ha-tls/vault.ca" - - "-vault-addr=https://vault.vault:8200" - -# Vault is able to collect and publish various runtime metrics. -# Enabling this feature requires setting adding `telemetry{}` stanza to -# the Vault configuration. There are a few examples included in the `config` sections above. -# -# For more information see: -# https://developer.hashicorp.com/vault/docs/configuration/telemetry -# https://developer.hashicorp.com/vault/docs/internals/telemetry -serverTelemetry: - # Enable support for the Prometheus Operator. Currently, this chart does not support - # authenticating to Vault's metrics endpoint, so the following `telemetry{}` must be included - # in the `listener "tcp"{}` stanza - # telemetry { - # unauthenticated_metrics_access = "true" - # } - # - # See the `standalone.config` for a more complete example of this. - # - # In addition, a top level `telemetry{}` stanza must also be included in the Vault configuration: - # - # example: - # telemetry { - # prometheus_retention_time = "30s" - # disable_hostname = true - # } - # - # Configuration for monitoring the Vault server. - serviceMonitor: - # The Prometheus operator *must* be installed before enabling this feature, - # if not the chart will fail to install due to missing CustomResourceDefinitions - # provided by the operator. - # - # Instructions on how to install the Helm chart can be found here: - # https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack - # More information can be found here: - # https://github.com/prometheus-operator/prometheus-operator - # https://github.com/prometheus-operator/kube-prometheus - - # Enable deployment of the Vault Server ServiceMonitor CustomResource. - enabled: false - - # Selector labels to add to the ServiceMonitor. - # When empty, defaults to: - # release: prometheus - selectors: {} - - # Interval at which Prometheus scrapes metrics - interval: 30s - - # Timeout for Prometheus scrapes - scrapeTimeout: 10s - - prometheusRules: - # The Prometheus operator *must* be installed before enabling this feature, - # if not the chart will fail to install due to missing CustomResourceDefinitions - # provided by the operator. - - # Deploy the PrometheusRule custom resource for AlertManager based alerts. - # Requires that AlertManager is properly deployed. - enabled: false - - # Selector labels to add to the PrometheusRules. - # When empty, defaults to: - # release: prometheus - selectors: {} - - # Some example rules. - rules: [] - # - alert: vault-HighResponseTime - # annotations: - # message: The response time of Vault is over 500ms on average over the last 5 minutes. - # expr: vault_core_handle_request{quantile="0.5", namespace="mynamespace"} > 500 - # for: 5m - # labels: - # severity: warning - # - alert: vault-HighResponseTime - # annotations: - # message: The response time of Vault is over 1s on average over the last 5 minutes. - # expr: vault_core_handle_request{quantile="0.5", namespace="mynamespace"} > 1000 - # for: 5m - # labels: - # severity: critical From 0d6108e2a04931f52ec00e69ae7be1a248780e01 Mon Sep 17 00:00:00 2001 From: SonOfLope Date: Fri, 8 Mar 2024 13:33:41 -0500 Subject: [PATCH 18/20] Issue #56: revert value file changes --- kubernetes/aks/system/vault/helm/vault.values.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kubernetes/aks/system/vault/helm/vault.values.yml b/kubernetes/aks/system/vault/helm/vault.values.yml index 2acfc80f..f218c950 100644 --- a/kubernetes/aks/system/vault/helm/vault.values.yml +++ b/kubernetes/aks/system/vault/helm/vault.values.yml @@ -410,7 +410,7 @@ server: # configure the ingress to point to the Vault active service. activeService: true hosts: - - host: vault.azure.sonoflope.ca + - host: vault.inspection.alpha.canada.ca # For future use. paths: [] ## Extra paths to prepend to the host configuration. This is useful when working with annotation based services. extraPaths: [] @@ -423,7 +423,7 @@ server: tls: - secretName: vault-tls hosts: - - vault.azure.sonoflope.ca + - vault.inspection.alpha.canada.ca # For future use. # hostAliases is a list of aliases to be added to /etc/hosts. Specified as a YAML list. hostAliases: [] @@ -820,10 +820,10 @@ server: seal "azurekeyvault" { - tenant_id = "912b82dc-b892-42e5-ae31-c0ab5350455e" - client_id = "80bfe77e-97f0-46dd-9cc8-bd6c3b7fb050" - vault_name = "staging-vault-6a3fp" - key_name = "staging-vault-unseal-6a3fp" + tenant_id = "18b5a5ed-1d86-41d3-94a0-bc27dae32ab2" + client_id = "fa9cd720-ddc4-4dc8-9be6-5d0fc8c4a3fb" + vault_name = "staging-vault-6biac" + key_name = "staging-vault-unseal-6biac" } service_registration "kubernetes" {} From 305ab4f219a460ffa24087e217d606d4c794367b Mon Sep 17 00:00:00 2001 From: SonOfLope Date: Fri, 8 Mar 2024 13:35:01 -0500 Subject: [PATCH 19/20] Issue #56: update value file name --- kubernetes/aks/system/vault/argo-app.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/aks/system/vault/argo-app.yaml b/kubernetes/aks/system/vault/argo-app.yaml index bac74d5c..4726265a 100644 --- a/kubernetes/aks/system/vault/argo-app.yaml +++ b/kubernetes/aks/system/vault/argo-app.yaml @@ -18,7 +18,7 @@ spec: helm: releaseName: vault valueFiles: - - $values/kubernetes/aks/system/vault/helm/values.yaml + - $values/kubernetes/aks/system/vault/helm/vault.values.yml - repoURL: https://github.com/ai-cfia/howard.git targetRevision: HEAD ref: values From eb666e205f356ce6050f53e81c84689152595ca7 Mon Sep 17 00:00:00 2001 From: SonOfLope Date: Tue, 19 Mar 2024 10:08:44 -0400 Subject: [PATCH 20/20] Issue #56: update vault documentation with argo plugin --- docs/secret-management.md | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/secret-management.md b/docs/secret-management.md index 9284a1d8..4d920f97 100644 --- a/docs/secret-management.md +++ b/docs/secret-management.md @@ -31,8 +31,8 @@ access secrets as files, which is a secure and efficient way to manage secrets in a Kubernetes environment. The following diagram illustrates the workflow of the Vault Agent Injector and -how developers can manage secrets of hosted applications: -![Developer workflow diagram](img/vault-argocd-workflow.svg) +how developers can manage secrets of hosted applications: ![Developer workflow +diagram](img/vault-argocd-workflow.svg) ## Secret management process @@ -64,9 +64,9 @@ The secret management process involves the following steps: Vault provides a UI service to manage secrets. The UI service is a web-based user interface that allows administrators to create, read, update, and delete -secrets. The service also provides a way to manage access control policies -and audit logs. The service is accessible through a web browser and is -protected by the same security mechanisms as the Vault server. +secrets. The service also provides a way to manage access control policies and +audit logs. The service is accessible through a web browser and is protected by +the same security mechanisms as the Vault server. ### Steps @@ -87,3 +87,15 @@ protected by the same security mechanisms as the Vault server. 5. Once in the directory of your application secrets, simply click on 'create new version' and you will be able to add, update, or delete secrets as needed. ![Create mew secret](img/create-new-secret.png) + +## Argo CD Vault plugin (AVP) + +The [argocd-vault-plugin](https://argocd-vault-plugin.readthedocs.io/en/stable/) +is used to manage secrets inside our deployments the Gitops way. It allows to +use `` in any YAML or JSON files that have been templated and make +use of annotations to provide the path and version of a secret inside vault. + +An example of usage is showcased inside the demo app sample. The official +[documentation](https://argocd-vault-plugin.readthedocs.io/en/stable/howitworks/) +for the plugin is well explained and can be followed according to the usecase +needed.