From 013cdba98c8b81950088d9f1662efa9dc8e2eb26 Mon Sep 17 00:00:00 2001 From: Zhiwei Yin Date: Sun, 13 Oct 2024 16:31:07 +0000 Subject: [PATCH] update e2e to import hosted cluster Signed-off-by: Zhiwei Yin --- .github/workflows/e2e.yaml | 49 ++++++++-- Makefile | 7 +- configuration/klusterletconfig.yaml | 2 +- e2e/configuration/cluster.yaml | 14 +++ e2e/configuration/klusterletconfig.yaml | 6 ++ e2e/configuration/mce-values.yaml | 31 ++++++ e2e/configuration/multiclusterengine.yaml | 40 ++++++++ e2e/configuration/policy-values.yaml | 28 ++++++ .../templates/multiclusterengine.yaml | 2 +- e2e/mce-chart/values.yaml | 3 +- hack/e2e-import-cluster.sh | 98 +++++++++++++++++++ hack/{e2e.sh => e2e-install.sh} | 23 +++-- kubeconfig-kind-spoke-insternal | 20 ++++ 13 files changed, 302 insertions(+), 21 deletions(-) create mode 100644 e2e/configuration/cluster.yaml create mode 100644 e2e/configuration/klusterletconfig.yaml create mode 100644 e2e/configuration/mce-values.yaml create mode 100644 e2e/configuration/multiclusterengine.yaml create mode 100644 e2e/configuration/policy-values.yaml create mode 100755 hack/e2e-import-cluster.sh rename hack/{e2e.sh => e2e-install.sh} (72%) create mode 100644 kubeconfig-kind-spoke-insternal diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 84faf03..348bc83 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -28,19 +28,48 @@ jobs: uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} - - name: Setup kind - uses: engineerd/setup-kind@v0.5.0 + - name: Install kind + uses: helm/kind-action@v1.10.0 with: version: v0.22.0 - skipClusterCreation: ${{ env.USE_EXISTING_CLUSTER }} - - name: Set KUBECONFIG + install_only: true + - name: Test kind works and there is no cluster started + run: | + [[ $(kind get clusters | wc -l) -eq 0 ]] + - name: Set KUBECONFIG dir run: | mkdir -p /home/runner/.kube - kind get kubeconfig > /home/runner/.kube/config - if: ${{ env.USE_EXISTING_CLUSTER }} - - name: Test E2E + - name: Create Hub cluster + run: | + kind create cluster --name hub + env: + KUBECONFIG: /home/runner/.kube/hub-kubeconfig + - name: Create Spoke cluster + run: | + kind create cluster --name spoke + env: + KUBECONFIG: /home/runner/.kube/spoke-kubeconfig + - name: Test Hub and Spoke clusters + run: | + kubectl cluster-info --kubeconfig /home/runner/.kube/hub-kubeconfig + kubectl get pods -A --kubeconfig /home/runner/.kube/hub-kubeconfig + kubectl cluster-info --kubeconfig /home/runner/.kube/spoke-kubeconfig + kubectl get pods -A --kubeconfig /home/runner/.kube/spoke-kubeconfig + - name: Create internal kubeconfigs + run: | + kind get kubeconfig --internal --name hub > /home/runner/.kube/hub-internal-kubeconfig + kind get kubeconfig --internal --name spoke > /home/runner/.kube/spoke-internal-kubeconfig + - name: Install MCE+Policy + run: | + make e2e-install + env: + KUBECONFIG: /home/runner/.kube/hub-kubeconfig + + - name: Import hosted cluster run: | - make test-e2e + make e2e-import-cluster env: - KUBECONFIG: /home/runner/.kube/config - \ No newline at end of file + KUBECONFIG: /home/runner/.kube/hub-kubeconfig + MANAGED_CLUSTER_NAME: spoke + MANAGED_KUBECONFIG: /home/runner/.kube/spoke-internal-kubeconfig + EXTERNAL_MANAGED_KUBECONFIG: /home/runner/.kube/spoke-internal-kubeconfig diff --git a/Makefile b/Makefile index ce65aaa..3d1a2b1 100644 --- a/Makefile +++ b/Makefile @@ -35,8 +35,11 @@ install-mce: ensure-helm install-policy: ensure-helm $(HELM) install policy ./policy -test-e2e: - hack/e2e.sh +e2e-install: + hack/e2e-install.sh + +e2e-import-cluster: + hack/e2e-import-cluster.sh ensure-helm: ifeq "" "$(wildcard $(HELM))" diff --git a/configuration/klusterletconfig.yaml b/configuration/klusterletconfig.yaml index 4ddfa87..9c1895a 100644 --- a/configuration/klusterletconfig.yaml +++ b/configuration/klusterletconfig.yaml @@ -3,4 +3,4 @@ kind: KlusterletConfig metadata: name: global spec: - hubKubeAPIServerURL: "https://kind-control-plane:6443" + hubKubeAPIServerURL: "hub cluster api server url" diff --git a/e2e/configuration/cluster.yaml b/e2e/configuration/cluster.yaml new file mode 100644 index 0000000..cd08491 --- /dev/null +++ b/e2e/configuration/cluster.yaml @@ -0,0 +1,14 @@ +apiVersion: cluster.open-cluster-management.io/v1 +kind: ManagedCluster +metadata: + name: spoke + annotations: + import.open-cluster-management.io/klusterlet-deploy-mode: Hosted + import.open-cluster-management.io/hosting-cluster-name: local-cluster + addon.open-cluster-management.io/enable-hosted-mode-addons: "true" + open-cluster-management/created-via: other + labels: + cluster.open-cluster-management.io/clusterset: default +spec: + hubAcceptsClient: true + leaseDurationSeconds: 60 diff --git a/e2e/configuration/klusterletconfig.yaml b/e2e/configuration/klusterletconfig.yaml new file mode 100644 index 0000000..7751a81 --- /dev/null +++ b/e2e/configuration/klusterletconfig.yaml @@ -0,0 +1,6 @@ +apiVersion: config.open-cluster-management.io/v1alpha1 +kind: KlusterletConfig +metadata: + name: global +spec: + hubKubeAPIServerURL: "https://hub-control-plane:6443" diff --git a/e2e/configuration/mce-values.yaml b/e2e/configuration/mce-values.yaml new file mode 100644 index 0000000..303dae8 --- /dev/null +++ b/e2e/configuration/mce-values.yaml @@ -0,0 +1,31 @@ + +replicaCount: 1 +availabilityConfig: Basic + +images: + overrides: + backplane_operator: "quay.io/stolostron/backplane-operator:2.7.0-BACKPLANE-2024-09-30-05-39-18" + registration_operator: "quay.io/stolostron/registration-operator:2.7.0-BACKPLANE-2024-09-30-05-39-18" + hypershift_addon_operator: "quay.io/stolostron/hypershift-addon-operator:2.7.0-BACKPLANE-2024-09-30-05-39-18" + managedcluster_import_controller: "quay.io/stolostron/managedcluster-import-controller:2.7.0-BACKPLANE-2024-09-30-05-39-18" + multicloud_manager: "quay.io/stolostron/multicloud-manager:2.7.0-BACKPLANE-2024-09-30-05-39-18" + addon_manager: "quay.io/stolostron/addon-manager:2.7.0-BACKPLANE-2024-09-30-05-39-18" + work: "quay.io/stolostron/work:2.7.0-BACKPLANE-2024-09-30-05-39-18" + registration: "quay.io/stolostron/registration:2.7.0-BACKPLANE-2024-09-30-05-39-18" + placement: "quay.io/stolostron/placement:2.7.0-BACKPLANE-2024-09-30-05-39-18" + kube_rbac_proxy_mce: "quay.io/stolostron/kube-rbac-proxy-mce:2.7.0-BACKPLANE-2024-09-30-05-39-18" + + + #images in MCE 2.6.2 + # backplane_operator: "registry.redhat.io/multicluster-engine/backplane-rhel9-operator@sha256:eb15286f728e32851b426f86b3a1ce5ed186cdb1a67287e98ff5925dc558a2a9" + # registration_operator: "registry.redhat.io/multicluster-engine/registration-operator-rhel9@sha256:efe0091dd6d389190c0acf47cb8980ed8ba7d90bdf04b3b2e9dc07926c0d0bca" + # hypershift_addon_operator: "registry.redhat.io/multicluster-engine/hypershift-addon-rhel9-operator@sha256:7b375b10a5f6434aad3801e486e51a9404d88899a84593686856f3340a5889fa" + # managedcluster_import_controller: "registry.redhat.io/multicluster-engine/managedcluster-import-controller-rhel9@sha256:89dea3fa0cc7cb182d49436bb93214af73bac1383caa59766c97b6fdad2b14b5" + # multicloud_manager: "registry.redhat.io/multicluster-engine/multicloud-manager-rhel9@sha256:089188b25407a49ba042bd65a8afc6a6c86d68b8d7ac3da722196f0cd85383ae" + # addon_manager: "registry.redhat.io/multicluster-engine/addon-manager-rhel9@sha256:93b21a4356230da6f70edac4d8770b5e0af2f40a37ec454b00b12b5aa76fb4c3" + # work: "registry.redhat.io/multicluster-engine/work-rhel9@sha256:bf6c7384283046093605659e866ed6a4e2c7c81690f5180894099d4691b97aa3" + # registration: "registry.redhat.io/multicluster-engine/registration-rhel9@sha256:e05fe0fb10bd9abd3f653fc01ea25dab0e78b3dbe932b2da3375585dffc7f4b6" + # placement: "registry.redhat.io/multicluster-engine/placement-rhel9@sha256:307b984b4315b9e22c520be893c2522d4bf3090cd90b10ba1aefb740bc7b7cc2" + # kube_rbac_proxy_mce: "registry.redhat.io/multicluster-engine/kube-rbac-proxy-mce-rhel9@sha256:b5f6f36487e70b543afacd33255b0c4f504bbd62bd5b7f46648c098db4466393" + imageCredentials: + dockerConfigJson: "" diff --git a/e2e/configuration/multiclusterengine.yaml b/e2e/configuration/multiclusterengine.yaml new file mode 100644 index 0000000..8882bfe --- /dev/null +++ b/e2e/configuration/multiclusterengine.yaml @@ -0,0 +1,40 @@ +apiVersion: multicluster.openshift.io/v1 +kind: MultiClusterEngine +metadata: + name: multiclusterengine +spec: + availabilityConfig: Basic + imagePullSecret: open-cluster-management-image-pull-credentials + overrides: + components: + - enabled: true + name: local-cluster + - enabled: true + name: cluster-manager + - enabled: true + name: server-foundation + - enabled: true + name: hypershift-local-hosting + - enabled: true + name: hypershift + - enabled: false + name: cluster-lifecycle + - enabled: false + name: discovery + - enabled: false + name: console-mce + - enabled: false + name: hive + - enabled: false + name: assisted-service + - enabled: false + name: image-based-install-operator-preview + - enabled: false + name: cluster-proxy-addon + - enabled: false + name: managedserviceaccount + targetNamespace: multicluster-engine + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/infra + operator: Exists diff --git a/e2e/configuration/policy-values.yaml b/e2e/configuration/policy-values.yaml new file mode 100644 index 0000000..7543d6b --- /dev/null +++ b/e2e/configuration/policy-values.yaml @@ -0,0 +1,28 @@ +global: + imageOverrides: + # upstream images + governance_policy_propagator: "quay.io/stolostron/governance-policy-propagator:2.12.0-SNAPSHOT-2024-09-30-01-46-06" + governance_policy_addon_controller: "quay.io/stolostron/governance-policy-addon-controller:2.12.0-SNAPSHOT-2024-09-30-01-46-06" + config_policy_controller: "quay.io/stolostron/config-policy-controller:2.12.0-SNAPSHOT-2024-09-30-01-46-06" + governance_policy_framework_addon: "quay.io/stolostron/governance-policy-framework-addon:2.12.0-SNAPSHOT-2024-09-30-01-46-06" + klusterlet_addon_controller: "quay.io/stolostron/klusterlet-addon-controller:2.12.0-SNAPSHOT-2024-09-30-01-46-06" + + # images in ACM 2.11.2 + # governance_policy_propagator: "registry.redhat.io/rhacm2/governance-policy-propagator-rhel9@sha256:af848e7e31d8ec9b5ad1896a5d5ccc67f320a7740245c190ba8a76757984e65b" + # governance_policy_addon_controller: "registry.redhat.io/rhacm2/acm-governance-policy-addon-controller-rhel9@sha256:fc0708f0a6d5266fb544f41b61d9697d370c8c5e297e4e3f13de8656f9c2b049" + # config_policy_controller: "registry.redhat.io/rhacm2/config-policy-controller-rhel9@sha256:cecf914d7fb7759a4f512c1ec53a077dcb1c7e405c22a5bf6af1bf5878cf3c42" + # governance_policy_framework_addon: "registry.redhat.io/rhacm2/acm-governance-policy-framework-addon-rhel9@sha256:a4880f6e82d2b82606203ea855d0418bb29b3d4535f8bc7a9ef4074258c18674" + # klusterlet_addon_controller: "registry.redhat.io/rhacm2/klusterlet-addon-controller-rhel9@sha256:478e3e6cda0d74f43b0f05911d023344108a5cd79d57d5cc9f268ad064848a00" + namespace: multicluster-engine + pullSecret: open-cluster-management-image-pull-credentials + + cma: + defaultConfig: + name: addon-hosted-config + namespace: multicluster-engine +grc: + hubconfig: + replicaCount: 1 +cluster-lifecycle: + hubconfig: + replicaCount: 1 diff --git a/e2e/mce-chart/templates/multiclusterengine.yaml b/e2e/mce-chart/templates/multiclusterengine.yaml index ebe6051..1f05cb8 100644 --- a/e2e/mce-chart/templates/multiclusterengine.yaml +++ b/e2e/mce-chart/templates/multiclusterengine.yaml @@ -3,7 +3,7 @@ kind: MultiClusterEngine metadata: name: multiclusterengine spec: - availabilityConfig: High + availabilityConfig: {{ .Values.availabilityConfig }} imagePullSecret: open-cluster-management-image-pull-credentials overrides: components: diff --git a/e2e/mce-chart/values.yaml b/e2e/mce-chart/values.yaml index cc62e9f..28a1ca8 100644 --- a/e2e/mce-chart/values.yaml +++ b/e2e/mce-chart/values.yaml @@ -1,5 +1,6 @@ -replicaCount: 1 +replicaCount: 2 +availabilityConfig: High images: overrides: diff --git a/hack/e2e-import-cluster.sh b/hack/e2e-import-cluster.sh new file mode 100755 index 0000000..fa7ff53 --- /dev/null +++ b/hack/e2e-import-cluster.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +# env +# export KUBECONFIG=kubeconfig +# export MANAGED_KUBECONFIG=managed-kubeconfig +# export EXTERNAL_MANAGED_KUBECONFIG=external-managed-kubeconfig +# export MANAGED_CLUSTER_NAME=spoke + +KLUSTERLET_NS=klusterlet-$MANAGED_CLUSTER_NAME + +function waitForReady() { + FOUND=1 + SECOND=0 + local cmd="$1" + local rst="$2" + + echo "check if \"$cmd\" == $rst" + + while [ ${FOUND} -eq 1 ]; do + if [ $SECOND -gt 240 ]; then + echo "Timeout waiting for the result of cmd. " + + kubectl get pods -A + kubectl get mcl + kubectl get mce multiclusterengine -o yaml + + exit 1 + fi + + result=$(bash -c "$cmd" 2>/dev/null || true) + if [ "$rst" -eq "$result" ]; then + echo "pass " + break + fi + + echo "expected $rst, but got $result, re-try after 5 seconds..." + sleep 5 + (( SECOND = SECOND + 5 )) + done +} + + +echo "" +echo "###### Create managed cluster ns ###### " +result=$(kubectl get ns | grep -c $MANAGED_CLUSTER_NAME 2>/dev/null || true) +if [ $result -eq 0 ] ; then +kubectl create ns $MANAGED_CLUSTER_NAME; +fi + +echo "" +echo "###### Create auto-import secret" +result=$(kubectl get secret -n $MANAGED_CLUSTER_NAME | grep -c auto-import-secret 2>/dev/null || true) +if [ $result -eq 0 ]; then + kubectl create secret generic auto-import-secret --from-file=kubeconfig=$MANAGED_KUBECONFIG -n $MANAGED_CLUSTER_NAME +fi + +echo "" +echo "###### Create managedCluster ######" +cat << EOF | kubectl apply -f - +apiVersion: cluster.open-cluster-management.io/v1 +kind: ManagedCluster +metadata: + name: $MANAGED_CLUSTER_NAME + annotations: + import.open-cluster-management.io/klusterlet-deploy-mode: Hosted + import.open-cluster-management.io/hosting-cluster-name: local-cluster + addon.open-cluster-management.io/enable-hosted-mode-addons: "true" + open-cluster-management/created-via: other + labels: + cluster.open-cluster-management.io/clusterset: default +spec: + hubAcceptsClient: true + leaseDurationSeconds: 60 +EOF + +echo "" +echo "###### Create external-managed-kubeconfig secret ######" +waitForReady "kubectl get ns | grep -c \"klusterlet-$MANAGED_CLUSTER_NAME\"" 1 +result=$(kubectl get secret -n $KLUSTERLET_NS | grep -c external-managed-kubeconfig 2>/dev/null || true) +if [ $result -eq 0 ]; then + kubectl create secret generic external-managed-kubeconfig --from-file=kubeconfig=$EXTERNAL_MANAGED_KUBECONFIG -n $KLUSTERLET_NS +fi + +echo "" +echo "###### Wait for $MANAGED_CLUSTER_NAME is ready ######" +waitForReady "kubectl get mcl $MANAGED_CLUSTER_NAME | grep -c \"True\"" 1 + +echo "" +echo "###### Wait unitl 3 addons in $MANAGED_CLUSTER_NAME is Available ######" +waitForReady "kubectl get mca -n $MANAGED_CLUSTER_NAME | grep -c \"True\"" 3 + +echo "" +echo "!!!!!!!!!! hosted cluster $MANAGED_CLUSTER_NAME is imported succussfully !!!!!!!!!!!!" +echo "" diff --git a/hack/e2e.sh b/hack/e2e-install.sh similarity index 72% rename from hack/e2e.sh rename to hack/e2e-install.sh index 7aad67a..05380bd 100755 --- a/hack/e2e.sh +++ b/hack/e2e-install.sh @@ -4,6 +4,10 @@ set -o errexit set -o nounset set -o pipefail +# env +export HELM=_output/bin/helm +# export KUBECONFIG=kubeconfig + function waitForReady() { FOUND=1 SECOND=0 @@ -37,8 +41,10 @@ function waitForReady() { echo "" -echo "#### Install MCE ####" -make install-mce +echo "#### Install MCE on Hub cluster ####" +make ensure-helm +$HELM install mce ./e2e/mce-chart -f ./e2e/configuration/mce-values.yaml + echo "" echo "###### Wait until MCE pod is running ######" @@ -57,12 +63,11 @@ waitForReady "kubectl get crds | grep -c \"klusterletconfigs\"" 1 echo "" echo "###### Create global klusterletconfig ######" -# set hub api server to ./configuration/klusterletconfig.yaml -kubectl apply -f ./configuration/klusterletconfig.yaml +kubectl apply -f ./e2e/configuration/klusterletconfig.yaml echo "" echo "###### Wait unitl local-cluster is created ######" -waitForReady "kubectl get mcl | grep -c \"local-cluster\"" 1 +waitForReady "kubectl get mcl local-cluster | grep -c \"True\"" 1 echo "" @@ -73,10 +78,16 @@ echo "" echo "###### create addonhostedconfig ######" kubectl apply -f ./configuration/addonhostedconfig.yaml +echo "" +echo "###### patch clustermanagementaddon work-manager ######" +#kubectl patch clustermanagementaddon work-manager --type merge -p '{"spec":{"supportedConfigs":[{"defaultConfig":{"name":"addon-hosted-config","namespace":"multicluster-engine"},"group":"addon.open-cluster-management.io","resource":"addondeploymentconfigs"}]}}' +kubectl apply -f ./configuration/workmanagercma.yaml echo "" echo "#### Install Policy addons #####" -make install-policy +make ensure-helm +$HELM install policy ./policy -f ./e2e/configuration/policy-values.yaml + echo "" echo "###### Enable policy addons for local-cluster ######" diff --git a/kubeconfig-kind-spoke-insternal b/kubeconfig-kind-spoke-insternal new file mode 100644 index 0000000..69b735c --- /dev/null +++ b/kubeconfig-kind-spoke-insternal @@ -0,0 +1,20 @@ +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURCVENDQWUyZ0F3SUJBZ0lJV2Niak0yQ3RZTWd3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TkRFd01UTXhOakEwTkRkYUZ3MHpOREV3TVRFeE5qQTVORGRhTUJVeApFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLCkFvSUJBUUM1b1ltcjhpeTZ0dDd4TW8zRm5sTzcvL2JQOFlncXU5T3MwRWxabGMwYUZ0YXlpeUVNYTNtQ0dxeDUKWXdRWktaQXF3WkRRUCtCNk8wY2RVZGhOQmtjeEx0V0pBdzdzNEtsWGw4OVdzbVJiall3RE0vWCtPN1NyRWE1SApLMXIwY3pHcVhHTThWMTlIcVZhMVU2NlhYMFpaak1MaXpUWDM0Skpnb0llMlFWOHl4ZEgxd2NyMXRHT2JkRUhzCi9FdVlucllSZ2lJb2xYQ2hKQUxSVldNM2dyQnZ6SFduY3FGUHYwOE9hSmhpM0pCcWc4ZXFYbUdxU2Q2NFc1SHEKQklnb0Joa09UeklPbUs2NUFTQjIyUzlwdjVIREgrRTBrNDVUZXRFb0czWjZEWHVwR3FlUkJ6aVptWkVmNUw1VQp6WEZJMzR3OGtiQUtLV1N2ajNKMTJtbm4rU1N4QWdNQkFBR2pXVEJYTUE0R0ExVWREd0VCL3dRRUF3SUNwREFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUIwR0ExVWREZ1FXQkJURkI3MFQ0enFoSkNiSGhja2lwWUhrSEVmcVJEQVYKQmdOVkhSRUVEakFNZ2dwcmRXSmxjbTVsZEdWek1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQzRxRkkwNUduMApkZjBPd0o0YXNOcFZsN2d1Y0c4TmcyVjZQN3JSSkt6VFQ4bnpNb1ljbCt0amo4dldHcy84QUhrVUpxWDlUYk53ClZrNmF4YnVzSmdock03MkcyeExsMHo0amRNSXhuT2RqQ1lnajhNdmcycDRyYU1uUDVLWFVpWmRPQUdMeGJ4aC8KMjliemJOR0loQTFlMEpGWE9KV0tia25qcjFubGRyZ2U5T280cXVwWlJ5K3hiR0ZzNkhuMjlGQXNmWHI1K1dJWgpyOVlyVjNhWDh1RTdweThFbDRTU240ZnhINWZMVkgzVVdOdHF3OE54aHh2WXh2cjhvLzlTQnd2Zys3OFVhY1NxClFkWEtIc0l4NVNpVStjT1hMWkp6MFVPQlk2NFNZOXg0bXpZa3V5RTQ4aUFFZFhpTkdQUlFid2tNbkpXRDc4NFMKTGlLSzhBdFR2blpjCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + server: https://spoke-control-plane:6443 + name: kind-spoke +contexts: +- context: + cluster: kind-spoke + user: kind-spoke + name: kind-spoke +current-context: kind-spoke +kind: Config +preferences: {} +users: +- name: kind-spoke + user: + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURLVENDQWhHZ0F3SUJBZ0lJTmRSRDk2algzaGd3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TkRFd01UTXhOakEwTkRkYUZ3MHlOVEV3TVRNeE5qQTVORGxhTUR3eApIekFkQmdOVkJBb1RGbXQxWW1WaFpHMDZZMngxYzNSbGNpMWhaRzFwYm5NeEdUQVhCZ05WQkFNVEVHdDFZbVZ5CmJtVjBaWE10WVdSdGFXNHdnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCRHdBd2dnRUtBb0lCQVFDeWpWaTYKS0p6MzJwMXhick5ZWEJoUGpxblQvZlExa0I2ZWszTkxuSE5qdHVWZFJlRXZGeEVaTGVad2RoKzJQUWpCZzY2TApjK3Nvenc3RW9OMEwxWHl6Q3lmT0phMHFYeENvbmdyTjNMUk94S3RNamJTeUppUnl2RXNqZW9yU0VYUjgrclh0CktKQ0s3L2JzZ2JNTVJON1JPZVBYbzJGTk5VVkZxdmZYWWZHUTd6RXZtemJoN1FSQ2dPM2E2bGZhUzJXWXErZCsKRXczbjg1aHZQVnVjeStJZVRpWTRDZDMzY1BnOGU5cWsvZEpWNldoNXg3bWZNT1NKS1Evbk9VdHoraWFpWWllTAo4c0pTaGJGcnZ3a2RBUlZuRlRXSHF0TmFMNmdNdzdUaGJuSVl6WWQvTVFPOUowK1dQaUxwT1pKeXlWeTM1VEh6CmFTMVo4eG5rM1ROUU1lUlBBZ01CQUFHalZqQlVNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUsKQmdnckJnRUZCUWNEQWpBTUJnTlZIUk1CQWY4RUFqQUFNQjhHQTFVZEl3UVlNQmFBRk1VSHZSUGpPcUVrSnNlRgp5U0tsZ2VRY1IrcEVNQTBHQ1NxR1NJYjNEUUVCQ3dVQUE0SUJBUUFQUnBtUjlnbWE1STc2cWtIMFllcXl2cmVsCjJ5MjVxcGQyK1ZxSXhhVDliQ3M1eG14Rk9TaHAzVXZUbXFjUzlkZzd1TnNDUERUakZVY2ZjUEd5bCszeGt6ZFkKV24xSmxhZWJOUlFUcjdUZjVyR0dnUk9UVHlYRTBVUmt1QzVMN2thdHZQamNhK0dtSHV3NEp5elVRV2g1VkdVYQpRdXZURHVHeUF3aCt5MGlsRUY1V1pId3lsRFhudVB1Ylo5ZXIzR21ranNzY05TV2lGYXhDMkJZTERHejRYclIzCmVkR3B2ZkJZajllWGhCbjN5OGxZeDdKeGZEb3h1Nm5ZMnJlZjVybG1xck94WmJqd0hrbHpBQmJTNnNaR3ViVnIKY0h5WFg3WDh3ay9rWDZrZzhnR3ZsZy9rWURsQ1pGcEk2ZUZYR29VOXRGbFdxeGY5VnhEL3F1aDVxTDFHCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBc28xWXVpaWM5OXFkY1c2eldGd1lUNDZwMC8zME5aQWVucE56UzV4elk3YmxYVVhoCkx4Y1JHUzNtY0hZZnRqMEl3WU91aTNQcktNOE94S0RkQzlWOHN3c256aVd0S2w4UXFKNEt6ZHkwVHNTclRJMjAKc2lZa2NyeExJM3FLMGhGMGZQcTE3U2lRaXUvMjdJR3pERVRlMFRuajE2TmhUVFZGUmFyMzEySHhrTzh4TDVzMgo0ZTBFUW9EdDJ1cFgya3RsbUt2bmZoTU41L09ZYnoxYm5NdmlIazRtT0FuZDkzRDRQSHZhcFAzU1ZlbG9lY2U1Cm56RGtpU2tQNXpsTGMvb21vbUluaS9MQ1VvV3hhNzhKSFFFVlp4VTFoNnJUV2krb0RNTzA0VzV5R00ySGZ6RUQKdlNkUGxqNGk2VG1TY3NsY3QrVXg4Mmt0V2ZNWjVOMHpVREhrVHdJREFRQUJBb0lCQUNxOFZnNUloamwwbzladwpqcENKaW5BUVQybWtFUDE0L3pzcFE2RkhVdjc4MmEwWGVxeFFzeEQ1d1h3TmFjMnVraGZ2TXFuQ0Z2anZoak5kCjVVcGQwTGp4NUpjY3dYT3d5VVh5ZmRlRUNjd0ltbzk2STFPNGVXVFUySThuVjI2c0t0dDNEbkF6RTFXTjFlWUoKclJjMHJMZ2JTUU5sNFhZZlYvTXkxcHRBb0JiMklrUEg1UlM1bEt2MFVaZFJKclFla0hNK1VmNXJoS2ZSWTJmSgowNzJHcXBvMDBtOThtOHlDbTJVOWZXemg0QXU5ZVZ1NjJhcGNIZlM5M2xQaUhZdUNSY2tXMFFDeTRoRC9ONjA4CnR5Z2YyV0Y5OGVZQkdmUFM2N3FOVFFqM3FHVk9JWDBIVGdDT3dhYXR5OVBhc0k5WHh6WDdWQjlabjJjOU9SdEsKczM3LyttRUNnWUVBN1RlQkdSQU5sV2E1blQ0b3lpMVY2elV0b0V1cEluK0orTVhQMWwwZjdmdVQ3UU5TaXFIdApQVjJIVFg4eDU5dTRobkFnM2xVdEpHL1Y0NG1YZEZGQTJBNDVVazFrTlZLaVk4dXNaVG1FYkswWFhpa0lYYzN2CnRiYzBqTkpjTVlZWEp2UDlENmJKcEFENkxod0QxRVdTU2tYejcwd25sYjNFK3hOQ0I4TWVUMWtDZ1lFQXdMQ3UKalQ2ZTBJYlU1cG9JNThhSFJqTVRJYTVLN1JWWGRKOVNYd21EckUvcFVEWmxlWUZZSDZyaERkZmxHV3UwMUxOaApIUktnak8vb0tQbWxtbmFENHNjQThkaHRKYUQ3RmxNaDhUTlBNdWVpVTdzbmdMcDh1d3k0cGo0eFp0aEN5SzVQCnJsNk44VGJ4ZHh2RWNORmtSU3dsNkd4NWhwOTNkVVoyWjJzMFErY0NnWUI2UnVBZkFTMWZVOEplRVlxSkhQRnEKK3RCNERrZ3k1amRDcXdURlpOOXVsdjNiY0pqOXFSWlhTWHpUTDd1VDNxaXhjOThkZkI1MjU4Y0RNUXVIQTlNLwpDNlgwakx3WTJHMm9yM1kvLzNRSmFQZmdxNW9LY0hzZDJrQVdUdUVERGpHay9LUDNpMnZwUkMreDBQTVFXb1JjCkVNNzdEUEJpdVoxdUpyRWt1eWxHa1FLQmdRQ0dGaXpSUXlNMnpxeTJiN2dFdVlVRzE2SFhqVGE4ZGw1dVhEUHcKeUdrbGJsSllMQkkvRkQvamdZQ3NwbkRaV0xiMFVJTWl2UVNXTHBmcWM1Ykd2dlFWeWcraE84N3dJVzY0WVU4OApUazB0aWp6T2NXMFN5akxqNHYxWFNlNE11Qzc1QVR1WGhsclY0VHIwZkpFZFJNaTJ6ZnJ2R3hVU0ZrUUZpdXY0CkdZUzhQd0tCZ0QzQlRneGJxNXVHWmlvMFRkbFlhZW8vL1c5Y3F0akoyS0JZTm1Jb2N2VWV6T2ljeTlZRVAzVVEKNnVOU2YxRXZWN3BoNFlGNzJNN0ExYUZzTklhSnYyb3IwVWZWVWVKT3k0ejFkaUxHQklJcFpnT0JySmREcXhVKwphek1ZUW9CK0lnbmVWWDZ3YkRzaTBGU00xaUh5QTFka1lHQkpxTUFNaTFwR012SDlaTlpNCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== +