From 8cffe672ea2144fedeb702da71091003d921629f Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Mon, 11 Dec 2023 17:51:23 +0100 Subject: [PATCH] Add helm to CI --- .../terraform-commons-dss/main.tf | 66 ++++++++++++++++- .../terraform-commons-dss/output.tf | 6 +- deploy/operations/Dockerfile | 28 ++++++- .../ci/aws-1/kubernetes_admin_access.tf | 73 ++++++++++--------- deploy/operations/ci/aws-1/main.tf | 21 ++++-- deploy/operations/ci/aws-1/output.tf | 7 ++ deploy/operations/ci/aws-1/providers.tf | 12 ++- deploy/operations/ci/aws-1/test.sh | 47 ++++++++++-- deploy/operations/docker-compose.yaml | 4 +- .../dss/templates/dss-core-service.yaml | 2 +- 10 files changed, 206 insertions(+), 60 deletions(-) diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/main.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/main.tf index 10523890c..680f57fe5 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/main.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/main.tf @@ -65,4 +65,68 @@ resource "local_file" "get_credentials" { get_credentials_cmd = var.kubernetes_get_credentials_cmd }) filename = "${local.workspace_location}/get-credentials.sh" -} \ No newline at end of file +} + +resource "local_file" "helm_chart_values" { + filename = "${local.workspace_location}/helm_values.yml" + content = yamlencode({ + cockroachdb = { + fullnameOverride = "dss-cockroachdb" + + conf = { + join = var.crdb_external_nodes + cluster-name = "dss-aws-1" + single-node = false + locality = "zone=${var.crdb_locality}" + } + + statefulset = { + args = [ + "--locality-advertise-addr=zone=${var.crdb_locality}@$(hostname -f)", + "--advertise-addr=$${HOSTNAME##*-}.${var.crdb_hostname_suffix}" + ] + } + + storage = { + persistentVolume = { + storageClass = var.kubernetes_storage_class + } + } + } + + loadBalancers = { + cockroachdbNodes = [ + for ip in var.crdb_internal_nodes[*].ip : + { + ip = ip + subnet = var.workload_subnet + } + ] + + dssGateway = { + ip = var.ip_gateway + subnet = var.workload_subnet + certName = var.gateway_cert_name + } + } + + dss = { + image = local.image + + conf = { + pubKeys = [ + "/test-certs/auth2.pem" + ] + jwksEndpoint = var.authorization.jwks != null ? var.authorization.jwks.endpoint : "" + jwksKeyIds = var.authorization.jwks != null ? [var.authorization.jwks.key_id] : [] + hostname = var.app_hostname + enableScd = var.enable_scd + } + } + + global = { + cloudProvider = var.kubernetes_cloud_provider_name + } + }) +} + diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/output.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/output.tf index eff4a3ce7..129081388 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/output.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/output.tf @@ -2,4 +2,8 @@ output "generated_files_location" { value = <<-EOT Workspace location with generated files: ${local.workspace_location} EOT -} \ No newline at end of file +} + +output "workspace_location" { + value = local.workspace_location +} diff --git a/deploy/operations/Dockerfile b/deploy/operations/Dockerfile index 2ca5ff552..3d7eb0e2f 100644 --- a/deploy/operations/Dockerfile +++ b/deploy/operations/Dockerfile @@ -1,7 +1,9 @@ FROM ubuntu:22.04 +ENV COCKROACH_VERSION 21.2.7 + RUN apt-get update \ -&& apt-get install -y unzip curl gnupg lsb-release +&& apt-get install -y unzip curl gnupg lsb-release apt-transport-https ca-certificates # Terraform CLI RUN curl -s https://apt.releases.hashicorp.com/gpg | gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg \ @@ -16,7 +18,27 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv && rm awscliv2.zip \ && ./aws/install +# Kubectl && Helm +RUN curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | tee /usr/share/keyrings/helm.gpg > /dev/null \ +&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | tee /etc/apt/sources.list.d/helm-stable-debian.list \ +&& curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.28/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg \ +&& echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.28/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list \ +&& apt-get update \ +&& apt-get install -y kubectl helm + +# Cockroach +RUN curl "https://binaries.cockroachdb.com/cockroach-v${COCKROACH_VERSION}.linux-amd64.tgz" -o "cockroach-v${COCKROACH_VERSION}.tgz" \ +&& tar -xvf "cockroach-v${COCKROACH_VERSION}.tgz" \ +&& cp -i cockroach-v${COCKROACH_VERSION}.*/cockroach /usr/local/bin/ \ +&& mkdir -p /usr/local/lib/cockroach \ +&& cp -i cockroach-v${COCKROACH_VERSION}.*/lib/libgeos.so /usr/local/lib/cockroach/ \ +&& cp -i cockroach-v${COCKROACH_VERSION}.*/lib/libgeos_c.so /usr/local/lib/cockroach/ + +# TODO: Migrate scripts to python3 commands +RUN ln -s /usr/bin/python3 /usr/bin/python & \ + ln -s /usr/bin/pip3 /usr/bin/pip + # Clean up apt -RUN apt-get clean && rm -rf /var/lib/apt/lists/* +RUN apt-get clean && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* -RUN terraform --version \ No newline at end of file +RUN terraform --version diff --git a/deploy/operations/ci/aws-1/kubernetes_admin_access.tf b/deploy/operations/ci/aws-1/kubernetes_admin_access.tf index 5e808c2f7..9de035e53 100644 --- a/deploy/operations/ci/aws-1/kubernetes_admin_access.tf +++ b/deploy/operations/ci/aws-1/kubernetes_admin_access.tf @@ -1,38 +1,41 @@ -# This module is expected to be applied by the Github CI user. By default, only the user has permission to -# connect to the cluster. This file gathers resources to grant access to AWS administrators. +// This module is expected to be applied by the Github CI user. By default, only the user who created the cluster +// has permission to connect to the cluster. This file gathers resources to grant access to AWS administrators. -resource "kubernetes_config_map_v1_data" "aws-auth" { - metadata { - name = "aws-auth" - namespace = "kube-system" - } +resource "local_file" "aws-auth-config-map" { + content = yamlencode({ + apiVersion = "v1" + kind = "ConfigMap" + metadata = { + name = "aws-auth" + namespace = "kube-system" + } + data = { + mapRoles = yamlencode([ + { + groups = [ + "system:bootstrappers", + "system:nodes" + ] + rolearn = module.terraform-aws-kubernetes.iam_role_node_group_arn + username = "system:node:{{EC2PrivateDNSName}}" + }, + { + groups = [ + "system:masters" + ] + rolearn = var.aws_iam_administrator_role + username = "interuss-aws-administrator" + }, + { + groups = [ + "system:masters" + ] + rolearn = var.aws_iam_ci_role + username = "interuss-ci" + } + ]) + } + }) - force = true # EKS provisions this file by default. - - data = { - mapRoles = yamlencode([ - { - groups = [ - "system:bootstrappers", - "system:nodes" - ] - rolearn = module.terraform-aws-kubernetes.iam_role_node_group_arn - username = "system:node:{{EC2PrivateDNSName}}" - }, - { - groups = [ - "system:masters" - ] - rolearn = var.aws_iam_administrator_role - username = "interuss-aws-administrator" - }, - { - groups = [ - "system:masters" - ] - rolearn = var.aws_iam_ci_role - username = "interuss-ci" - } - ]) - } + filename = "${module.terraform-commons-dss.workspace_location}/aws_auth_config_map.yml" } diff --git a/deploy/operations/ci/aws-1/main.tf b/deploy/operations/ci/aws-1/main.tf index a9ee3093e..159d949bb 100644 --- a/deploy/operations/ci/aws-1/main.tf +++ b/deploy/operations/ci/aws-1/main.tf @@ -8,14 +8,14 @@ terraform { module "terraform-aws-kubernetes" { # See variables.tf for variables description. - cluster_name = var.cluster_name - aws_region = var.aws_region - app_hostname = var.app_hostname - crdb_hostname_suffix = var.crdb_hostname_suffix - aws_instance_type = var.aws_instance_type - aws_route53_zone_id = var.aws_route53_zone_id + cluster_name = var.cluster_name + aws_region = var.aws_region + app_hostname = var.app_hostname + crdb_hostname_suffix = var.crdb_hostname_suffix + aws_instance_type = var.aws_instance_type + aws_route53_zone_id = var.aws_route53_zone_id aws_iam_permissions_boundary = var.aws_iam_permissions_boundary - node_count = var.node_count + node_count = var.node_count source = "../../../infrastructure/dependencies/terraform-aws-kubernetes" } @@ -42,3 +42,10 @@ module "terraform-commons-dss" { source = "../../../infrastructure/dependencies/terraform-commons-dss" } +terraform { + backend "s3" { + bucket = "interuss-tf-backend-ci" + key = "aws-1" + region = "us-east-1" + } +} diff --git a/deploy/operations/ci/aws-1/output.tf b/deploy/operations/ci/aws-1/output.tf index ade9609d1..295b20f35 100644 --- a/deploy/operations/ci/aws-1/output.tf +++ b/deploy/operations/ci/aws-1/output.tf @@ -2,3 +2,10 @@ output "generated_files_location" { value = module.terraform-commons-dss.generated_files_location } +output "workspace_location" { + value = module.terraform-commons-dss.workspace_location +} + +output "kubernetes_context" { + value = module.terraform-aws-kubernetes.kubernetes_context_name +} diff --git a/deploy/operations/ci/aws-1/providers.tf b/deploy/operations/ci/aws-1/providers.tf index 8daa12fe7..629198205 100644 --- a/deploy/operations/ci/aws-1/providers.tf +++ b/deploy/operations/ci/aws-1/providers.tf @@ -3,12 +3,12 @@ provider "aws" { } data "aws_eks_cluster_auth" "kubernetes_cluster" { - name = var.cluster_name + name = var.cluster_name depends_on = [module.terraform-aws-kubernetes] } data "aws_eks_cluster" "kubernetes_cluster" { - name = var.cluster_name + name = var.cluster_name depends_on = [module.terraform-aws-kubernetes] } @@ -17,3 +17,11 @@ provider kubernetes { cluster_ca_certificate = base64decode(data.aws_eks_cluster.kubernetes_cluster.certificate_authority[0].data) token = data.aws_eks_cluster_auth.kubernetes_cluster.token } + +provider "helm" { + kubernetes { + host = data.aws_eks_cluster.kubernetes_cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.kubernetes_cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.kubernetes_cluster.token + } +} diff --git a/deploy/operations/ci/aws-1/test.sh b/deploy/operations/ci/aws-1/test.sh index 7caeafa93..dc8f8be7e 100755 --- a/deploy/operations/ci/aws-1/test.sh +++ b/deploy/operations/ci/aws-1/test.sh @@ -12,15 +12,46 @@ else fi cd "${BASEDIR}" || exit 1 -clean () { - echo "Cleaning infrastructure" - terraform destroy -auto-approve -} - terraform init -clean +# TODO: Fail if env is not clean + +# Deploy the Kubernetes cluster terraform apply -auto-approve -# TODO: Deploy the DSS +KUBE_CONTEXT="$(terraform output -raw kubernetes_context)" +WORKSPACE_LOCATION="$(terraform output -raw workspace_location)" + +cd "${WORKSPACE_LOCATION}" +./get-credentials.sh +aws sts get-caller-identity + +# Allow access to the cluster to AWS admins +kubectl apply -f "aws_auth_config_map.yml" + +## Generate cockroachdb certificates +./make-certs.sh +./apply-certs.sh + +cd "$BASEDIR/../../../services/helm-charts/dss" +RELEASE_NAME="dss" +helm dep update --kube-context="$KUBE_CONTEXT" +helm upgrade --install --kube-context="$KUBE_CONTEXT" -f "${WORKSPACE_LOCATION}/helm_values.yml" "$RELEASE_NAME" . + # TODO: Test the deployment of the DSS -clean + +if [ -n "$DO_NOT_DESTROY" ]; then + "No destroy required. Stop." + exit 0 +fi + +# Cleanup +# Delete workloads +helm uninstall --kube-context="$KUBE_CONTEXT" "$RELEASE_NAME" + +# Delete PVC to delete persistant volumes +kubectl delete pvc --all=true + +# Delete cluster +cd "$BASEDIR" +terraform destroy -auto-approve + diff --git a/deploy/operations/docker-compose.yaml b/deploy/operations/docker-compose.yaml index 4618089db..23266d73c 100644 --- a/deploy/operations/docker-compose.yaml +++ b/deploy/operations/docker-compose.yaml @@ -4,12 +4,12 @@ services: image: interuss-deploy profiles: ["aws-1"] command: operations/ci/aws-1/test.sh - working_dir: /opt/dss + working_dir: /opt/dss/deploy environment: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY - AWS_SESSION_TOKEN volumes: - type: bind - source: ../ + source: ../../ target: /opt/dss/ diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index 77cccafa3..da3a2dc33 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -42,7 +42,7 @@ spec: - --cockroach_ssl_mode=verify-full - --cockroach_user=root - --dump_requests=true - - --enable_scd={{$dss.enableScd | default false}} + - --enable_scd={{$dss.enableScd | default true}} - --garbage_collector_spec=@every 30m - --gcp_prof_service_name= {{- if $dss.conf.jwksEndpoint }}