Add helm to CI

Orbitalize · Dec 22, 2023 · 8cffe67 · 8cffe67
1 parent 8e407f0
commit 8cffe67
Show file tree

Hide file tree

Showing 10 changed files with 206 additions and 60 deletions.
diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/main.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/main.tf
@@ -65,4 +65,68 @@ resource "local_file" "get_credentials" {
     get_credentials_cmd = var.kubernetes_get_credentials_cmd
   })
   filename = "${local.workspace_location}/get-credentials.sh"
-}
+}
+
+resource "local_file" "helm_chart_values" {
+  filename = "${local.workspace_location}/helm_values.yml"
+  content = yamlencode({
+    cockroachdb = {
+      fullnameOverride = "dss-cockroachdb"
+
+      conf = {
+        join         = var.crdb_external_nodes
+        cluster-name = "dss-aws-1"
+        single-node  = false
+        locality     = "zone=${var.crdb_locality}"
+      }
+
+      statefulset = {
+        args = [
+          "--locality-advertise-addr=zone=${var.crdb_locality}@$(hostname -f)",
+          "--advertise-addr=$${HOSTNAME##*-}.${var.crdb_hostname_suffix}"
+        ]
+      }
+
+      storage = {
+        persistentVolume = {
+          storageClass = var.kubernetes_storage_class
+        }
+      }
+    }
+
+    loadBalancers = {
+      cockroachdbNodes = [
+      for ip in var.crdb_internal_nodes[*].ip :
+      {
+        ip     = ip
+        subnet = var.workload_subnet
+      }
+      ]
+
+      dssGateway = {
+        ip       = var.ip_gateway
+        subnet   = var.workload_subnet
+        certName = var.gateway_cert_name
+      }
+    }
+
+    dss = {
+      image = local.image
+
+      conf = {
+        pubKeys      = [
+          "/test-certs/auth2.pem"
+        ]
+        jwksEndpoint = var.authorization.jwks != null ? var.authorization.jwks.endpoint : ""
+        jwksKeyIds   = var.authorization.jwks != null ? [var.authorization.jwks.key_id] : []
+        hostname     = var.app_hostname
+        enableScd    = var.enable_scd
+      }
+    }
+
+    global = {
+      cloudProvider = var.kubernetes_cloud_provider_name
+    }
+  })
+}
+
diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/output.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/output.tf
@@ -2,4 +2,8 @@ output "generated_files_location" {
   value = <<-EOT
   Workspace location with generated files: ${local.workspace_location}
   EOT
-}
+}
+
+output "workspace_location" {
+  value = local.workspace_location
+}
diff --git a/deploy/operations/Dockerfile b/deploy/operations/Dockerfile
@@ -1,7 +1,9 @@
 FROM ubuntu:22.04
 
+ENV COCKROACH_VERSION 21.2.7
+
 RUN  apt-get update \
-&& apt-get install -y unzip curl gnupg lsb-release
+&& apt-get install -y unzip curl gnupg lsb-release apt-transport-https ca-certificates
 
 # Terraform CLI
 RUN curl -s https://apt.releases.hashicorp.com/gpg | gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg \
@@ -16,7 +18,27 @@ RUN  curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv
 && rm awscliv2.zip \
 && ./aws/install
 
+# Kubectl && Helm
+RUN curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | tee /usr/share/keyrings/helm.gpg > /dev/null \
+&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | tee /etc/apt/sources.list.d/helm-stable-debian.list \
+&& curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.28/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg \
+&& echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.28/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list \
+&& apt-get update \
+&& apt-get install -y kubectl helm
+
+# Cockroach
+RUN curl "https://binaries.cockroachdb.com/cockroach-v${COCKROACH_VERSION}.linux-amd64.tgz" -o "cockroach-v${COCKROACH_VERSION}.tgz" \
+&& tar -xvf "cockroach-v${COCKROACH_VERSION}.tgz" \
+&& cp -i cockroach-v${COCKROACH_VERSION}.*/cockroach /usr/local/bin/ \
+&& mkdir -p /usr/local/lib/cockroach \
+&& cp -i cockroach-v${COCKROACH_VERSION}.*/lib/libgeos.so /usr/local/lib/cockroach/ \
+&& cp -i cockroach-v${COCKROACH_VERSION}.*/lib/libgeos_c.so /usr/local/lib/cockroach/
+
+# TODO: Migrate scripts to python3 commands
+RUN ln -s /usr/bin/python3 /usr/bin/python & \
+    ln -s /usr/bin/pip3 /usr/bin/pip
+
 # Clean up apt
-RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN apt-get clean && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
-RUN terraform --version
+RUN terraform --version
diff --git a/deploy/operations/ci/aws-1/kubernetes_admin_access.tf b/deploy/operations/ci/aws-1/kubernetes_admin_access.tf
@@ -1,38 +1,41 @@
-# This module is expected to be applied by the Github CI user. By default, only the user has permission to
-# connect to the cluster. This file gathers resources to grant access to AWS administrators.
+// This module is expected to be applied by the Github CI user. By default, only the user who created the cluster
+// has permission to connect to the cluster. This file gathers resources to grant access to AWS administrators.
 
-resource "kubernetes_config_map_v1_data" "aws-auth" {
-  metadata {
-    name      = "aws-auth"
-    namespace = "kube-system"
-  }
+resource "local_file" "aws-auth-config-map" {
+  content = yamlencode({
+    apiVersion = "v1"
+    kind = "ConfigMap"
+    metadata = {
+      name      = "aws-auth"
+      namespace = "kube-system"
+    }
+    data = {
+      mapRoles = yamlencode([
+        {
+          groups   = [
+            "system:bootstrappers",
+            "system:nodes"
+          ]
+          rolearn  = module.terraform-aws-kubernetes.iam_role_node_group_arn
+          username = "system:node:{{EC2PrivateDNSName}}"
+        },
+        {
+          groups   = [
+            "system:masters"
+          ]
+          rolearn  = var.aws_iam_administrator_role
+          username = "interuss-aws-administrator"
+        },
+        {
+          groups   = [
+            "system:masters"
+          ]
+          rolearn  = var.aws_iam_ci_role
+          username = "interuss-ci"
+        }
+      ])
+    }
+  })
 
-  force = true # EKS provisions this file by default.
-
-  data = {
-    mapRoles = yamlencode([
-      {
-        groups   = [
-          "system:bootstrappers",
-          "system:nodes"
-        ]
-        rolearn  = module.terraform-aws-kubernetes.iam_role_node_group_arn
-        username = "system:node:{{EC2PrivateDNSName}}"
-      },
-      {
-        groups   = [
-          "system:masters"
-        ]
-        rolearn  = var.aws_iam_administrator_role
-        username = "interuss-aws-administrator"
-      },
-      {
-        groups   = [
-          "system:masters"
-        ]
-        rolearn  = var.aws_iam_ci_role
-        username = "interuss-ci"
-      }
-    ])
-  }
+  filename   = "${module.terraform-commons-dss.workspace_location}/aws_auth_config_map.yml"
 }
diff --git a/deploy/operations/ci/aws-1/main.tf b/deploy/operations/ci/aws-1/main.tf
@@ -8,14 +8,14 @@ terraform {
 
 module "terraform-aws-kubernetes" {
   # See variables.tf for variables description.
-  cluster_name         = var.cluster_name
-  aws_region           = var.aws_region
-  app_hostname         = var.app_hostname
-  crdb_hostname_suffix = var.crdb_hostname_suffix
-  aws_instance_type    = var.aws_instance_type
-  aws_route53_zone_id  = var.aws_route53_zone_id
+  cluster_name                 = var.cluster_name
+  aws_region                   = var.aws_region
+  app_hostname                 = var.app_hostname
+  crdb_hostname_suffix         = var.crdb_hostname_suffix
+  aws_instance_type            = var.aws_instance_type
+  aws_route53_zone_id          = var.aws_route53_zone_id
   aws_iam_permissions_boundary = var.aws_iam_permissions_boundary
-  node_count           = var.node_count
+  node_count                   = var.node_count
 
   source = "../../../infrastructure/dependencies/terraform-aws-kubernetes"
 }
@@ -42,3 +42,10 @@ module "terraform-commons-dss" {
 
   source = "../../../infrastructure/dependencies/terraform-commons-dss"
 }
+terraform {
+  backend "s3" {
+    bucket = "interuss-tf-backend-ci"
+    key    = "aws-1"
+    region = "us-east-1"
+  }
+}
diff --git a/deploy/operations/ci/aws-1/output.tf b/deploy/operations/ci/aws-1/output.tf
@@ -2,3 +2,10 @@ output "generated_files_location" {
   value = module.terraform-commons-dss.generated_files_location
 }
 
+output "workspace_location" {
+  value = module.terraform-commons-dss.workspace_location
+}
+
+output "kubernetes_context" {
+  value = module.terraform-aws-kubernetes.kubernetes_context_name
+}
diff --git a/deploy/operations/ci/aws-1/providers.tf b/deploy/operations/ci/aws-1/providers.tf
@@ -3,12 +3,12 @@ provider "aws" {
 }
 
 data "aws_eks_cluster_auth" "kubernetes_cluster" {
-  name = var.cluster_name
+  name       = var.cluster_name
   depends_on = [module.terraform-aws-kubernetes]
 }
 
 data "aws_eks_cluster" "kubernetes_cluster" {
-  name = var.cluster_name
+  name       = var.cluster_name
   depends_on = [module.terraform-aws-kubernetes]
 }
 
@@ -17,3 +17,11 @@ provider kubernetes {
   cluster_ca_certificate = base64decode(data.aws_eks_cluster.kubernetes_cluster.certificate_authority[0].data)
   token                  = data.aws_eks_cluster_auth.kubernetes_cluster.token
 }
+
+provider "helm" {
+  kubernetes {
+    host                   = data.aws_eks_cluster.kubernetes_cluster.endpoint
+    cluster_ca_certificate = base64decode(data.aws_eks_cluster.kubernetes_cluster.certificate_authority[0].data)
+    token                  = data.aws_eks_cluster_auth.kubernetes_cluster.token
+  }
+}
diff --git a/deploy/operations/ci/aws-1/test.sh b/deploy/operations/ci/aws-1/test.sh
@@ -12,15 +12,46 @@ else
 fi
 cd "${BASEDIR}" || exit 1
 
-clean () {
-  echo "Cleaning infrastructure"
-  terraform destroy -auto-approve
-}
-
 terraform init
-clean
+# TODO: Fail if env is not clean
+
+# Deploy the Kubernetes cluster
 terraform apply -auto-approve
-# TODO: Deploy the DSS
+KUBE_CONTEXT="$(terraform output -raw kubernetes_context)"
+WORKSPACE_LOCATION="$(terraform output -raw workspace_location)"
+
+cd "${WORKSPACE_LOCATION}"
+./get-credentials.sh
+aws sts get-caller-identity
+
+# Allow access to the cluster to AWS admins
+kubectl apply -f "aws_auth_config_map.yml"
+
+## Generate cockroachdb certificates
+./make-certs.sh
+./apply-certs.sh
+
+cd "$BASEDIR/../../../services/helm-charts/dss"
+RELEASE_NAME="dss"
+helm dep update --kube-context="$KUBE_CONTEXT"
+helm upgrade --install --kube-context="$KUBE_CONTEXT" -f "${WORKSPACE_LOCATION}/helm_values.yml" "$RELEASE_NAME" .
+
 # TODO: Test the deployment of the DSS
-clean
+
+if [ -n "$DO_NOT_DESTROY" ]; then
+  "No destroy required. Stop."
+  exit 0
+fi
+
+# Cleanup
+# Delete workloads
+helm uninstall --kube-context="$KUBE_CONTEXT" "$RELEASE_NAME"
+
+# Delete PVC to delete persistant volumes
+kubectl delete pvc --all=true
+
+# Delete cluster
+cd "$BASEDIR"
+terraform destroy -auto-approve
+
 
diff --git a/deploy/operations/docker-compose.yaml b/deploy/operations/docker-compose.yaml
@@ -4,12 +4,12 @@ services:
     image: interuss-deploy
     profiles: ["aws-1"]
     command: operations/ci/aws-1/test.sh
-    working_dir: /opt/dss
+    working_dir: /opt/dss/deploy
     environment:
       - AWS_ACCESS_KEY_ID
       - AWS_SECRET_ACCESS_KEY
       - AWS_SESSION_TOKEN
     volumes:
       - type: bind
-        source: ../
+        source: ../../
         target: /opt/dss/
diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml
@@ -42,7 +42,7 @@ spec:
             - --cockroach_ssl_mode=verify-full
             - --cockroach_user=root
             - --dump_requests=true
-            - --enable_scd={{$dss.enableScd | default false}}
+            - --enable_scd={{$dss.enableScd | default true}}
             - --garbage_collector_spec=@every 30m
             - --gcp_prof_service_name=
             {{- if $dss.conf.jwksEndpoint }}