Add certificate authority selection support and Google CAS integration (

#122) Closes #107. This PR introduces the concept of Certificate Authority (CA) providers to deployment scripts. It allows selecting a certificate provider to configure Terraform files, helm charts and cluster resources necessary to generate certificates using the selected provider. The existing integration with Let's Encrypt has been kept as a certificate provider option. Integration with Google's Certificate Authority Service (CAS) has been added and can be selected as a certificate provider. ## Changes 1. Certificate providers. There are two options available: `lets-encrypt`, and `google-cas`. The default option was set to `lets-encrypt` for backwards compatibility purposes. 2. Some configuration variables have been added. Existing configurations can be updated by running `deploy.sh set-config`. ## How to test 1. Run `./deploy.sh set-config <PROJECT_ID>` to configure new environment variables. * Select `google-cas` as the certificate provider. * Fill in the Common Name (`CN`) and Organization (`O`) certificate fields. Optionally, fill in the Organizational Unit (`OU`) certificate field. 2. Run `./deploy.sh update <PROJECT_ID>` to deploy the configuration. 3. Verify that the `cloud-robotics` certificate was issued correctly. The output should look something like this ```bash # Make sure kubectl is using the correct config and context $ kubectl get certificate cloud-robotics NAME READY SECRET AGE cloud-robotics True cloud-robotics-tls XXm ``` 4. Verify that `www.endpoints.$PROJECT_ID.cloud.goog` is using the certificate. ```sh # Make sure to set <PROJECT_ID> openssl s_client -showcerts -connect www.endpoints.$PROJECT_ID.cloud.goog:443 ``` --------- Signed-off-by: Alejo Carballude <[email protected]>
googlecloudrobotics · Mar 28, 2023 · 1c2898b · 1c2898b
1 parent 01b1e29
commit 1c2898b
Show file tree

Hide file tree

Showing 15 changed files with 365 additions and 48 deletions.
diff --git a/config.sh.tmpl b/config.sh.tmpl
@@ -14,7 +14,7 @@ GCP_ZONE=europe-west1-c
 ### Optional settings ###
 
 # The Docker registry all Cloud Robotics images are deployed to when installing
-# from srouces. It is ignored during binary installs.
+# from sources. It is ignored during binary installs.
 # If unset, defaults to "gcr.io/${GCP_PROJECT_ID}"
 #CLOUD_ROBOTICS_CONTAINER_REGISTRY=gcr.io/my-project
 

diff --git a/deploy.sh b/deploy.sh
@@ -50,6 +50,11 @@ function include_config_and_defaults {
   CLOUD_ROBOTICS_DOMAIN=${CLOUD_ROBOTICS_DOMAIN:-"www.endpoints.${GCP_PROJECT_ID}.cloud.goog"}
   APP_MANAGEMENT=${APP_MANAGEMENT:-false}
 
+  # lets-encrypt is used as the default certificate provider for backwards compatibility purposes
+  CLOUD_ROBOTICS_CERTIFICATE_PROVIDER=${CLOUD_ROBOTICS_CERTIFICATE_PROVIDER:-lets-encrypt}
+  CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_COMMON_NAME=${CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_COMMON_NAME:-GCP_PROJECT_ID}
+  CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_ORGANIZATION=${CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_ORGANIZATION:-GCP_PROJECT_ID}
+
   CLOUD_ROBOTICS_OWNER_EMAIL=${CLOUD_ROBOTICS_OWNER_EMAIL:-$(gcloud config get-value account)}
   KUBE_CONTEXT="gke_${GCP_PROJECT_ID}_${GCP_ZONE}_${PROJECT_NAME}"
 
@@ -127,7 +132,7 @@ function terraform_init {
   # This variable is set by src/bootstrap/cloud/run-install.sh for binary installs
   local CRC_VERSION
   if [[ -z "${TARGET}" ]]; then
-    # TODO(ensonic): keep this in sync with the nighly release script
+    # TODO(ensonic): keep this in sync with the nightly release script
     VERSION=${VERSION:-"0.1.0"}
     if [[ -d .git ]]; then
       SHA=$(git rev-parse --short HEAD)
@@ -151,14 +156,33 @@ shared_owner_group = "${CLOUD_ROBOTICS_SHARED_OWNER_GROUP}"
 robot_image_reference = "${SOURCE_CONTAINER_REGISTRY}/setup-robot@${ROBOT_IMAGE_DIGEST}"
 crc_version = "${CRC_VERSION}"
 cr_syncer_rbac = "${CR_SYNCER_RBAC}"
+certificate_provider = "${CLOUD_ROBOTICS_CERTIFICATE_PROVIDER}"
+EOF
+
+# Add certificate information if the configured provider requires it
+  if [[ ! "none self-signed" =~ (" "|^)"${CLOUD_ROBOTICS_CERTIFICATE_PROVIDER}"(" "|$) ]]; then
+    cat >> "${TERRAFORM_DIR}/terraform.tfvars" <<EOF
+certificate_subject_common_name = "${CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_COMMON_NAME}"
+certificate_subject_organization = "${CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_ORGANIZATION}"
 EOF
 
+  if [[ -n "${CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_ORGANIZATIONAL_UNIT}" ]]; then
+      cat >> "${TERRAFORM_DIR}/terraform.tfvars" <<EOF
+certificate_subject_organizational_unit = "${CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_ORGANIZATIONAL_UNIT}"
+EOF
+    fi
+  fi
+
+# Docker private projects
+
   if [[ -n "${PRIVATE_DOCKER_PROJECTS:-}" ]]; then
     cat >> "${TERRAFORM_DIR}/terraform.tfvars" <<EOF
 private_image_repositories = ["${PRIVATE_DOCKER_PROJECTS// /\", \"}"]
 EOF
   fi
 
+# Terraform bucket
+
   if [[ -n "${TERRAFORM_GCS_BUCKET:-}" ]]; then
     cat > "${TERRAFORM_DIR}/backend.tf" <<EOF
 # autogenerated by deploy.sh, do not edit!
@@ -184,6 +208,13 @@ function terraform_apply {
   # We've stopped managing Google Cloud projects in Terraform, make sure they
   # aren't deleted.
   terraform_exec state rm google_project.project 2>/dev/null || true
+#  echo "Importing CA resources"
+#  echo -n "CA: "
+  terraform_exec state rm google_privateca_certificate_authority.ca
+#  terraform_exec import google_privateca_certificate_authority.ca robco-ensonic/europe-west1/robco-ensonic-ca-pool/robco-ensonic-ca
+#  echo -n "CA Pool: "
+#  terraform_exec state rm google_privateca_ca_pool.ca_pool
+#  terraform_exec import google_privateca_ca_pool.ca_pool robco-ensonic/europe-west1/robco-ensonic-ca-pool
 
   terraform_exec apply ${TERRAFORM_APPLY_FLAGS} \
     || die "terraform apply failed"
@@ -319,7 +350,7 @@ function helm_charts {
     kc delete secrets cluster-authority 2> /dev/null || true
   fi
 
-  # Delete permissive binding if it exists because from previous deployments
+  # Delete permissive binding if it exists from previous deployments
   if kc get clusterrolebinding permissive-binding &>/dev/null; then
     kc delete clusterrolebinding permissive-binding
   fi
@@ -335,6 +366,7 @@ function helm_charts {
     --set-string registry=${SOURCE_CONTAINER_REGISTRY}
     --set-string owner_email=${CLOUD_ROBOTICS_OWNER_EMAIL}
     --set-string app_management=${APP_MANAGEMENT}
+    --set-string certificate_provider=${CLOUD_ROBOTICS_CERTIFICATE_PROVIDER}
     --set-string deploy_environment=${CLOUD_ROBOTICS_DEPLOY_ENVIRONMENT}
     --set-string oauth2_proxy.client_id=${CLOUD_ROBOTICS_OAUTH2_CLIENT_ID}
     --set-string oauth2_proxy.client_secret=${CLOUD_ROBOTICS_OAUTH2_CLIENT_SECRET}
@@ -386,7 +418,7 @@ function update {
   create $1
 }
 
-# This is a shortcut for skipping Terrafrom configs checks if you know the config has not changed.
+# This is a shortcut for skipping Terraform config checks if you know the config has not changed.
 function fast_push {
   include_config_and_defaults $1
   if is_source_install; then
@@ -395,9 +427,15 @@ function fast_push {
   helm_charts
 }
 
+# This is a shortcut for skipping building and applying Terraform configs if you know the build has not changed.
+function update_infra {
+  include_config_and_defaults $1
+  terraform_apply
+}
+
 # main
-if [[ "$#" -lt 2 ]] || [[ ! "$1" =~ ^(set_config|create|delete|update|fast_push)$ ]]; then
-  die "Usage: $0 {set_config|create|delete|update|fast_push} <project id>"
+if [[ "$#" -lt 2 ]] || [[ ! "$1" =~ ^(set_config|create|delete|update|fast_push|update_infra)$ ]]; then
+  die "Usage: $0 {set_config|create|delete|update|fast_push|update_infra} <project id>"
 fi
 
 # call arguments verbatim:

diff --git a/scripts/set-config.sh b/scripts/set-config.sh
@@ -224,6 +224,9 @@ function set_default_vars {
   if [[ "${PRIVATE_DOCKER_PROJECTS}" == "none" ]]; then
     PRIVATE_DOCKER_PROJECTS=
   fi
+
+  # Certificate provider
+  set_certificate_provider_vars
 }
 
 function set_oauth_vars {
@@ -241,6 +244,38 @@ function set_oauth_vars {
   fi
 }
 
+function set_certificate_provider_vars {
+  CA_OPTIONS="lets-encrypt, google-cas"
+  CA_DEFAULT="lets-encrypt"
+
+  # Select provider
+  read_variable CLOUD_ROBOTICS_CERTIFICATE_PROVIDER \
+    "Select the certificate provider. Should be one of: ${CA_OPTIONS}." \
+    "${CLOUD_ROBOTICS_CERTIFICATE_PROVIDER:-${CA_DEFAULT}}"
+
+  # Request certificate configuration if the provider requires it
+  if [[ ! "lets-encrypt" =~ (" "|^)"${CLOUD_ROBOTICS_CERTIFICATE_PROVIDER}"(" "|$) ]]; then
+    set_certificate_vars
+  fi
+}
+
+function set_certificate_vars {
+  echo "Configuring certificate information."
+  echo "Refer to RFC 4519 for explanations of the fields: https://datatracker.ietf.org/doc/html/rfc4519#section-2"
+
+  read_variable CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_ORGANIZATION \
+    "Organization (O)" \
+    "${CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_ORGANIZATION:-${GCP_PROJECT_ID}}"
+
+  read_variable CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_COMMON_NAME \
+    "Common Name (CN)" \
+    "${CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_COMMON_NAME:-${GCP_PROJECT_ID}}"
+
+  read_variable CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_ORGANIZATIONAL_UNIT \
+    "(Optional) Organizational Unit (OU)" \
+    "${CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_ORGANIZATIONAL_UNIT}"
+}
+
 if [[ -n "${FLAG_EDIT_OAUTH}" ]]; then
   set_oauth_vars
 else
@@ -261,6 +296,10 @@ print_variable "Projects for private Docker images" "${PRIVATE_DOCKER_PROJECTS}"
 print_variable "OAuth client id" "${CLOUD_ROBOTICS_OAUTH2_CLIENT_ID}"
 print_variable "OAuth client secret" "${CLOUD_ROBOTICS_OAUTH2_CLIENT_SECRET}"
 print_variable "OAuth cookie secret" "${CLOUD_ROBOTICS_COOKIE_SECRET}"
+print_variable "Certificate provider" "${CLOUD_ROBOTICS_CERTIFICATE_PROVIDER}"
+print_variable "Certificate Subject Organization (O)" "${CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_ORGANIZATION}"
+print_variable "Certificate Subject Common Name (CN)" "${CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_COMMON_NAME}"
+print_variable "Certificate Subject Organizational Unit (OU)" "${CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_ORGANIZATIONAL_UNIT}"
 
 if ! ask_yn "Would you like to save this configuration?"; then
   exit 0
@@ -289,6 +328,10 @@ save_variable "${CONFIG_FILE}" PRIVATE_DOCKER_PROJECTS "${PRIVATE_DOCKER_PROJECT
 save_variable "${CONFIG_FILE}" CLOUD_ROBOTICS_OAUTH2_CLIENT_ID "${CLOUD_ROBOTICS_OAUTH2_CLIENT_ID}"
 save_variable "${CONFIG_FILE}" CLOUD_ROBOTICS_OAUTH2_CLIENT_SECRET "${CLOUD_ROBOTICS_OAUTH2_CLIENT_SECRET}"
 save_variable "${CONFIG_FILE}" CLOUD_ROBOTICS_COOKIE_SECRET "${CLOUD_ROBOTICS_COOKIE_SECRET}"
+save_variable "${CONFIG_FILE}" CLOUD_ROBOTICS_CERTIFICATE_PROVIDER "${CLOUD_ROBOTICS_CERTIFICATE_PROVIDER}"
+save_variable "${CONFIG_FILE}" CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_ORGANIZATION "${CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_ORGANIZATION}"
+save_variable "${CONFIG_FILE}" CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_COMMON_NAME "${CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_COMMON_NAME}"
+save_variable "${CONFIG_FILE}" CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_ORGANIZATIONAL_UNIT "${CLOUD_ROBOTICS_CERTIFICATE_SUBJECT_ORGANIZATIONAL_UNIT}"
 
 # Upload config to the cloud.
 if ! gsutil ls -p ${GCP_PROJECT_ID} | grep "^${CLOUD_BUCKET}/$" >/dev/null; then

diff --git a/src/app_charts/base/BUILD.bazel b/src/app_charts/base/BUILD.bazel
@@ -1,6 +1,8 @@
 load("//bazel:build_rules/helm_template.bzl", "helm_template")
 load("//bazel:app_chart.bzl", "app_chart")
 
+# Tests
+
 app_chart(
     name = "base-test",
     extra_templates = [
@@ -21,6 +23,18 @@ app_chart(
     visibility = ["//visibility:public"],
 )
 
+sh_test(
+    name = "app_management_test",
+    srcs = ["app_management_test.sh"],
+    data = [
+        ":base-cloud",
+        ":base-robot",
+        "@kubernetes_helm//:helm",
+    ],
+)
+
+# Robot
+
 helm_template(
     name = "cert-manager-chart.robot",
     chart = "//third_party/cert-manager:cert-manager-v1.7.2.tgz",
@@ -52,6 +66,8 @@ app_chart(
     visibility = ["//visibility:public"],
 )
 
+# Cloud
+
 helm_template(
     name = "cert-manager-chart.cloud",
     chart = "//third_party/cert-manager:cert-manager-v1.7.2.tgz",
@@ -61,6 +77,15 @@ helm_template(
     values = "cert-manager-cloud.values.yaml",
 )
 
+helm_template(
+    name = "cert-manager-google-cas-issuer-chart.cloud",
+    chart = "//third_party/cert-manager-google-cas-issuer:cert-manager-google-cas-issuer-v0.6.2.tgz",
+    # The namespace will later be replaced with the actual one.
+    namespace = "HELM-NAMESPACE",
+    release_name = "cert-manager-google-cas-issuer",
+    values = "cert-manager-google-cas-issuer-cloud.values.yaml",
+)
+
 app_chart(
     name = "base-cloud",
     extra_templates = [
@@ -70,6 +95,7 @@ app_chart(
     files = [
         "relay-dashboard.json",
         ":cert-manager-chart.cloud",
+        ":cert-manager-google-cas-issuer-chart.cloud",
         "@ingress-nginx//:ingress-nginx-dashboards",
     ],
     images = {
@@ -79,13 +105,3 @@ app_chart(
     values = "values-cloud.yaml",
     visibility = ["//visibility:public"],
 )
-
-sh_test(
-    name = "app_management_test",
-    srcs = ["app_management_test.sh"],
-    data = [
-        ":base-cloud",
-        ":base-robot",
-        "@kubernetes_helm//:helm",
-    ],
-)
diff --git a/src/app_charts/base/cert-manager-google-cas-issuer-cloud.values.yaml b/src/app_charts/base/cert-manager-google-cas-issuer-cloud.values.yaml
@@ -0,0 +1,20 @@
+# Configuration for the cert-manager chart.
+# Reference: https://github.com/jetstack/google-cas-issuer/blob/main/deploy/charts/google-cas-issuer/values.yaml
+
+# No values are required for now.
+# This was put in place to add values in the future without requiring additional configuration in other files.
+# If values are added this disclaimer should be removed.
+
+# The Kubernetes service account must be annotated in order to impersonate a GCP service account using workload identity.
+serviceAccount:
+  annotations:
+    # PROJECT-ID will be replaced by a script in a future step with the contents of the `PROJECT_ID` env var.
+    iam.gke.io/gcp-service-account: [email protected]
+
+app:
+  approval:
+    subjects:
+    - kind: ServiceAccount
+      name: cert-manager
+      # TODO(alejoasd): this should be set from configuration dynamically
+      namespace: default
diff --git a/.../base/cloud/cert-manager-certificate.yaml → ...base/cloud/cert-manager-certificates.yaml b/.../base/cloud/cert-manager-certificate.yaml → ...base/cloud/cert-manager-certificates.yaml
@@ -1,23 +1,5 @@
 apiVersion: cert-manager.io/v1
 kind: Certificate
-metadata:
-  name: cloud-robotics
-spec:
-  secretName: tls
-  issuerRef:
-    name: letsencrypt-prod
-  commonName: {{ .Values.domain }}
-  dnsNames:
-    - {{ .Values.domain }}
-  acme:
-    config:
-    - http01:
-        ingressClass: nginx
-      domains:
-      - {{ .Values.domain }}
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
 metadata:
   name: selfsigned-ca
 spec:
@@ -32,3 +14,30 @@ spec:
     name: selfsigned-issuer
     kind: ClusterIssuer
     group: cert-manager.io
+{{ if .Values.certificate_provider }}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: cloud-robotics
+spec:
+  commonName: {{ .Values.domain }}
+  secretName: tls
+  dnsNames:
+    - {{ .Values.domain }}
+{{ if eq .Values.certificate_provider "lets-encrypt" }}
+  issuerRef:
+    name: letsencrypt-prod
+  acme:
+    config:
+    - http01:
+        ingressClass: nginx
+      domains:
+      - {{ .Values.domain }}
+{{ else if eq .Values.certificate_provider "google-cas" }}
+  issuerRef:
+    name: google-cas
+    group: cas-issuer.jetstack.io
+    kind: GoogleCASClusterIssuer
+{{ end }}
+{{ end }}
diff --git a/src/app_charts/base/cloud/cert-manager-google-cas-issuer.yaml b/src/app_charts/base/cloud/cert-manager-google-cas-issuer.yaml
@@ -0,0 +1,6 @@
+{{ if eq .Values.certificate_provider "google-cas" }}
+# This includes all resources expanded from the cert-manager chart using
+# the values in ../cert-manager-cloud.values.yaml.
+# Some pseudo-variables that were inserted there are replaced with actual runtime values.
+{{ .Files.Get "files/cert-manager-google-cas-issuer-chart.cloud.yaml" | replace "HELM-NAMESPACE" .Release.Namespace | replace "PROJECT-ID" .Values.project }}
+{{ end }}