From dbda2df5248f5716b31d666aa34749ba87dab099 Mon Sep 17 00:00:00 2001
From: Veronika Fisarova <vfisarov@redhat.com>
Date: Wed, 16 Oct 2024 10:05:00 +0200
Subject: [PATCH] [kuttl] Add tls scenarios for non API services

Signed-off-by: Veronika Fisarova <vfisarov@redhat.com>
---
 .../common/osp_check_noapi_service_certs.sh   | 99 +++++++++++++++++++
 .../04-assert-service-cert-rotation.yaml      |  4 +
 .../02-assert-service-certs-issuers.yaml      |  4 +
 ...-assert-service-certs-default-issuers.yaml |  4 +
 ...-assert-service-certs-default-issuers.yaml |  4 +
 .../10-assert-service-certs-issuers.yaml      |  4 +
 6 files changed, 119 insertions(+)
 create mode 100755 tests/kuttl/common/osp_check_noapi_service_certs.sh

diff --git a/tests/kuttl/common/osp_check_noapi_service_certs.sh b/tests/kuttl/common/osp_check_noapi_service_certs.sh
new file mode 100755
index 000000000..245415bd5
--- /dev/null
+++ b/tests/kuttl/common/osp_check_noapi_service_certs.sh
@@ -0,0 +1,99 @@
+#!/bin/bash
+
+NAMESPACE=${NAMESPACE}
+
+declare -A services_secrets=(
+    ["ceilometer-internal"]="cert-ceilometer-internal-svc"
+    ["ovsdbserver-nb-0"]="cert-ovndbcluster-nb-ovndbs"
+    ["ovsdbserver-sb-0"]="cert-ovndbcluster-sb-ovndbs"
+    ["rabbitmq"]="cert-rabbitmq-svc"
+    ["rabbitmq-cell1"]="cert-rabbitmq-cell1-svc"
+)
+
+declare -A database_secrets=(
+    ["openstack"]="cert-galera-openstack-svc"
+    ["openstack-cell1"]="cert-galera-openstack-cell1-svc"
+)
+
+mismatched_services=()
+
+# Gather the ClusterIP and ports for general services
+for service in "${!services_secrets[@]}"; do
+    secret="${services_secrets[$service]}"
+
+    service_info=$(oc get service "$service" -n "$NAMESPACE" -o jsonpath="{.spec.clusterIP} {.spec.ports[*].port}")
+    cluster_ip=$(echo "$service_info" | awk '{print $1}')
+    ports=$(echo "$service_info" | cut -d' ' -f2-)
+
+    echo "Checking service: $service (ClusterIP: $cluster_ip, Ports: $ports)"
+
+    # Fetch the certificate from the secret and decode it
+    secret_cert=$(oc get secret "$secret" -n "$NAMESPACE" -o jsonpath="{.data['tls\.crt']}" | base64 --decode 2>&1)
+    if [[ -z "$secret_cert" ]]; then
+        echo "Error retrieving or decoding certificate from secret $secret for service $service."
+        continue
+    fi
+
+    for port in $ports; do
+        echo "Connecting to $service on port $port..."
+
+        # Captures the certificate section from the openssl output
+        pod_cert=$(oc rsh -n "$NAMESPACE" openstackclient openssl s_client -connect "$cluster_ip:$port" -servername "$cluster_ip" </dev/null 2>/dev/null | sed -ne '/-----BEGIN CERTIFICATE-----/,/-----END CERTIFICATE-----/p')
+
+        if [[ -z "$pod_cert" ]]; then
+            echo "Error retrieving certificate from $service at $cluster_ip:$port."
+            continue
+        fi
+
+        if [[ "$pod_cert" == "$secret_cert" ]]; then
+            echo "Certificates for $service on port $port match the secret."
+        else
+            echo "Certificates for $service on port $port DO NOT match the secret."
+            mismatched_services+=("$service on port $port")
+        fi
+    done
+done
+
+# Gather the ClusterIP and ports for databases
+for database in "${!database_secrets[@]}"; do
+    secret="${database_secrets[$database]}"
+
+    database_info=$(oc get service "$database" -n "$NAMESPACE" -o jsonpath="{.spec.clusterIP} {.spec.ports[*].port}")
+    cluster_ip=$(echo "$database_info" | awk '{print $1}')
+    ports=$(echo "$database_info" | cut -d' ' -f2-)
+
+    echo "Checking database: $database (ClusterIP: $cluster_ip, Ports: $ports)"
+
+    # Fetch the certificate from the secret and decode it
+    secret_cert=$(oc get secret "$secret" -n "$NAMESPACE" -o jsonpath="{.data['tls\.crt']}" | base64 --decode 2>&1)
+    if [[ -z "$secret_cert" ]]; then
+        echo "Error retrieving or decoding certificate from secret $secret for database $database."
+        continue
+    fi
+
+    for port in $ports; do
+        echo "Connecting to $database on port $port..."
+
+        pod_cert=$(oc rsh -n "$NAMESPACE" openstackclient openssl s_client -starttls mysql -connect "$cluster_ip:$port" </dev/null 2>/dev/null | sed -ne '/-----BEGIN CERTIFICATE-----/,/-----END CERTIFICATE-----/p')
+
+        if [[ -z "$pod_cert" ]]; then
+            echo "Error retrieving certificate from $database at $cluster_ip:$port."
+            continue
+        fi
+
+        if [[ "$pod_cert" == "$secret_cert" ]]; then
+            echo "Certificates for $database on port $port match the secret."
+        else
+            echo "Certificates for $database on port $port DO NOT match the secret."
+            mismatched_services+=("$database on port $port")
+        fi
+    done
+done
+
+if [[ ${#mismatched_services[@]} -ne 0 ]]; then
+    echo "The following services had certificate mismatches:"
+    for mismatch in "${mismatched_services[@]}"; do
+        echo " - $mismatch"
+    done
+    exit 1
+fi
diff --git a/tests/kuttl/tests/ctlplane-tls-cert-rotation/04-assert-service-cert-rotation.yaml b/tests/kuttl/tests/ctlplane-tls-cert-rotation/04-assert-service-cert-rotation.yaml
index 35b4c4583..6525e4e14 100644
--- a/tests/kuttl/tests/ctlplane-tls-cert-rotation/04-assert-service-cert-rotation.yaml
+++ b/tests/kuttl/tests/ctlplane-tls-cert-rotation/04-assert-service-cert-rotation.yaml
@@ -2,6 +2,10 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 timeout: 900
 commands:
+  - script: |
+      echo "Checking rotation of non API service certificates..."
+      NAMESPACE=$NAMESPACE bash ../../common/osp_check_noapi_service_certs.sh
+
   - script: |
       echo "Get fingerprints of all service certs"
       oc exec -i openstackclient -n $NAMESPACE -- bash -s < ../../common/osp_endpoint_fingerprints.sh > /tmp/endpoint_fingerprints_after
diff --git a/tests/kuttl/tests/ctlplane-tls-custom-issuers/02-assert-service-certs-issuers.yaml b/tests/kuttl/tests/ctlplane-tls-custom-issuers/02-assert-service-certs-issuers.yaml
index ec986a033..480c287b0 100644
--- a/tests/kuttl/tests/ctlplane-tls-custom-issuers/02-assert-service-certs-issuers.yaml
+++ b/tests/kuttl/tests/ctlplane-tls-custom-issuers/02-assert-service-certs-issuers.yaml
@@ -2,6 +2,10 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 timeout: 60
 commands:
+  - script: |
+      echo "Checking rotation of non API service certificates..."
+      NAMESPACE=$NAMESPACE bash ../../common/osp_check_noapi_service_certs.sh
+
   - script: |
       echo "Checking issuer of internal certificates..."
       oc exec -i openstackclient -n $NAMESPACE -- bash -s < ../../common/osp_check_cert_issuer.sh "issuer=CN=rootca-internal-custom" "internal"
diff --git a/tests/kuttl/tests/ctlplane-tls-custom-issuers/04-assert-service-certs-default-issuers.yaml b/tests/kuttl/tests/ctlplane-tls-custom-issuers/04-assert-service-certs-default-issuers.yaml
index 3228171eb..56c923699 100644
--- a/tests/kuttl/tests/ctlplane-tls-custom-issuers/04-assert-service-certs-default-issuers.yaml
+++ b/tests/kuttl/tests/ctlplane-tls-custom-issuers/04-assert-service-certs-default-issuers.yaml
@@ -6,6 +6,10 @@ commands:
       echo "Waiting for OpenStack control plane to be ready..."
       oc wait openstackcontrolplane -n $NAMESPACE --for=condition=Ready --timeout=400s -l core.openstack.org/openstackcontrolplane
 
+  - script: |
+      echo "Checking rotation of non API service certificates..."
+      NAMESPACE=$NAMESPACE bash ../../common/osp_check_noapi_service_certs.sh
+
   - script: |
       echo "Checking issuer of internal certificates..."
       oc exec -i openstackclient -n $NAMESPACE -- bash -s < ../../common/osp_check_cert_issuer.sh "issuer=CN=rootca-internal" "internal"
diff --git a/tests/kuttl/tests/ctlplane-tls-custom-issuers/07-assert-service-certs-default-issuers.yaml b/tests/kuttl/tests/ctlplane-tls-custom-issuers/07-assert-service-certs-default-issuers.yaml
index c663b6e54..29cab09b9 100644
--- a/tests/kuttl/tests/ctlplane-tls-custom-issuers/07-assert-service-certs-default-issuers.yaml
+++ b/tests/kuttl/tests/ctlplane-tls-custom-issuers/07-assert-service-certs-default-issuers.yaml
@@ -2,6 +2,10 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 timeout: 60
 commands:
+  - script: |
+        echo "Checking rotation of non API service certificates..."
+        NAMESPACE=$NAMESPACE bash ../../common/osp_check_noapi_service_certs.sh
+
   - script: |
       echo "Checking issuer of internal certificates..."
       oc exec -i openstackclient -n $NAMESPACE -- bash -s < ../../common/osp_check_cert_issuer.sh "issuer=CN=rootca-internal" "internal"
diff --git a/tests/kuttl/tests/ctlplane-tls-custom-issuers/10-assert-service-certs-issuers.yaml b/tests/kuttl/tests/ctlplane-tls-custom-issuers/10-assert-service-certs-issuers.yaml
index 7327c8efa..279a3e1ef 100644
--- a/tests/kuttl/tests/ctlplane-tls-custom-issuers/10-assert-service-certs-issuers.yaml
+++ b/tests/kuttl/tests/ctlplane-tls-custom-issuers/10-assert-service-certs-issuers.yaml
@@ -6,6 +6,10 @@ commands:
       echo "Waiting for OpenStack control plane to be ready..."
       oc wait openstackcontrolplane -n $NAMESPACE --for=condition=Ready --timeout=400s -l core.openstack.org/openstackcontrolplane
 
+  - script: |
+      echo "Checking rotation of non API service certificates..."
+      NAMESPACE=$NAMESPACE bash ../../common/osp_check_noapi_service_certs.sh
+
   - script: |
       echo "Checking issuer of internal certificates..."
       oc exec -i openstackclient -n $NAMESPACE -- bash -s < ../../common/osp_check_cert_issuer.sh "issuer=CN=rootca-internal-custom" "internal"