Add a conductor ramdisk-logs sidecar

This implements the runlogwatch.sh approach of metal3[1] to write any deployment logs to the console, making them accessable to the end user via the same mechanism as other openstack service logs. As part of this change the deploy logs directory needs to be moved to a directory on a volume shared by the pod. This is now /var/lib/ironic/ramdisk-logs Jira: OSPRH-1947 [1] https://github.com/metal3-io/ironic-image/blob/main/scripts/runlogwatch.sh
openstack-k8s-operators · Sep 16, 2024 · 52d00a2 · 52d00a2
1 parent 93e827d
commit 52d00a2
Show file tree

Hide file tree

Showing 7 changed files with 52 additions and 4 deletions.
diff --git a/controllers/ironicconductor_controller.go b/controllers/ironicconductor_controller.go
@@ -827,8 +827,9 @@ func (r *IronicConductorReconciler) generateServiceConfigMaps(
 			Type:         util.TemplateTypeScripts,
 			InstanceType: instance.Kind,
 			AdditionalTemplate: map[string]string{
-				"common.sh":  "/common/bin/common.sh",
-				"get_net_ip": "/common/bin/get_net_ip",
+				"common.sh":      "/common/bin/common.sh",
+				"get_net_ip":     "/common/bin/get_net_ip",
+				"runlogwatch.sh": "/common/bin/runlogwatch.sh",
 			},
 			Labels: cmLabels,
 		},

diff --git a/pkg/ironicconductor/statefulset.go b/pkg/ironicconductor/statefulset.go
@@ -151,10 +151,15 @@ func StatefulSet(
 	httpbootEnvVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS")
 	httpbootEnvVars["CONFIG_HASH"] = env.SetValue(configHash)
 
+	ramdiskLogsEnvVars := map[string]env.Setter{}
+	ramdiskLogsEnvVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS")
+	ramdiskLogsEnvVars["CONFIG_HASH"] = env.SetValue(configHash)
+
 	volumes := GetVolumes(instance)
 	conductorVolumeMounts := GetVolumeMounts("ironic-conductor")
 	httpbootVolumeMounts := GetVolumeMounts("httpboot")
 	dnsmasqVolumeMounts := GetVolumeMounts("dnsmasq")
+	ramdiskLogsVolumeMounts := GetVolumeMounts("ramdisk-logs")
 	initVolumeMounts := GetInitVolumeMounts()
 
 	// Add the CA bundle
@@ -163,6 +168,7 @@ func StatefulSet(
 		conductorVolumeMounts = append(conductorVolumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
 		httpbootVolumeMounts = append(httpbootVolumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
 		dnsmasqVolumeMounts = append(dnsmasqVolumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
+		ramdiskLogsVolumeMounts = append(ramdiskLogsVolumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
 		initVolumeMounts = append(initVolumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
 	}
 
@@ -201,10 +207,24 @@ func StatefulSet(
 		LivenessProbe:  httpbootLivenessProbe,
 		// StartupProbe:   startupProbe,
 	}
+	ramdiskLogsContainer := corev1.Container{
+		Name: "ramdisk-logs",
+		Command: []string{
+			"/bin/bash",
+		},
+		Args:         args,
+		Image:        instance.Spec.ContainerImage,
+		Env:          env.MergeEnvs([]corev1.EnvVar{}, ramdiskLogsEnvVars),
+		VolumeMounts: ramdiskLogsVolumeMounts,
+		SecurityContext: &corev1.SecurityContext{
+			RunAsUser: &runAsUser,
+		},
+	}
 
 	containers := []corev1.Container{
 		conductorContainer,
 		httpbootContainer,
+		ramdiskLogsContainer,
 	}
 
 	if instance.Spec.ProvisionNetwork != "" {

diff --git a/templates/common/bin/runlogwatch.sh b/templates/common/bin/runlogwatch.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/bash
+
+# Ramdisk logs path
+LOG_DIR=${LOG_DIR:-/var/lib/ironic/ramdisk-logs}
+
+while :; do
+    sleep 5
+
+    while read -r fn; do
+        echo
+        echo "************ Contents of $fn ramdisk log file bundle **************"
+        tar -xOzvvf "$fn" | sed -e "s/^/$(basename "$fn"): /"
+        rm -f "$fn"
+    # find all *.tar.gz files which are older than six seconds
+    done < <(find "${LOG_DIR}" -mmin +0.1 -type f -name "*.tar.gz")
+
+done
diff --git a/templates/common/config/ironic.conf b/templates/common/config/ironic.conf
@@ -36,6 +36,10 @@ auth_strategy={{if .Standalone}}noauth{{else}}keystone{{end}}
 grub_config_path=EFI/BOOT/grub.cfg
 isolinux_bin=/usr/share/syslinux/isolinux.bin
 
+
+[agent]
+deploy_logs_local_path=/var/lib/ironic/ramdisk-logs
+
 {{if .Standalone}}
 [service_catalog]
 auth_type=none

diff --git a/templates/ironicconductor/bin/init.sh b/templates/ironicconductor/bin/init.sh
@@ -51,6 +51,9 @@ fi
 if [ ! -d "/var/lib/ironic/httpboot" ]; then
     mkdir /var/lib/ironic/httpboot
 fi
+if [ ! -d "/var/lib/ironic/ramdisk-logs" ]; then
+    mkdir /var/lib/ironic/ramdisk-logs
+fi
 # Build an ESP image
 pushd /var/lib/ironic/httpboot
 if [ ! -a "esp.img" ]; then

diff --git a/templates/ironicconductor/config/ramdisk-logs-config.json b/templates/ironicconductor/config/ramdisk-logs-config.json
@@ -0,0 +1,3 @@
+{
+    "command": "/usr/local/bin/container-scripts/runlogwatch.sh"
+}
diff --git a/tests/functional/ironicconductor_controller_test.go b/tests/functional/ironicconductor_controller_test.go
@@ -276,7 +276,7 @@ var _ = Describe("IronicConductor controller", func() {
 			// Check the resulting deployment fields
 			Expect(int(*depl.Spec.Replicas)).To(Equal(1))
 			Expect(depl.Spec.Template.Spec.Volumes).To(HaveLen(6))
-			Expect(depl.Spec.Template.Spec.Containers).To(HaveLen(2))
+			Expect(depl.Spec.Template.Spec.Containers).To(HaveLen(3))
 
 			// cert deployment volumes
 			th.AssertVolumeExists(ironicNames.CaBundleSecretName.Name, depl.Spec.Template.Spec.Volumes)
@@ -309,7 +309,7 @@ var _ = Describe("IronicConductor controller", func() {
 			// Check the resulting deployment fields
 			Expect(int(*depl.Spec.Replicas)).To(Equal(1))
 			Expect(depl.Spec.Template.Spec.Volumes).To(HaveLen(6))
-			Expect(depl.Spec.Template.Spec.Containers).To(HaveLen(2))
+			Expect(depl.Spec.Template.Spec.Containers).To(HaveLen(3))
 
 			// Grab the current config hash
 			originalHash := GetEnvVarValue(