Merge pull request #469 from steveb/runlogwatch

Add a ramdisk-logs sidecar
openstack-k8s-operators · Sep 17, 2024 · c622913 · c622913
2 parents 13e2efa + 27719e8
commit c622913
Show file tree

Hide file tree

Showing 13 changed files with 86 additions and 10 deletions.
diff --git a/controllers/ironicconductor_controller.go b/controllers/ironicconductor_controller.go
@@ -827,8 +827,9 @@ func (r *IronicConductorReconciler) generateServiceConfigMaps(
 			Type:         util.TemplateTypeScripts,
 			InstanceType: instance.Kind,
 			AdditionalTemplate: map[string]string{
-				"common.sh":  "/common/bin/common.sh",
-				"get_net_ip": "/common/bin/get_net_ip",
+				"common.sh":      "/common/bin/common.sh",
+				"get_net_ip":     "/common/bin/get_net_ip",
+				"runlogwatch.sh": "/common/bin/runlogwatch.sh",
 			},
 			Labels: cmLabels,
 		},

diff --git a/controllers/ironicinspector_controller.go b/controllers/ironicinspector_controller.go
@@ -1452,8 +1452,9 @@ func (r *IronicInspectorReconciler) generateServiceConfigMaps(
 			Type:         util.TemplateTypeScripts,
 			InstanceType: instance.Kind,
 			AdditionalTemplate: map[string]string{
-				"common.sh":  "/common/bin/common.sh",
-				"get_net_ip": "/common/bin/get_net_ip",
+				"common.sh":      "/common/bin/common.sh",
+				"get_net_ip":     "/common/bin/get_net_ip",
+				"runlogwatch.sh": "/common/bin/runlogwatch.sh",
 			},
 			Labels: cmLabels,
 		},

diff --git a/pkg/ironicconductor/statefulset.go b/pkg/ironicconductor/statefulset.go
@@ -151,10 +151,15 @@ func StatefulSet(
 	httpbootEnvVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS")
 	httpbootEnvVars["CONFIG_HASH"] = env.SetValue(configHash)
 
+	ramdiskLogsEnvVars := map[string]env.Setter{}
+	ramdiskLogsEnvVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS")
+	ramdiskLogsEnvVars["CONFIG_HASH"] = env.SetValue(configHash)
+
 	volumes := GetVolumes(instance)
 	conductorVolumeMounts := GetVolumeMounts("ironic-conductor")
 	httpbootVolumeMounts := GetVolumeMounts("httpboot")
 	dnsmasqVolumeMounts := GetVolumeMounts("dnsmasq")
+	ramdiskLogsVolumeMounts := GetVolumeMounts("ramdisk-logs")
 	initVolumeMounts := GetInitVolumeMounts()
 
 	// Add the CA bundle
@@ -163,6 +168,7 @@ func StatefulSet(
 		conductorVolumeMounts = append(conductorVolumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
 		httpbootVolumeMounts = append(httpbootVolumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
 		dnsmasqVolumeMounts = append(dnsmasqVolumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
+		ramdiskLogsVolumeMounts = append(ramdiskLogsVolumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
 		initVolumeMounts = append(initVolumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
 	}
 
@@ -201,10 +207,24 @@ func StatefulSet(
 		LivenessProbe:  httpbootLivenessProbe,
 		// StartupProbe:   startupProbe,
 	}
+	ramdiskLogsContainer := corev1.Container{
+		Name: "ramdisk-logs",
+		Command: []string{
+			"/bin/bash",
+		},
+		Args:         args,
+		Image:        instance.Spec.ContainerImage,
+		Env:          env.MergeEnvs([]corev1.EnvVar{}, ramdiskLogsEnvVars),
+		VolumeMounts: ramdiskLogsVolumeMounts,
+		SecurityContext: &corev1.SecurityContext{
+			RunAsUser: &runAsUser,
+		},
+	}
 
 	containers := []corev1.Container{
 		conductorContainer,
 		httpbootContainer,
+		ramdiskLogsContainer,
 	}
 
 	if instance.Spec.ProvisionNetwork != "" {

diff --git a/pkg/ironicinspector/statefulset.go b/pkg/ironicinspector/statefulset.go
@@ -145,6 +145,7 @@ func StatefulSet(
 	httpdVolumeMounts := GetVolumeMounts("httpd")
 	inspectorVolumeMounts := GetVolumeMounts("ironic-inspector")
 	dnsmasqVolumeMounts := GetVolumeMounts("dnsmasq")
+	ramdiskLogsVolumeMounts := GetVolumeMounts("ramdisk-logs")
 	initVolumeMounts := GetInitVolumeMounts()
 
 	// add CA cert if defined
@@ -154,6 +155,7 @@ func StatefulSet(
 		inspectorVolumeMounts = append(inspectorVolumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
 		httpbootVolumeMounts = append(httpbootVolumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
 		dnsmasqVolumeMounts = append(dnsmasqVolumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
+		ramdiskLogsVolumeMounts = append(ramdiskLogsVolumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
 		initVolumeMounts = append(initVolumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
 	}
 
@@ -240,6 +242,24 @@ func StatefulSet(
 	}
 	containers = append(containers, httpbootContainer)
 
+	ramdiskLogsEnvVars := map[string]env.Setter{}
+	ramdiskLogsEnvVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS")
+	ramdiskLogsEnvVars["CONFIG_HASH"] = env.SetValue(configHash)
+	ramdiskLogsContainer := corev1.Container{
+		Name: "ramdisk-logs",
+		Command: []string{
+			"/bin/bash",
+		},
+		Args:         args,
+		Image:        instance.Spec.ContainerImage,
+		Env:          env.MergeEnvs([]corev1.EnvVar{}, ramdiskLogsEnvVars),
+		VolumeMounts: ramdiskLogsVolumeMounts,
+		SecurityContext: &corev1.SecurityContext{
+			RunAsUser: &runAsUser,
+		},
+	}
+	containers = append(containers, ramdiskLogsContainer)
+
 	if instance.Spec.InspectionNetwork != "" {
 		// Only include dnsmasq container if there is an inspection network
 		dnsmasqEnvVars := map[string]env.Setter{}

diff --git a/templates/common/bin/runlogwatch.sh b/templates/common/bin/runlogwatch.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/bash
+
+# Ramdisk logs path
+LOG_DIR=${LOG_DIR:-/var/lib/ironic/ramdisk-logs}
+
+while :; do
+    sleep 5
+
+    while read -r fn; do
+        echo
+        echo "************ Contents of $fn ramdisk log file bundle **************"
+        tar -xOzvvf "$fn" | sed -e "s/^/$(basename "$fn"): /"
+        rm -f "$fn"
+    # find all *.tar.gz files which are older than six seconds
+    done < <(find "${LOG_DIR}" -mmin +0.1 -type f -name "*.tar.gz")
+
+done
diff --git a/templates/common/config/ironic.conf b/templates/common/config/ironic.conf
@@ -36,6 +36,10 @@ auth_strategy={{if .Standalone}}noauth{{else}}keystone{{end}}
 grub_config_path=EFI/BOOT/grub.cfg
 isolinux_bin=/usr/share/syslinux/isolinux.bin
 
+
+[agent]
+deploy_logs_local_path=/var/lib/ironic/ramdisk-logs
+
 {{if .Standalone}}
 [service_catalog]
 auth_type=none

diff --git a/templates/ironicconductor/bin/init.sh b/templates/ironicconductor/bin/init.sh
@@ -51,6 +51,9 @@ fi
 if [ ! -d "/var/lib/ironic/httpboot" ]; then
     mkdir /var/lib/ironic/httpboot
 fi
+if [ ! -d "/var/lib/ironic/ramdisk-logs" ]; then
+    mkdir /var/lib/ironic/ramdisk-logs
+fi
 # Build an ESP image
 pushd /var/lib/ironic/httpboot
 if [ ! -a "esp.img" ]; then

diff --git a/templates/ironicconductor/config/ramdisk-logs-config.json b/templates/ironicconductor/config/ramdisk-logs-config.json
@@ -0,0 +1,3 @@
+{
+    "command": "/usr/local/bin/container-scripts/runlogwatch.sh"
+}
diff --git a/templates/ironicinspector/bin/init.sh b/templates/ironicinspector/bin/init.sh
@@ -20,6 +20,10 @@ export TRANSPORTURL=${TransportURL:-""}
 
 export CUSTOMCONF=${CustomConf:-""}
 
+if [ ! -d "/var/lib/ironic/ramdisk-logs" ]; then
+    mkdir /var/lib/ironic/ramdisk-logs
+fi
+
 SVC_CFG=/etc/ironic-inspector/inspector.conf
 SVC_CFG_MERGED=/var/lib/config-data/merged/inspector.conf
 

diff --git a/templates/ironicinspector/config/inspector.conf b/templates/ironicinspector/config/inspector.conf
@@ -67,7 +67,7 @@ enforce_new_defaults=True
 [processing]
 keep_ports=all
 processing_hooks=$default_processing_hooks,extra_hardware,lldp_basic,local_link_connection,physnet_cidr_map
-ramdisk_logs_dir=/var/log/ironic-inspector/ramdisk/
+ramdisk_logs_dir=/var/lib/ironic/ramdisk-logs/
 always_store_ramdisk_logs=True
 store_data=database
 

diff --git a/templates/ironicinspector/config/ramdisk-logs-config.json b/templates/ironicinspector/config/ramdisk-logs-config.json
@@ -0,0 +1,3 @@
+{
+    "command": "/usr/local/bin/container-scripts/runlogwatch.sh"
+}
diff --git a/tests/functional/ironicconductor_controller_test.go b/tests/functional/ironicconductor_controller_test.go
@@ -276,7 +276,7 @@ var _ = Describe("IronicConductor controller", func() {
 			// Check the resulting deployment fields
 			Expect(int(*depl.Spec.Replicas)).To(Equal(1))
 			Expect(depl.Spec.Template.Spec.Volumes).To(HaveLen(6))
-			Expect(depl.Spec.Template.Spec.Containers).To(HaveLen(2))
+			Expect(depl.Spec.Template.Spec.Containers).To(HaveLen(3))
 
 			// cert deployment volumes
 			th.AssertVolumeExists(ironicNames.CaBundleSecretName.Name, depl.Spec.Template.Spec.Volumes)
@@ -309,7 +309,7 @@ var _ = Describe("IronicConductor controller", func() {
 			// Check the resulting deployment fields
 			Expect(int(*depl.Spec.Replicas)).To(Equal(1))
 			Expect(depl.Spec.Template.Spec.Volumes).To(HaveLen(6))
-			Expect(depl.Spec.Template.Spec.Containers).To(HaveLen(2))
+			Expect(depl.Spec.Template.Spec.Containers).To(HaveLen(3))
 
 			// Grab the current config hash
 			originalHash := GetEnvVarValue(

diff --git a/tests/functional/ironicinspector_controller_test.go b/tests/functional/ironicinspector_controller_test.go
@@ -199,7 +199,7 @@ var _ = Describe("IronicInspector controller", func() {
 			// Check the resulting deployment fields
 			Expect(int(*ss.Spec.Replicas)).To(Equal(1))
 			Expect(ss.Spec.Template.Spec.Volumes).To(HaveLen(6))
-			Expect(ss.Spec.Template.Spec.Containers).To(HaveLen(3))
+			Expect(ss.Spec.Template.Spec.Containers).To(HaveLen(4))
 
 			// Check the ironic-inspector-httpd container
 			container := ss.Spec.Template.Spec.Containers[0]
@@ -393,7 +393,7 @@ var _ = Describe("IronicInspector controller", func() {
 			// Check the resulting deployment fields
 			Expect(int(*depl.Spec.Replicas)).To(Equal(1))
 			Expect(depl.Spec.Template.Spec.Volumes).To(HaveLen(9))
-			Expect(depl.Spec.Template.Spec.Containers).To(HaveLen(3))
+			Expect(depl.Spec.Template.Spec.Containers).To(HaveLen(4))
 
 			// cert deployment volumes
 			th.AssertVolumeExists(ironicNames.CaBundleSecretName.Name, depl.Spec.Template.Spec.Volumes)
@@ -483,7 +483,7 @@ var _ = Describe("IronicInspector controller", func() {
 			// Check the resulting deployment fields
 			Expect(int(*depl.Spec.Replicas)).To(Equal(1))
 			Expect(depl.Spec.Template.Spec.Volumes).To(HaveLen(9))
-			Expect(depl.Spec.Template.Spec.Containers).To(HaveLen(3))
+			Expect(depl.Spec.Template.Spec.Containers).To(HaveLen(4))
 
 			// Grab the current config hash
 			originalHash := GetEnvVarValue(