feat: Add TLS certificate mount on ModelMesh (#255)

* feat: Add TLS certificate mount on ModelMesh

* Revert from http to https until kserve/modelmesh#147 is merged

controllers/certificates.go

@@ -3,10 +3,41 @@ package controllers
 import (
 	"context"
 	trustyaiopendatahubiov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 )
 
+const (
+	tlsMountPath = "/etc/trustyai/tls"
+)
+
+// TLSCertVolumes holds the volume and volume mount for the TLS certificates
+type TLSCertVolumes struct {
+	volume      corev1.Volume
+	volumeMount corev1.VolumeMount
+}
+
+// createFor creates the required volumes and volume mount for the TLS certificates for a specific Kubernetes secret
+func (cert *TLSCertVolumes) createFor(instance *trustyaiopendatahubiov1alpha1.TrustyAIService) {
+	volume := corev1.Volume{
+		Name: instance.Name + "-internal",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName: instance.Name + "-internal",
+			},
+		},
+	}
+
+	volumeMount := corev1.VolumeMount{
+		Name:      instance.Name + "-internal",
+		MountPath: tlsMountPath,
+		ReadOnly:  true,
+	}
+	cert.volume = volume
+	cert.volumeMount = volumeMount
+}
+
 func (r *TrustyAIServiceReconciler) GetCustomCertificatesBundle(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService) CustomCertificatesBundle {
 
 	var customCertificatesBundle CustomCertificatesBundle

controllers/constants.go

@@ -9,6 +9,8 @@ const (
 	serviceMonitorName   = "trustyai-metrics"
 	finalizerName        = "trustyai.opendatahub.io/finalizer"
 	payloadProcessorName = "MM_PAYLOAD_PROCESSORS"
+	tlsKeyCertPathName   = "MM_TLS_KEY_CERT_PATH"
+	mmContainerName      = "mm"
 	modelMeshLabelKey    = "modelmesh-service"
 	modelMeshLabelValue  = "modelmesh-serving"
 	volumeMountName      = "volume"

controllers/inference_services.go

@@ -13,6 +13,10 @@ import (
 )
 
 func (r *TrustyAIServiceReconciler) patchEnvVarsForDeployments(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService, deployments []appsv1.Deployment, envVarName string, url string, remove bool) (bool, error) {
+	// Create volume and volume mount for this intance's TLS secrets
+	certVolumes := TLSCertVolumes{}
+	certVolumes.createFor(instance)
+
 	// Loop over the Deployments
 	for _, deployment := range deployments {
 
@@ -23,8 +27,31 @@ func (r *TrustyAIServiceReconciler) patchEnvVarsForDeployments(ctx context.Conte
 			return false, nil
 		}
 
+		// If the secret volume doesn't exist, add it
+		volumeExists := false
+		for _, vol := range deployment.Spec.Template.Spec.Volumes {
+			if vol.Name == instance.Name+"-internal" {
+				volumeExists = true
+				break
+			}
+		}
+		if !volumeExists {
+			deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, certVolumes.volume)
+		}
+
 		// Loop over all containers in the Deployment's Pod template
 		for i := range deployment.Spec.Template.Spec.Containers {
+			mountExists := false
+			for _, mount := range deployment.Spec.Template.Spec.Containers[i].VolumeMounts {
+				if mount.Name == instance.Name+"-internal" {
+					mountExists = true
+					break
+				}
+			}
+			if !mountExists {
+				deployment.Spec.Template.Spec.Containers[i].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[i].VolumeMounts, certVolumes.volumeMount)
+			}
+
 			// Store the original environment variable list
 			// Get the existing env var
 			var envVar *corev1.EnvVar
@@ -50,14 +77,17 @@ func (r *TrustyAIServiceReconciler) patchEnvVarsForDeployments(ctx context.Conte
 			} else if envVar != nil {
 				// If the env var exists and already contains the value, don't do anything
 				existingValues := strings.Split(envVar.Value, " ")
+				valueExists := false
 				for _, v := range existingValues {
 					if v == url {
-						continue
+						valueExists = true
+						break
 					}
 				}
 
-				// Modify the existing env var based on the remove flag and current value
-				envVar.Value = generateEnvVarValue(envVar.Value, url, remove)
+				if !valueExists {
+					envVar.Value = generateEnvVarValue(envVar.Value, url, remove)
+				}
 			}
 
 			// Only update the deployment if the var value has to change, or we are removing it
@@ -70,6 +100,32 @@ func (r *TrustyAIServiceReconciler) patchEnvVarsForDeployments(ctx context.Conte
 				r.eventModelMeshConfigured(instance)
 				log.FromContext(ctx).Info("Updating Deployment " + deployment.Name + ", container spec " + deployment.Spec.Template.Spec.Containers[i].Name + ", env var " + envVarName + " to " + url)
 			}
+
+			// Check TLS environment variable on ModelMesh
+			if deployment.Spec.Template.Spec.Containers[i].Name == mmContainerName {
+				tlsKeyCertPathEnvValue := tlsMountPath + "/tls.crt"
+				tlsKeyCertPathExists := false
+				for _, envVar := range deployment.Spec.Template.Spec.Containers[i].Env {
+					if envVar.Name == tlsKeyCertPathName {
+						tlsKeyCertPathExists = true
+						break
+					}
+				}
+
+				// Doesn't exist, so we can add
+				if !tlsKeyCertPathExists {
+					deployment.Spec.Template.Spec.Containers[i].Env = append(deployment.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{
+						Name:  tlsKeyCertPathName,
+						Value: tlsKeyCertPathEnvValue,
+					})
+
+					if err := r.Update(ctx, &deployment); err != nil {
+						log.FromContext(ctx).Error(err, "Could not update Deployment", "Deployment", deployment.Name)
+						return false, err
+					}
+					log.FromContext(ctx).Info("Added environment variable " + tlsKeyCertPathName + " to deployment " + deployment.Name + " for container " + mmContainerName)
+				}
+			}
 		}
 	}
 

controllers/utils.go

@@ -2,9 +2,10 @@ package controllers
 
 import (
 	"context"
+	"os"
+
 	appsv1 "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/labels"
-	"os"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 )
@@ -63,5 +64,5 @@ func (r *TrustyAIServiceReconciler) GetDeploymentsByLabel(ctx context.Context, n
 
 // generateServiceURL generates an internal URL for a TrustyAI service
 func generateServiceURL(crName string, namespace string) string {
-	return "http://" + crName + "." + namespace + ".svc.cluster.local"
+	return "http://" + crName + "." + namespace + ".svc"
 }