From e9a0204b5df2c90e49473c58ffae3b2e78a21d61 Mon Sep 17 00:00:00 2001 From: Rui Vieira Date: Mon, 22 Jul 2024 10:40:35 +0100 Subject: [PATCH] feat: Add TLS certificate mount on ModelMesh (#255) * feat: Add TLS certificate mount on ModelMesh * Revert from http to https until https://github.com/kserve/modelmesh/pull/147 is merged --- controllers/certificates.go | 31 ++++++++++++++++ controllers/constants.go | 2 + controllers/inference_services.go | 62 +++++++++++++++++++++++++++++-- controllers/utils.go | 5 ++- 4 files changed, 95 insertions(+), 5 deletions(-) diff --git a/controllers/certificates.go b/controllers/certificates.go index 7a3fc1fb..16faa1f9 100644 --- a/controllers/certificates.go +++ b/controllers/certificates.go @@ -3,10 +3,41 @@ package controllers import ( "context" trustyaiopendatahubiov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" ) +const ( + tlsMountPath = "/etc/trustyai/tls" +) + +// TLSCertVolumes holds the volume and volume mount for the TLS certificates +type TLSCertVolumes struct { + volume corev1.Volume + volumeMount corev1.VolumeMount +} + +// createFor creates the required volumes and volume mount for the TLS certificates for a specific Kubernetes secret +func (cert *TLSCertVolumes) createFor(instance *trustyaiopendatahubiov1alpha1.TrustyAIService) { + volume := corev1.Volume{ + Name: instance.Name + "-internal", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: instance.Name + "-internal", + }, + }, + } + + volumeMount := corev1.VolumeMount{ + Name: instance.Name + "-internal", + MountPath: tlsMountPath, + ReadOnly: true, + } + cert.volume = volume + cert.volumeMount = volumeMount +} + func (r *TrustyAIServiceReconciler) GetCustomCertificatesBundle(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService) CustomCertificatesBundle { var customCertificatesBundle CustomCertificatesBundle diff --git a/controllers/constants.go b/controllers/constants.go index f55da3a2..1fc4eb90 100644 --- a/controllers/constants.go +++ b/controllers/constants.go @@ -9,6 +9,8 @@ const ( serviceMonitorName = "trustyai-metrics" finalizerName = "trustyai.opendatahub.io/finalizer" payloadProcessorName = "MM_PAYLOAD_PROCESSORS" + tlsKeyCertPathName = "MM_TLS_KEY_CERT_PATH" + mmContainerName = "mm" modelMeshLabelKey = "modelmesh-service" modelMeshLabelValue = "modelmesh-serving" volumeMountName = "volume" diff --git a/controllers/inference_services.go b/controllers/inference_services.go index bde29551..773d4924 100644 --- a/controllers/inference_services.go +++ b/controllers/inference_services.go @@ -13,6 +13,10 @@ import ( ) func (r *TrustyAIServiceReconciler) patchEnvVarsForDeployments(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService, deployments []appsv1.Deployment, envVarName string, url string, remove bool) (bool, error) { + // Create volume and volume mount for this intance's TLS secrets + certVolumes := TLSCertVolumes{} + certVolumes.createFor(instance) + // Loop over the Deployments for _, deployment := range deployments { @@ -23,8 +27,31 @@ func (r *TrustyAIServiceReconciler) patchEnvVarsForDeployments(ctx context.Conte return false, nil } + // If the secret volume doesn't exist, add it + volumeExists := false + for _, vol := range deployment.Spec.Template.Spec.Volumes { + if vol.Name == instance.Name+"-internal" { + volumeExists = true + break + } + } + if !volumeExists { + deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, certVolumes.volume) + } + // Loop over all containers in the Deployment's Pod template for i := range deployment.Spec.Template.Spec.Containers { + mountExists := false + for _, mount := range deployment.Spec.Template.Spec.Containers[i].VolumeMounts { + if mount.Name == instance.Name+"-internal" { + mountExists = true + break + } + } + if !mountExists { + deployment.Spec.Template.Spec.Containers[i].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[i].VolumeMounts, certVolumes.volumeMount) + } + // Store the original environment variable list // Get the existing env var var envVar *corev1.EnvVar @@ -50,14 +77,17 @@ func (r *TrustyAIServiceReconciler) patchEnvVarsForDeployments(ctx context.Conte } else if envVar != nil { // If the env var exists and already contains the value, don't do anything existingValues := strings.Split(envVar.Value, " ") + valueExists := false for _, v := range existingValues { if v == url { - continue + valueExists = true + break } } - // Modify the existing env var based on the remove flag and current value - envVar.Value = generateEnvVarValue(envVar.Value, url, remove) + if !valueExists { + envVar.Value = generateEnvVarValue(envVar.Value, url, remove) + } } // Only update the deployment if the var value has to change, or we are removing it @@ -70,6 +100,32 @@ func (r *TrustyAIServiceReconciler) patchEnvVarsForDeployments(ctx context.Conte r.eventModelMeshConfigured(instance) log.FromContext(ctx).Info("Updating Deployment " + deployment.Name + ", container spec " + deployment.Spec.Template.Spec.Containers[i].Name + ", env var " + envVarName + " to " + url) } + + // Check TLS environment variable on ModelMesh + if deployment.Spec.Template.Spec.Containers[i].Name == mmContainerName { + tlsKeyCertPathEnvValue := tlsMountPath + "/tls.crt" + tlsKeyCertPathExists := false + for _, envVar := range deployment.Spec.Template.Spec.Containers[i].Env { + if envVar.Name == tlsKeyCertPathName { + tlsKeyCertPathExists = true + break + } + } + + // Doesn't exist, so we can add + if !tlsKeyCertPathExists { + deployment.Spec.Template.Spec.Containers[i].Env = append(deployment.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{ + Name: tlsKeyCertPathName, + Value: tlsKeyCertPathEnvValue, + }) + + if err := r.Update(ctx, &deployment); err != nil { + log.FromContext(ctx).Error(err, "Could not update Deployment", "Deployment", deployment.Name) + return false, err + } + log.FromContext(ctx).Info("Added environment variable " + tlsKeyCertPathName + " to deployment " + deployment.Name + " for container " + mmContainerName) + } + } } } diff --git a/controllers/utils.go b/controllers/utils.go index 4a8415e1..91ee9bdb 100644 --- a/controllers/utils.go +++ b/controllers/utils.go @@ -2,9 +2,10 @@ package controllers import ( "context" + "os" + appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/labels" - "os" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -63,5 +64,5 @@ func (r *TrustyAIServiceReconciler) GetDeploymentsByLabel(ctx context.Context, n // generateServiceURL generates an internal URL for a TrustyAI service func generateServiceURL(crName string, namespace string) string { - return "http://" + crName + "." + namespace + ".svc.cluster.local" + return "http://" + crName + "." + namespace + ".svc" }