From 4c8550ed13a65be7c90228fbfceb31f4bf32fe89 Mon Sep 17 00:00:00 2001 From: Rui Vieira Date: Fri, 12 Jul 2024 21:47:45 +0100 Subject: [PATCH] feat: Initial database support (#246) * Initial database support - Add status checking - Add better storage flags - Add spec.storage.format validation - Add DDL -Add HIBERNATE format to DB (test) - Update service image - Revert identifier to DATABASE - Update CR options (remove mandatory data) * Remove default DDL generation env var * Update service image to latest tag * Add migration awareness * Add updating pods for migration * Change JDBC url from mysql to mariadb * Fix TLS mount * Revert images * Remove redundant logic * Fix comments --- api/v1alpha1/trustyaiservice_types.go | 39 +- ...styai.opendatahub.io_trustyaiservices.yaml | 12 +- controllers/constants.go | 11 +- controllers/deployment.go | 100 +- controllers/deployment_test.go | 877 ++++++++++++------ controllers/monitor_test.go | 2 +- controllers/route.go | 1 + controllers/route_test.go | 102 +- controllers/secrets.go | 60 ++ controllers/service_accounts_test.go | 4 +- controllers/statuses.go | 44 +- controllers/statuses_test.go | 185 +++- controllers/storage_test.go | 16 +- controllers/suite_test.go | 87 +- .../templates/service/deployment.tmpl.yaml | 73 +- controllers/trustyaiservice_controller.go | 63 +- 16 files changed, 1270 insertions(+), 406 deletions(-) create mode 100644 controllers/secrets.go diff --git a/api/v1alpha1/trustyaiservice_types.go b/api/v1alpha1/trustyaiservice_types.go index 59f2bb7c..7ac65ae6 100644 --- a/api/v1alpha1/trustyaiservice_types.go +++ b/api/v1alpha1/trustyaiservice_types.go @@ -34,14 +34,17 @@ type TrustyAIService struct { } type StorageSpec struct { - Format string `json:"format"` - Folder string `json:"folder"` - Size string `json:"size"` + // Format only supports "PVC" or "DATABASE" values + // +kubebuilder:validation:Enum=PVC;DATABASE + Format string `json:"format"` + Folder string `json:"folder,omitempty"` + Size string `json:"size,omitempty"` + DatabaseConfigurations string `json:"databaseConfigurations,omitempty"` } type DataSpec struct { - Filename string `json:"filename"` - Format string `json:"format"` + Filename string `json:"filename,omitempty"` + Format string `json:"format,omitempty"` } type MetricsSpec struct { @@ -55,7 +58,7 @@ type TrustyAIServiceSpec struct { // +optional Replicas *int32 `json:"replicas"` Storage StorageSpec `json:"storage"` - Data DataSpec `json:"data"` + Data DataSpec `json:"data,omitempty"` Metrics MetricsSpec `json:"metrics"` } @@ -90,6 +93,30 @@ func init() { SchemeBuilder.Register(&TrustyAIService{}, &TrustyAIServiceList{}) } +// IsDatabaseConfigurationsSet returns true if the DatabaseConfigurations field is set. +func (s *StorageSpec) IsDatabaseConfigurationsSet() bool { + return s.DatabaseConfigurations != "" +} + +// IsStoragePVC returns true if the storage is set to PVC. +func (s *StorageSpec) IsStoragePVC() bool { + return s.Format == "PVC" +} + +// IsStorageDatabase returns true if the storage is set to database. +func (s *StorageSpec) IsStorageDatabase() bool { + return s.Format == "DATABASE" +} + +// IsMigration returns true if the migration fields are set. +func (t *TrustyAIService) IsMigration() bool { + if t.Spec.Storage.Format == "DATABASE" && t.Spec.Storage.Folder != "" && t.Spec.Data.Filename != "" { + return true + } else { + return false + } +} + // SetStatus sets the status of the TrustyAIService func (t *TrustyAIService) SetStatus(condType, reason, message string, status corev1.ConditionStatus) { now := metav1.Now() diff --git a/config/crd/bases/trustyai.opendatahub.io_trustyaiservices.yaml b/config/crd/bases/trustyai.opendatahub.io_trustyaiservices.yaml index 56921595..076a8082 100644 --- a/config/crd/bases/trustyai.opendatahub.io_trustyaiservices.yaml +++ b/config/crd/bases/trustyai.opendatahub.io_trustyaiservices.yaml @@ -41,9 +41,6 @@ spec: type: string format: type: string - required: - - filename - - format type: object metrics: properties: @@ -60,19 +57,22 @@ spec: type: integer storage: properties: + databaseConfigurations: + type: string folder: type: string format: + description: Format only supports "PVC" or "DATABASE" values + enum: + - PVC + - DATABASE type: string size: type: string required: - - folder - format - - size type: object required: - - data - metrics - storage type: object diff --git a/controllers/constants.go b/controllers/constants.go index adb130f5..f55da3a2 100644 --- a/controllers/constants.go +++ b/controllers/constants.go @@ -12,7 +12,14 @@ const ( modelMeshLabelKey = "modelmesh-service" modelMeshLabelValue = "modelmesh-serving" volumeMountName = "volume" - defaultRequeueDelay = 10 * time.Second + defaultRequeueDelay = 30 * time.Second + dbCredentialsSuffix = "-db-credentials" +) + +// Allowed storage formats +const ( + STORAGE_PVC = "PVC" + STORAGE_DATABASE = "DATABASE" ) // Configuration constants @@ -56,3 +63,5 @@ const ( EventReasonInferenceServiceConfigured = "InferenceServiceConfigured" EventReasonServiceMonitorCreated = "ServiceMonitorCreated" ) + +const migrationAnnotationKey = "trustyai.opendatahub.io/db-migration" diff --git a/controllers/deployment.go b/controllers/deployment.go index d498a2a6..816c9c17 100644 --- a/controllers/deployment.go +++ b/controllers/deployment.go @@ -37,13 +37,14 @@ type DeploymentConfig struct { PVCClaimName string CustomCertificatesBundle CustomCertificatesBundle Version string + BatchSize int } // createDeploymentObject returns a Deployment for the TrustyAI Service instance func (r *TrustyAIServiceReconciler) createDeploymentObject(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService, serviceImage string, caBunble CustomCertificatesBundle) (*appsv1.Deployment, error) { var batchSize int - // If not batch size is provided, assume the default one + // If no batch size is provided, assume the default one if instance.Spec.Metrics.BatchSize == nil { batchSize = defaultBatchSize } else { @@ -66,6 +67,7 @@ func (r *TrustyAIServiceReconciler) createDeploymentObject(ctx context.Context, PVCClaimName: pvcName, CustomCertificatesBundle: caBunble, Version: Version, + BatchSize: batchSize, } var deployment *appsv1.Deployment @@ -81,42 +83,78 @@ func (r *TrustyAIServiceReconciler) createDeploymentObject(ctx context.Context, // reconcileDeployment returns a Deployment object with the same name/namespace as the cr func (r *TrustyAIServiceReconciler) createDeployment(ctx context.Context, cr *trustyaiopendatahubiov1alpha1.TrustyAIService, imageName string, caBundle CustomCertificatesBundle) error { - pvcName := generatePVCName(cr) + if !cr.Spec.Storage.IsDatabaseConfigurationsSet() { - pvc := &corev1.PersistentVolumeClaim{} - pvcerr := r.Get(ctx, types.NamespacedName{Name: pvcName, Namespace: cr.Namespace}, pvc) - if pvcerr != nil { - log.FromContext(ctx).Error(pvcerr, "PVC not found") - return pvcerr - } - if pvcerr == nil { - // The PVC is ready. We can now create the Deployment. - deployment, err := r.createDeploymentObject(ctx, cr, imageName, caBundle) - if err != nil { - // Error creating the deployment resource object - return err - } + pvcName := generatePVCName(cr) - if err := ctrl.SetControllerReference(cr, deployment, r.Scheme); err != nil { - log.FromContext(ctx).Error(err, "Error setting TrustyAIService as owner of Deployment.") - return err + pvc := &corev1.PersistentVolumeClaim{} + pvcerr := r.Get(ctx, types.NamespacedName{Name: pvcName, Namespace: cr.Namespace}, pvc) + if pvcerr != nil { + log.FromContext(ctx).Error(pvcerr, "PVC not found") + return pvcerr } - log.FromContext(ctx).Info("Creating Deployment.") - err = r.Create(ctx, deployment) - if err != nil { - log.FromContext(ctx).Error(err, "Error creating Deployment.") - return err + } + + // We can now create the Deployment. + deployment, err := r.createDeploymentObject(ctx, cr, imageName, caBundle) + if err != nil { + // Error creating the deployment resource object + return err + } + + if err := ctrl.SetControllerReference(cr, deployment, r.Scheme); err != nil { + log.FromContext(ctx).Error(err, "Error setting TrustyAIService as owner of Deployment.") + return err + } + log.FromContext(ctx).Info("Creating Deployment.") + err = r.Create(ctx, deployment) + if err != nil { + log.FromContext(ctx).Error(err, "Error creating Deployment.") + return err + } + // Created successfully + return nil + +} + +// updateDeployment returns a Deployment object with the same name/namespace as the cr +func (r *TrustyAIServiceReconciler) updateDeployment(ctx context.Context, cr *trustyaiopendatahubiov1alpha1.TrustyAIService, imageName string, caBundle CustomCertificatesBundle) error { + + if !cr.Spec.Storage.IsDatabaseConfigurationsSet() { + + pvcName := generatePVCName(cr) + + pvc := &corev1.PersistentVolumeClaim{} + pvcerr := r.Get(ctx, types.NamespacedName{Name: pvcName, Namespace: cr.Namespace}, pvc) + if pvcerr != nil { + log.FromContext(ctx).Error(pvcerr, "PVC not found") + return pvcerr } - // Created successfully - return nil + } - } else { - return ErrPVCNotReady + // We can now create the Deployment object. + deployment, err := r.createDeploymentObject(ctx, cr, imageName, caBundle) + if err != nil { + // Error creating the deployment resource object + return err + } + + if err := ctrl.SetControllerReference(cr, deployment, r.Scheme); err != nil { + log.FromContext(ctx).Error(err, "Error setting TrustyAIService as owner of Deployment.") + return err + } + log.FromContext(ctx).Info("Updating Deployment.") + err = r.Update(ctx, deployment) + if err != nil { + log.FromContext(ctx).Error(err, "Error updating Deployment.") + return err } + // Created successfully + return nil } -func (r *TrustyAIServiceReconciler) ensureDeployment(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService, caBundle CustomCertificatesBundle) error { +func (r *TrustyAIServiceReconciler) ensureDeployment(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService, caBundle CustomCertificatesBundle, migration bool) error { // Get image and tag from ConfigMap // If there's a ConfigMap with custom images, it is only applied when the operator is first deployed @@ -138,6 +176,12 @@ func (r *TrustyAIServiceReconciler) ensureDeployment(ctx context.Context, instan // Some other error occurred when trying to get the Deployment return err } + // Deployment exists, but we are migrating + if migration { + log.FromContext(ctx).Info("Found migration annotation. Migrating.") + return r.updateDeployment(ctx, instance, image, caBundle) + } + // Deployment is ready and using the PVC return nil } diff --git a/controllers/deployment_test.go b/controllers/deployment_test.go index b513edeb..de80a1c4 100644 --- a/controllers/deployment_test.go +++ b/controllers/deployment_test.go @@ -27,6 +27,297 @@ func printKubeObject(obj interface{}) { } } +func setupAndTestDeploymentDefault(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, namespace string) { + Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) + caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + + Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) + Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) + Expect(reconciler.ensureDeployment(ctx, instance, caBundle, false)).To(Succeed()) + + deployment := &appsv1.Deployment{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace}, deployment) + Expect(err).ToNot(HaveOccurred()) + Expect(deployment).ToNot(BeNil()) + + Expect(*deployment.Spec.Replicas).Should(Equal(int32(1))) + Expect(deployment.Namespace).Should(Equal(namespace)) + Expect(deployment.Name).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/name"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/instance"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) + Expect(deployment.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) + + Expect(len(deployment.Spec.Template.Spec.Containers)).Should(Equal(2)) + Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal("quay.io/trustyai/trustyai-service:latest")) + Expect(deployment.Spec.Template.Spec.Containers[1].Image).Should(Equal("registry.redhat.io/openshift4/ose-oauth-proxy:latest")) + + WaitFor(func() error { + service, _ := reconciler.reconcileService(ctx, instance) + return reconciler.Create(ctx, service) + }, "failed to create service") + + service := &corev1.Service{} + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: defaultServiceName, Namespace: namespace}, service) + }, "failed to get Service") + + Expect(service.Annotations["prometheus.io/path"]).Should(Equal("/q/metrics")) + Expect(service.Annotations["prometheus.io/scheme"]).Should(Equal("http")) + Expect(service.Annotations["prometheus.io/scrape"]).Should(Equal("true")) + Expect(service.Namespace).Should(Equal(namespace)) + + WaitFor(func() error { + err := reconciler.reconcileOAuthService(ctx, instance, caBundle) + return err + }, "failed to create oauth service") + + desiredOAuthService, err := generateTrustyAIOAuthService(ctx, instance, caBundle) + Expect(err).ToNot(HaveOccurred()) + + oauthService := &corev1.Service{} + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: desiredOAuthService.Name, Namespace: namespace}, oauthService) + }, "failed to get OAuth Service") + + // Check if the OAuth service has the expected labels + Expect(oauthService.Labels["app"]).Should(Equal(instance.Name)) + Expect(oauthService.Labels["app.kubernetes.io/instance"]).Should(Equal(instance.Name)) + Expect(oauthService.Labels["app.kubernetes.io/name"]).Should(Equal(instance.Name)) + Expect(oauthService.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) + Expect(oauthService.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) + Expect(oauthService.Labels["trustyai-service-name"]).Should(Equal(instance.Name)) + +} + +func setupAndTestDeploymentConfigMap(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, namespace string) { + serviceImage := "custom-service-image:foo" + oauthImage := "custom-oauth-proxy:bar" + Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) + + WaitFor(func() error { + configMap := createConfigMap(operatorNamespace, oauthImage, serviceImage) + return k8sClient.Create(ctx, configMap) + }, "failed to create ConfigMap") + + caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + + Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) + Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) + WaitFor(func() error { + return reconciler.ensureDeployment(ctx, instance, caBundle, false) + }, "failed to reconcile deployment") + + deployment := &appsv1.Deployment{} + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: instance.Name, Namespace: namespace}, deployment) + }, "failed to get updated deployment") + Expect(deployment).ToNot(BeNil()) + + Expect(*deployment.Spec.Replicas).Should(Equal(int32(1))) + Expect(deployment.Namespace).Should(Equal(namespace)) + Expect(deployment.Name).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/name"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/instance"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) + Expect(deployment.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) + + Expect(len(deployment.Spec.Template.Spec.Containers)).Should(Equal(2)) + Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal(serviceImage)) + Expect(deployment.Spec.Template.Spec.Containers[1].Image).Should(Equal(oauthImage)) + + WaitFor(func() error { + service, _ := reconciler.reconcileService(ctx, instance) + return reconciler.Create(ctx, service) + }, "failed to create service") + + service := &corev1.Service{} + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: defaultServiceName, Namespace: namespace}, service) + }, "failed to get Service") + + Expect(service.Annotations["prometheus.io/path"]).Should(Equal("/q/metrics")) + Expect(service.Annotations["prometheus.io/scheme"]).Should(Equal("http")) + Expect(service.Annotations["prometheus.io/scrape"]).Should(Equal("true")) + Expect(service.Namespace).Should(Equal(namespace)) + + WaitFor(func() error { + err := reconciler.reconcileOAuthService(ctx, instance, caBundle) + return err + }, "failed to create oauth service") + + desiredOAuthService, err := generateTrustyAIOAuthService(ctx, instance, caBundle) + Expect(err).ToNot(HaveOccurred()) + + oauthService := &corev1.Service{} + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: desiredOAuthService.Name, Namespace: namespace}, oauthService) + }, "failed to get OAuth Service") + + // Check if the OAuth service has the expected labels + Expect(oauthService.Labels["app"]).Should(Equal(instance.Name)) + Expect(oauthService.Labels["app.kubernetes.io/instance"]).Should(Equal(instance.Name)) + Expect(oauthService.Labels["app.kubernetes.io/name"]).Should(Equal(instance.Name)) + Expect(oauthService.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) + Expect(oauthService.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) + Expect(oauthService.Labels["trustyai-service-name"]).Should(Equal(instance.Name)) + +} + +func setupAndTestDeploymentNoCustomCABundle(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, namespace string) { + Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) + + caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + + Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) + Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) + WaitFor(func() error { + return reconciler.ensureDeployment(ctx, instance, caBundle, false) + }, "failed to create deployment") + + deployment := &appsv1.Deployment{} + namespacedName := types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace} + Expect(k8sClient.Get(ctx, namespacedName, deployment)).Should(Succeed()) + + Expect(deployment.Spec.Template.Spec.ServiceAccountName).To(Equal(instance.Name + "-proxy")) + + customCertificatesBundleVolumeName := caBundle + for _, volume := range deployment.Spec.Template.Spec.Volumes { + Expect(volume.Name).ToNot(Equal(customCertificatesBundleVolumeName)) + } + + for _, container := range deployment.Spec.Template.Spec.Containers { + for _, volumeMount := range container.VolumeMounts { + Expect(volumeMount.Name).ToNot(Equal(customCertificatesBundleVolumeName)) + } + for _, arg := range container.Args { + Expect(arg).ToNot(ContainSubstring("--openshift-ca")) + } + } + +} + +func setupAndTestDeploymentCustomCABundle(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, namespace string) { + caBundleConfigMap := createTrustedCABundleConfigMap(namespace) + Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) + Expect(k8sClient.Create(ctx, caBundleConfigMap)).To(Succeed()) + + caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + + Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) + Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) + WaitFor(func() error { + return reconciler.ensureDeployment(ctx, instance, caBundle, false) + }, "failed to create deployment") + + deployment := &appsv1.Deployment{} + namespacedName := types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace} + Expect(k8sClient.Get(ctx, namespacedName, deployment)).Should(Succeed()) + + Expect(deployment.Spec.Template.Spec.ServiceAccountName).To(Equal(instance.Name + "-proxy")) + + foundCustomCertificatesBundleVolumeMount := false + + customCertificatesBundleMountPath := "/etc/ssl/certs/ca-bundle.crt" + for _, container := range deployment.Spec.Template.Spec.Containers { + for _, volumeMount := range container.VolumeMounts { + if volumeMount.Name == caBundleName && volumeMount.MountPath == customCertificatesBundleMountPath { + foundCustomCertificatesBundleVolumeMount = true + } + } + } + Expect(foundCustomCertificatesBundleVolumeMount).To(BeTrue(), caBundleName+" volume mount not found in any container") + + Expect(k8sClient.Delete(ctx, caBundleConfigMap)).To(Succeed(), "failed to delete custom certificates bundle ConfigMap") + +} + +func setupAndTestDeploymentServiceAccount(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, namespace string, mode string) { + Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) + + caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + + if mode == "PVC" { + Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) + } + Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) + WaitFor(func() error { + return reconciler.ensureDeployment(ctx, instance, caBundle, false) + }, "failed to create deployment") + + deployment := &appsv1.Deployment{} + namespacedName := types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace} + Expect(k8sClient.Get(ctx, namespacedName, deployment)).Should(Succeed()) + + Expect(deployment.Spec.Template.Spec.ServiceAccountName).To(Equal(instance.Name + "-proxy")) +} + +func setupAndTestDeploymentInferenceService(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, namespace string, mode string) { + WaitFor(func() error { + return createNamespace(ctx, k8sClient, namespace) + }, "failed to create namespace") + + caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + + WaitFor(func() error { + return createTestPVC(ctx, k8sClient, instance) + }, "failed to create PVC") + WaitFor(func() error { + return reconciler.ensureDeployment(ctx, instance, caBundle, false) + }, "failed to create deployment") + + // Creating the InferenceService + inferenceService := createInferenceService("my-model", namespace) + WaitFor(func() error { + return k8sClient.Create(ctx, inferenceService) + }, "failed to create deployment") + + Expect(reconciler.patchKServe(ctx, instance, *inferenceService, namespace, instance.Name, false)).ToNot(HaveOccurred()) + + deployment := &appsv1.Deployment{} + WaitFor(func() error { + // Define defaultServiceName for the deployment created by the operator + namespacedNamed := types.NamespacedName{ + Namespace: namespace, + Name: instance.Name, + } + return k8sClient.Get(ctx, namespacedNamed, deployment) + }, "failed to get Deployment") + + Expect(*deployment.Spec.Replicas).Should(Equal(int32(1))) + Expect(deployment.Namespace).Should(Equal(namespace)) + Expect(deployment.Name).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/name"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/instance"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) + Expect(deployment.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) + + WaitFor(func() error { + err := reconciler.reconcileOAuthService(ctx, instance, caBundle) + return err + }, "failed to create oauth service") + + desiredOAuthService, err := generateTrustyAIOAuthService(ctx, instance, caBundle) + Expect(err).ToNot(HaveOccurred()) + + oauthService := &corev1.Service{} + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: desiredOAuthService.Name, Namespace: namespace}, oauthService) + }, "failed to get OAuth Service") + + // Check if the OAuth service has the expected labels + Expect(oauthService.Labels["app"]).Should(Equal(instance.Name)) + Expect(oauthService.Labels["app.kubernetes.io/instance"]).Should(Equal(instance.Name)) + Expect(oauthService.Labels["app.kubernetes.io/name"]).Should(Equal(instance.Name)) + Expect(oauthService.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) + Expect(oauthService.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) + Expect(oauthService.Labels["trustyai-service-name"]).Should(Equal(instance.Name)) + +} + var _ = Describe("TrustyAI operator", func() { BeforeEach(func() { @@ -59,173 +350,153 @@ var _ = Describe("TrustyAI operator", func() { Context("When deploying with default settings without an InferenceService", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService + It("Creates a deployment and a service with the default configuration in PVC-mode", func() { + namespace := "trusty-ns-a-1-pvc" + instance = createDefaultPVCCustomResource(namespace) + setupAndTestDeploymentDefault(instance, namespace) + }) + It("Creates a deployment and a service with the default configuration in DB-mode (mysql)", func() { + namespace := "trusty-ns-a-1-db" + instance = createDefaultDBCustomResource(namespace) + WaitFor(func() error { + secret := createDatabaseConfiguration(namespace, defaultDatabaseConfigurationName, "mysql") + return k8sClient.Create(ctx, secret) + }, "failed to create ConfigMap") + setupAndTestDeploymentDefault(instance, namespace) + }) + It("Creates a deployment and a service with the default configuration in DB-mode (mariadb)", func() { + namespace := "trusty-ns-a-1-db" + instance = createDefaultDBCustomResource(namespace) + WaitFor(func() error { + secret := createDatabaseConfiguration(namespace, defaultDatabaseConfigurationName, "mariadb") + return k8sClient.Create(ctx, secret) + }, "failed to create ConfigMap") + setupAndTestDeploymentDefault(instance, namespace) + }) + + }) - It("Creates a deployment and a service with the default configuration", func() { + Context("When deploying with a ConfigMap and without an InferenceService", func() { + var instance *trustyaiopendatahubiov1alpha1.TrustyAIService + + It("Creates a deployment and a service with the ConfigMap configuration in PVC-mode", func() { + namespace := "trusty-ns-a-1-cm-pvc" + instance = createDefaultPVCCustomResource(namespace) + setupAndTestDeploymentConfigMap(instance, namespace) + }) + It("Creates a deployment and a service with the ConfigMap configuration in DB-mode", func() { + namespace := "trusty-ns-a-1-cm-db" + instance = createDefaultDBCustomResource(namespace) + setupAndTestDeploymentConfigMap(instance, namespace) + }) + + }) - namespace := "trusty-ns-a-1" + Context("When deploying with default settings without an InferenceService", func() { + var instance *trustyaiopendatahubiov1alpha1.TrustyAIService + + It("should set environment variables correctly in PVC mode", func() { + + namespace := "trusty-ns-a-4-pvc" + instance = createDefaultPVCCustomResource(namespace) + //printKubeObject(instance) Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) - instance = createDefaultCR(namespace) caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) - Expect(reconciler.ensureDeployment(ctx, instance, caBundle)).To(Succeed()) + Expect(reconciler.ensureDeployment(ctx, instance, caBundle, false)).To(Succeed()) deployment := &appsv1.Deployment{} - err := k8sClient.Get(ctx, types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace}, deployment) - Expect(err).ToNot(HaveOccurred()) - Expect(deployment).ToNot(BeNil()) - - Expect(*deployment.Spec.Replicas).Should(Equal(int32(1))) - Expect(deployment.Namespace).Should(Equal(namespace)) - Expect(deployment.Name).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/name"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/instance"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) - Expect(deployment.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) - - Expect(len(deployment.Spec.Template.Spec.Containers)).Should(Equal(2)) - Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal("quay.io/trustyai/trustyai-service:latest")) - Expect(deployment.Spec.Template.Spec.Containers[1].Image).Should(Equal("registry.redhat.io/openshift4/ose-oauth-proxy:latest")) - - WaitFor(func() error { - service, _ := reconciler.reconcileService(ctx, instance) - return reconciler.Create(ctx, service) - }, "failed to create service") - - service := &corev1.Service{} - WaitFor(func() error { - return k8sClient.Get(ctx, types.NamespacedName{Name: defaultServiceName, Namespace: namespace}, service) - }, "failed to get Service") - - Expect(service.Annotations["prometheus.io/path"]).Should(Equal("/q/metrics")) - Expect(service.Annotations["prometheus.io/scheme"]).Should(Equal("http")) - Expect(service.Annotations["prometheus.io/scrape"]).Should(Equal("true")) - Expect(service.Namespace).Should(Equal(namespace)) - - WaitFor(func() error { - err := reconciler.reconcileOAuthService(ctx, instance, caBundle) - return err - }, "failed to create oauth service") + namespacedName := types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace} + Expect(k8sClient.Get(ctx, namespacedName, deployment)).Should(Succeed()) - desiredOAuthService, err := generateTrustyAIOAuthService(ctx, instance, caBundle) - Expect(err).ToNot(HaveOccurred()) + //printKubeObject(deployment) - oauthService := &corev1.Service{} - WaitFor(func() error { - return k8sClient.Get(ctx, types.NamespacedName{Name: desiredOAuthService.Name, Namespace: namespace}, oauthService) - }, "failed to get OAuth Service") + foundEnvVar := func(envVars []corev1.EnvVar, name string) *corev1.EnvVar { + for _, env := range envVars { + if env.Name == name { + return &env + } + } + return nil + } - // Check if the OAuth service has the expected labels - Expect(oauthService.Labels["app"]).Should(Equal(instance.Name)) - Expect(oauthService.Labels["app.kubernetes.io/instance"]).Should(Equal(instance.Name)) - Expect(oauthService.Labels["app.kubernetes.io/name"]).Should(Equal(instance.Name)) - Expect(oauthService.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) - Expect(oauthService.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) - Expect(oauthService.Labels["trustyai-service-name"]).Should(Equal(instance.Name)) + var trustyaiServiceContainer *corev1.Container + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name == "trustyai-service" { + trustyaiServiceContainer = &container + break + } + } - }) - }) + Expect(trustyaiServiceContainer).NotTo(BeNil(), "trustyai-service container not found") - Context("When deploying with a ConfigMap and without an InferenceService", func() { - var instance *trustyaiopendatahubiov1alpha1.TrustyAIService + // Checking the environment variables of the trustyai-service container + var envVar *corev1.EnvVar - It("Creates a deployment and a service with the ConfigMap configuration", func() { + //envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_BATCH_SIZE") + //Expect(envVar).NotTo(BeNil(), "Env var SERVICE_BATCH_SIZE not found") + //Expect(envVar.Value).To(Equal("5000")) - namespace := "trusty-ns-a-1-cm" - serviceImage := "custom-service-image:foo" - oauthImage := "custom-oauth-proxy:bar" - Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) + envVar = foundEnvVar(trustyaiServiceContainer.Env, "STORAGE_DATA_FILENAME") + Expect(envVar).NotTo(BeNil(), "Env var STORAGE_DATA_FILENAME not found") + Expect(envVar.Value).To(Equal("data.csv")) - WaitFor(func() error { - configMap := createConfigMap(operatorNamespace, oauthImage, serviceImage) - return k8sClient.Create(ctx, configMap) - }, "failed to create ConfigMap") + envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_STORAGE_FORMAT") + Expect(envVar).NotTo(BeNil(), "Env var SERVICE_STORAGE_FORMAT not found") + Expect(envVar.Value).To(Equal("PVC")) - instance = createDefaultCR(namespace) + envVar = foundEnvVar(trustyaiServiceContainer.Env, "STORAGE_DATA_FOLDER") + Expect(envVar).NotTo(BeNil(), "Env var STORAGE_DATA_FOLDER not found") + Expect(envVar.Value).To(Equal("/data")) - caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_DATA_FORMAT") + Expect(envVar).NotTo(BeNil(), "Env var SERVICE_DATA_FORMAT not found") + Expect(envVar.Value).To(Equal("CSV")) - Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) - Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) - WaitFor(func() error { - return reconciler.ensureDeployment(ctx, instance, caBundle) - }, "failed to reconcile deployment") + envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_METRICS_SCHEDULE") + Expect(envVar).NotTo(BeNil(), "Env var SERVICE_METRICS_SCHEDULE not found") + Expect(envVar.Value).To(Equal("5s")) - deployment := &appsv1.Deployment{} - WaitFor(func() error { - return k8sClient.Get(ctx, types.NamespacedName{Name: instance.Name, Namespace: namespace}, deployment) - }, "failed to get updated deployment") - Expect(deployment).ToNot(BeNil()) - - Expect(*deployment.Spec.Replicas).Should(Equal(int32(1))) - Expect(deployment.Namespace).Should(Equal(namespace)) - Expect(deployment.Name).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/name"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/instance"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) - Expect(deployment.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) - - Expect(len(deployment.Spec.Template.Spec.Containers)).Should(Equal(2)) - Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal(serviceImage)) - Expect(deployment.Spec.Template.Spec.Containers[1].Image).Should(Equal(oauthImage)) + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_HIBERNATE_ORM_ACTIVE") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_HIBERNATE_ORM_ACTIVE not found") + Expect(envVar.Value).To(Equal("false")) - WaitFor(func() error { - service, _ := reconciler.reconcileService(ctx, instance) - return reconciler.Create(ctx, service) - }, "failed to create service") + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_DB_KIND") + Expect(envVar).To(BeNil()) - service := &corev1.Service{} - WaitFor(func() error { - return k8sClient.Get(ctx, types.NamespacedName{Name: defaultServiceName, Namespace: namespace}, service) - }, "failed to get Service") + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_JDBC_MAX_SIZE") + Expect(envVar).To(BeNil()) - Expect(service.Annotations["prometheus.io/path"]).Should(Equal("/q/metrics")) - Expect(service.Annotations["prometheus.io/scheme"]).Should(Equal("http")) - Expect(service.Annotations["prometheus.io/scrape"]).Should(Equal("true")) - Expect(service.Namespace).Should(Equal(namespace)) + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_USERNAME") + Expect(envVar).To(BeNil()) - WaitFor(func() error { - err := reconciler.reconcileOAuthService(ctx, instance, caBundle) - return err - }, "failed to create oauth service") + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_PASSWORD") + Expect(envVar).To(BeNil()) - desiredOAuthService, err := generateTrustyAIOAuthService(ctx, instance, caBundle) - Expect(err).ToNot(HaveOccurred()) + envVar = foundEnvVar(trustyaiServiceContainer.Env, "DATABASE_SERVICE") + Expect(envVar).To(BeNil()) - oauthService := &corev1.Service{} - WaitFor(func() error { - return k8sClient.Get(ctx, types.NamespacedName{Name: desiredOAuthService.Name, Namespace: namespace}, oauthService) - }, "failed to get OAuth Service") + envVar = foundEnvVar(trustyaiServiceContainer.Env, "DATABASE_PORT") + Expect(envVar).To(BeNil()) - // Check if the OAuth service has the expected labels - Expect(oauthService.Labels["app"]).Should(Equal(instance.Name)) - Expect(oauthService.Labels["app.kubernetes.io/instance"]).Should(Equal(instance.Name)) - Expect(oauthService.Labels["app.kubernetes.io/name"]).Should(Equal(instance.Name)) - Expect(oauthService.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) - Expect(oauthService.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) - Expect(oauthService.Labels["trustyai-service-name"]).Should(Equal(instance.Name)) + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_JDBC_URL") + Expect(envVar).To(BeNil()) }) - }) - Context("When deploying with default settings without an InferenceService", func() { - var instance *trustyaiopendatahubiov1alpha1.TrustyAIService - - It("should set environment variables correctly", func() { + It("should set environment variables correctly in DB mode", func() { - namespace := "trusty-ns-a-4" - instance = createDefaultCR(namespace) + namespace := "trusty-ns-a-4-db" + instance = createDefaultDBCustomResource(namespace) Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) - caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) - Expect(reconciler.ensureDeployment(ctx, instance, caBundle)).To(Succeed()) + Expect(reconciler.ensureDeployment(ctx, instance, caBundle, false)).To(Succeed()) deployment := &appsv1.Deployment{} namespacedName := types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace} @@ -253,135 +524,255 @@ var _ = Describe("TrustyAI operator", func() { // Checking the environment variables of the trustyai-service container var envVar *corev1.EnvVar - envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_BATCH_SIZE") - Expect(envVar).NotTo(BeNil(), "Env var SERVICE_BATCH_SIZE not found") - Expect(envVar.Value).To(Equal("5000")) - envVar = foundEnvVar(trustyaiServiceContainer.Env, "STORAGE_DATA_FILENAME") - Expect(envVar).NotTo(BeNil(), "Env var STORAGE_DATA_FILENAME not found") - Expect(envVar.Value).To(Equal("data.csv")) + Expect(envVar).To(BeNil()) envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_STORAGE_FORMAT") Expect(envVar).NotTo(BeNil(), "Env var SERVICE_STORAGE_FORMAT not found") - Expect(envVar.Value).To(Equal("PVC")) + Expect(envVar.Value).To(Equal(STORAGE_DATABASE)) envVar = foundEnvVar(trustyaiServiceContainer.Env, "STORAGE_DATA_FOLDER") - Expect(envVar).NotTo(BeNil(), "Env var STORAGE_DATA_FOLDER not found") - Expect(envVar.Value).To(Equal("/data")) + Expect(envVar).To(BeNil()) - envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_DATA_FORMAT") - Expect(envVar).NotTo(BeNil(), "Env var SERVICE_DATA_FORMAT not found") - Expect(envVar.Value).To(Equal("CSV")) + //envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_DATA_FORMAT") + //Expect(envVar).To(BeNil()) envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_METRICS_SCHEDULE") Expect(envVar).NotTo(BeNil(), "Env var SERVICE_METRICS_SCHEDULE not found") Expect(envVar.Value).To(Equal("5s")) - }) - }) - Context("When deploying with no custom CA bundle ConfigMap", func() { - var instance *trustyaiopendatahubiov1alpha1.TrustyAIService + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_HIBERNATE_ORM_ACTIVE") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_HIBERNATE_ORM_ACTIVE not found") + Expect(envVar.Value).To(Equal("true")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_DB_KIND") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_DB_KIND not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_DB_KIND does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_DB_KIND is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databaseKind"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_JDBC_MAX_SIZE") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_JDBC_MAX_SIZE not found") + Expect(envVar.Value).To(Equal("16")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_USERNAME") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_USERNAME not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_USERNAME does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_USERNAME is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databaseUsername"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_PASSWORD") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_PASSWORD not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_PASSWORD does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_PASSWORD is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databasePassword"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "DATABASE_SERVICE") + Expect(envVar).NotTo(BeNil(), "Env var DATABASE_SERVICE not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var DATABASE_SERVICE does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var DATABASE_SERVICE is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databaseService"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "DATABASE_PORT") + Expect(envVar).NotTo(BeNil(), "Env var DATABASE_PORT not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var DATABASE_PORT does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var DATABASE_PORT is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databasePort"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_JDBC_URL") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_JDBC_URL not found") + Expect(envVar.Value).To(Equal("jdbc:${QUARKUS_DATASOURCE_DB_KIND}://${DATABASE_SERVICE}:${DATABASE_PORT}/trustyai_database")) + + }) - It("should use the correct service account and not include CustomCertificatesBundle", func() { + It("should set environment variables correctly in migration mode", func() { - namespace := "trusty-ns-a-7" - instance = createDefaultCR(namespace) + namespace := "trusty-ns-a-4-migration" + instance = createDefaultMigrationCustomResource(namespace) Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) - + //printKubeObject(instance) caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) - WaitFor(func() error { - return reconciler.ensureDeployment(ctx, instance, caBundle) - }, "failed to create deployment") + Expect(reconciler.ensureDeployment(ctx, instance, caBundle, false)).To(Succeed()) deployment := &appsv1.Deployment{} namespacedName := types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace} Expect(k8sClient.Get(ctx, namespacedName, deployment)).Should(Succeed()) - Expect(deployment.Spec.Template.Spec.ServiceAccountName).To(Equal(instance.Name + "-proxy")) - - customCertificatesBundleVolumeName := caBundle - for _, volume := range deployment.Spec.Template.Spec.Volumes { - Expect(volume.Name).ToNot(Equal(customCertificatesBundleVolumeName)) + foundEnvVar := func(envVars []corev1.EnvVar, name string) *corev1.EnvVar { + for _, env := range envVars { + if env.Name == name { + return &env + } + } + return nil } + var trustyaiServiceContainer *corev1.Container for _, container := range deployment.Spec.Template.Spec.Containers { - for _, volumeMount := range container.VolumeMounts { - Expect(volumeMount.Name).ToNot(Equal(customCertificatesBundleVolumeName)) - } - for _, arg := range container.Args { - Expect(arg).ToNot(ContainSubstring("--openshift-ca")) + if container.Name == "trustyai-service" { + trustyaiServiceContainer = &container + break } } + + Expect(trustyaiServiceContainer).NotTo(BeNil(), "trustyai-service container not found") + + // Checking the environment variables of the trustyai-service container + var envVar *corev1.EnvVar + + //envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_BATCH_SIZE") + //Expect(envVar).To(BeNil(), "Env var SERVICE_BATCH_SIZE not found") + //Expect(envVar.Value).To(Equal("5000")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "STORAGE_DATA_FILENAME") + Expect(envVar).ToNot(BeNil()) + Expect(envVar.Value).To(Equal("data.csv")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_STORAGE_FORMAT") + Expect(envVar).NotTo(BeNil(), "Env var SERVICE_STORAGE_FORMAT not found") + Expect(envVar.Value).To(Equal(STORAGE_DATABASE)) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "STORAGE_DATA_FOLDER") + Expect(envVar).ToNot(BeNil()) + Expect(envVar.Value).To(Equal("/data")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_DATA_FORMAT") + Expect(envVar).ToNot(BeNil()) + Expect(envVar.Value).To(Equal("CSV")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_METRICS_SCHEDULE") + Expect(envVar).NotTo(BeNil(), "Env var SERVICE_METRICS_SCHEDULE not found") + Expect(envVar.Value).To(Equal("5s")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_HIBERNATE_ORM_ACTIVE") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_HIBERNATE_ORM_ACTIVE not found") + Expect(envVar.Value).To(Equal("true")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_DB_KIND") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_DB_KIND not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_DB_KIND does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_DB_KIND is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databaseKind"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_JDBC_MAX_SIZE") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_JDBC_MAX_SIZE not found") + Expect(envVar.Value).To(Equal("16")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_USERNAME") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_USERNAME not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_USERNAME does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_USERNAME is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databaseUsername"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_PASSWORD") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_PASSWORD not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_PASSWORD does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_PASSWORD is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databasePassword"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "DATABASE_SERVICE") + Expect(envVar).NotTo(BeNil(), "Env var DATABASE_SERVICE not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var DATABASE_SERVICE does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var DATABASE_SERVICE is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databaseService"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "DATABASE_PORT") + Expect(envVar).NotTo(BeNil(), "Env var DATABASE_PORT not found") + Expect(envVar.ValueFrom).NotTo(BeNil(), "Env var DATABASE_PORT does not have ValueFrom set") + Expect(envVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "Env var DATABASE_PORT is not using SecretKeyRef") + Expect(envVar.ValueFrom.SecretKeyRef.Name).To(Equal(defaultDatabaseConfigurationName), "Secret name does not match") + Expect(envVar.ValueFrom.SecretKeyRef.Key).To(Equal("databasePort"), "Secret key does not match") + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "QUARKUS_DATASOURCE_JDBC_URL") + Expect(envVar).NotTo(BeNil(), "Env var QUARKUS_DATASOURCE_JDBC_URL not found") + Expect(envVar.Value).To(Equal("jdbc:${QUARKUS_DATASOURCE_DB_KIND}://${DATABASE_SERVICE}:${DATABASE_PORT}/trustyai_database")) + }) + }) - Context("When deploying with a custom CA bundle ConfigMap", func() { + Context("When deploying with no custom CA bundle ConfigMap", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService - It("should use the correct service account and include CustomCertificatesBundle", func() { + It("should use the correct service account and not include CustomCertificatesBundle in PVC-mode", func() { - namespace := "trusty-ns-a-8" - instance = createDefaultCR(namespace) - caBundleConfigMap := createTrustedCABundleConfigMap(namespace) - Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) - Expect(k8sClient.Create(ctx, caBundleConfigMap)).To(Succeed()) + namespace := "trusty-ns-a-7-pvc" + instance = createDefaultPVCCustomResource(namespace) + setupAndTestDeploymentNoCustomCABundle(instance, namespace) + }) + It("should use the correct service account and not include CustomCertificatesBundle in DB-mode", func() { - caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + namespace := "trusty-ns-a-7-db" + instance = createDefaultDBCustomResource(namespace) + setupAndTestDeploymentNoCustomCABundle(instance, namespace) + }) + It("should use the correct service account and not include CustomCertificatesBundle in migration-mode", func() { - Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) - Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) - WaitFor(func() error { - return reconciler.ensureDeployment(ctx, instance, caBundle) - }, "failed to create deployment") + namespace := "trusty-ns-a-7-migration" + instance = createDefaultMigrationCustomResource(namespace) + setupAndTestDeploymentNoCustomCABundle(instance, namespace) + }) - deployment := &appsv1.Deployment{} - namespacedName := types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace} - Expect(k8sClient.Get(ctx, namespacedName, deployment)).Should(Succeed()) + }) - Expect(deployment.Spec.Template.Spec.ServiceAccountName).To(Equal(instance.Name + "-proxy")) + Context("When deploying with a custom CA bundle ConfigMap", func() { + var instance *trustyaiopendatahubiov1alpha1.TrustyAIService - foundCustomCertificatesBundleVolumeMount := false + It("should use the correct service account and include CustomCertificatesBundle in PVC-mode", func() { - customCertificatesBundleMountPath := "/etc/ssl/certs/ca-bundle.crt" - for _, container := range deployment.Spec.Template.Spec.Containers { - for _, volumeMount := range container.VolumeMounts { - if volumeMount.Name == caBundleName && volumeMount.MountPath == customCertificatesBundleMountPath { - foundCustomCertificatesBundleVolumeMount = true - } - } - } - Expect(foundCustomCertificatesBundleVolumeMount).To(BeTrue(), caBundleName+" volume mount not found in any container") + namespace := "trusty-ns-a-8-pvc" + instance = createDefaultPVCCustomResource(namespace) + setupAndTestDeploymentCustomCABundle(instance, namespace) + }) + It("should use the correct service account and include CustomCertificatesBundle in DB-mode", func() { - Expect(k8sClient.Delete(ctx, caBundleConfigMap)).To(Succeed(), "failed to delete custom certificates bundle ConfigMap") + namespace := "trusty-ns-a-8-db" + instance = createDefaultDBCustomResource(namespace) + setupAndTestDeploymentCustomCABundle(instance, namespace) + }) + It("should use the correct service account and include CustomCertificatesBundle in migration-mode", func() { + namespace := "trusty-ns-a-8-migration" + instance = createDefaultMigrationCustomResource(namespace) + setupAndTestDeploymentCustomCABundle(instance, namespace) }) }) Context("When deploying with default settings without an InferenceService", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService - It("should use the correct service account", func() { + It("should use the correct service account in PVC-mode", func() { - namespace := "trusty-ns-a-6" - instance = createDefaultCR(namespace) - Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) + namespace := "trusty-ns-a-6-pvc" + instance = createDefaultPVCCustomResource(namespace) + setupAndTestDeploymentServiceAccount(instance, namespace, "PVC") - caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + }) + It("should use the correct service account in DB-mode", func() { - Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) - Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) - WaitFor(func() error { - return reconciler.ensureDeployment(ctx, instance, caBundle) - }, "failed to create deployment") + namespace := "trusty-ns-a-6-db" + instance = createDefaultDBCustomResource(namespace) + setupAndTestDeploymentServiceAccount(instance, namespace, "DATABASE") - deployment := &appsv1.Deployment{} - namespacedName := types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace} - Expect(k8sClient.Get(ctx, namespacedName, deployment)).Should(Succeed()) + }) + It("should use the correct service account in migration-mode", func() { + + namespace := "trusty-ns-a-6-migration" + instance = createDefaultMigrationCustomResource(namespace) + setupAndTestDeploymentServiceAccount(instance, namespace, "DATABASE") - Expect(deployment.Spec.Template.Spec.ServiceAccountName).To(Equal(instance.Name + "-proxy")) }) }) @@ -402,72 +793,28 @@ var _ = Describe("TrustyAI operator", func() { Context("When deploying with an associated InferenceService", func() { - It("Sets up the InferenceService and links it to the TrustyAIService deployment", func() { + It("Sets up the InferenceService and links it to the TrustyAIService deployment in PVC-mode", func() { - namespace := "trusty-ns-2" - instance := createDefaultCR(namespace) - WaitFor(func() error { - return createNamespace(ctx, k8sClient, namespace) - }, "failed to create namespace") + namespace := "trusty-ns-2-pvc" + instance := createDefaultPVCCustomResource(namespace) + setupAndTestDeploymentInferenceService(instance, namespace, "PVC") - caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) - - WaitFor(func() error { - return createTestPVC(ctx, k8sClient, instance) - }, "failed to create PVC") - WaitFor(func() error { - return reconciler.ensureDeployment(ctx, instance, caBundle) - }, "failed to create deployment") - - // Creating the InferenceService - inferenceService := createInferenceService("my-model", namespace) - WaitFor(func() error { - return k8sClient.Create(ctx, inferenceService) - }, "failed to create deployment") - - Expect(reconciler.patchKServe(ctx, instance, *inferenceService, namespace, instance.Name, false)).ToNot(HaveOccurred()) - - deployment := &appsv1.Deployment{} - WaitFor(func() error { - // Define defaultServiceName for the deployment created by the operator - namespacedNamed := types.NamespacedName{ - Namespace: namespace, - Name: instance.Name, - } - return k8sClient.Get(ctx, namespacedNamed, deployment) - }, "failed to get Deployment") - - Expect(*deployment.Spec.Replicas).Should(Equal(int32(1))) - Expect(deployment.Namespace).Should(Equal(namespace)) - Expect(deployment.Name).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/name"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/instance"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) - Expect(deployment.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) - - WaitFor(func() error { - err := reconciler.reconcileOAuthService(ctx, instance, caBundle) - return err - }, "failed to create oauth service") + }) + It("Sets up the InferenceService and links it to the TrustyAIService deployment in DB-mode", func() { - desiredOAuthService, err := generateTrustyAIOAuthService(ctx, instance, caBundle) - Expect(err).ToNot(HaveOccurred()) + namespace := "trusty-ns-2-db" + instance := createDefaultDBCustomResource(namespace) + setupAndTestDeploymentInferenceService(instance, namespace, "DATABASE") - oauthService := &corev1.Service{} - WaitFor(func() error { - return k8sClient.Get(ctx, types.NamespacedName{Name: desiredOAuthService.Name, Namespace: namespace}, oauthService) - }, "failed to get OAuth Service") + }) + It("Sets up the InferenceService and links it to the TrustyAIService deployment in migration-mode", func() { - // Check if the OAuth service has the expected labels - Expect(oauthService.Labels["app"]).Should(Equal(instance.Name)) - Expect(oauthService.Labels["app.kubernetes.io/instance"]).Should(Equal(instance.Name)) - Expect(oauthService.Labels["app.kubernetes.io/name"]).Should(Equal(instance.Name)) - Expect(oauthService.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) - Expect(oauthService.Labels["app.kubernetes.io/version"]).Should(Equal(Version)) - Expect(oauthService.Labels["trustyai-service-name"]).Should(Equal(instance.Name)) + namespace := "trusty-ns-2-migration" + instance := createDefaultMigrationCustomResource(namespace) + setupAndTestDeploymentInferenceService(instance, namespace, "DATABASE") }) + }) }) @@ -493,7 +840,7 @@ var _ = Describe("TrustyAI operator", func() { It("Deploys services with defaults in each specified namespace", func() { for i, namespace := range namespaces { - instances[i] = createDefaultCR(namespace) + instances[i] = createDefaultPVCCustomResource(namespace) instances[i].Namespace = namespace WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) @@ -507,7 +854,7 @@ var _ = Describe("TrustyAI operator", func() { return createTestPVC(ctx, k8sClient, instance) }, "failed to create PVC") WaitFor(func() error { - return reconciler.ensureDeployment(ctx, instance, caBundle) + return reconciler.ensureDeployment(ctx, instance, caBundle, false) }, "failed to create deployment") //Expect(k8sClient.Create(ctx, instance)).Should(Succeed()) deployment := &appsv1.Deployment{} diff --git a/controllers/monitor_test.go b/controllers/monitor_test.go index 41cd8fd7..ce9bc9c9 100644 --- a/controllers/monitor_test.go +++ b/controllers/monitor_test.go @@ -60,7 +60,7 @@ var _ = Describe("Service Monitor Reconciliation", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService It("Should have correct values", func() { namespace := "sm-test-namespace-1" - instance = createDefaultCR(namespace) + instance = createDefaultPVCCustomResource(namespace) WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) diff --git a/controllers/route.go b/controllers/route.go index 6d5963e5..3016a321 100644 --- a/controllers/route.go +++ b/controllers/route.go @@ -100,6 +100,7 @@ func (r *TrustyAIServiceReconciler) checkRouteReady(ctx context.Context, cr *tru err := r.Client.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: cr.Namespace}, existingRoute) if err != nil { + log.FromContext(ctx).Info("Unable to find the Route") if errors.IsNotFound(err) { return false, nil } diff --git a/controllers/route_test.go b/controllers/route_test.go index b84f237b..da35a689 100644 --- a/controllers/route_test.go +++ b/controllers/route_test.go @@ -11,6 +11,42 @@ import ( "k8s.io/client-go/tools/record" ) +func setupAndTestRouteCreation(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, namespace string) { + WaitFor(func() error { + return createNamespace(ctx, k8sClient, namespace) + }, "failed to create namespace") + + err := reconciler.reconcileRouteAuth(instance, ctx, reconciler.createRouteObject) + Expect(err).ToNot(HaveOccurred()) + + route := &routev1.Route{} + err = reconciler.Client.Get(ctx, types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace}, route) + Expect(err).ToNot(HaveOccurred()) + Expect(route).ToNot(BeNil()) + Expect(route.Spec.To.Name).To(Equal(instance.Name + "-tls")) + +} + +func setupAndTestSameRouteCreation(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, namespace string) { + WaitFor(func() error { + return createNamespace(ctx, k8sClient, namespace) + }, "failed to create namespace") + + // Create a Route with the expected spec + existingRoute, err := reconciler.createRouteObject(ctx, instance) + Expect(err).ToNot(HaveOccurred()) + Expect(reconciler.Client.Create(ctx, existingRoute)).To(Succeed()) + + err = reconciler.reconcileRouteAuth(instance, ctx, reconciler.createRouteObject) + Expect(err).ToNot(HaveOccurred()) + + // Fetch the Route + route := &routev1.Route{} + err = reconciler.Client.Get(ctx, types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace}, route) + Expect(err).ToNot(HaveOccurred()) + Expect(route.Spec).To(Equal(existingRoute.Spec)) +} + var _ = Describe("Route Reconciliation", func() { BeforeEach(func() { @@ -25,47 +61,41 @@ var _ = Describe("Route Reconciliation", func() { Context("When Route does not exist", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService - It("Should create Route successfully", func() { - namespace := "route-test-namespace-1" - instance = createDefaultCR(namespace) - - WaitFor(func() error { - return createNamespace(ctx, k8sClient, namespace) - }, "failed to create namespace") - - err := reconciler.reconcileRouteAuth(instance, ctx, reconciler.createRouteObject) - Expect(err).ToNot(HaveOccurred()) - - route := &routev1.Route{} - err = reconciler.Client.Get(ctx, types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace}, route) - Expect(err).ToNot(HaveOccurred()) - Expect(route).ToNot(BeNil()) - Expect(route.Spec.To.Name).To(Equal(instance.Name + "-tls")) + It("Should create Route successfully in PVC-mode", func() { + namespace := "route-test-namespace-1-pvc" + instance = createDefaultPVCCustomResource(namespace) + setupAndTestRouteCreation(instance, namespace) + }) + It("Should create Route successfully in DB-mode", func() { + namespace := "route-test-namespace-1-db" + instance = createDefaultDBCustomResource(namespace) + setupAndTestRouteCreation(instance, namespace) + }) + It("Should create Route successfully in migration-mode", func() { + namespace := "route-test-namespace-1-migration" + instance = createDefaultMigrationCustomResource(namespace) + setupAndTestRouteCreation(instance, namespace) }) + }) Context("When Route exists and is the same", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService - It("Should not update Route", func() { - namespace := "route-test-namespace-2" - instance = createDefaultCR(namespace) - WaitFor(func() error { - return createNamespace(ctx, k8sClient, namespace) - }, "failed to create namespace") - - // Create a Route with the expected spec - existingRoute, err := reconciler.createRouteObject(ctx, instance) - Expect(err).ToNot(HaveOccurred()) - Expect(reconciler.Client.Create(ctx, existingRoute)).To(Succeed()) - - err = reconciler.reconcileRouteAuth(instance, ctx, reconciler.createRouteObject) - Expect(err).ToNot(HaveOccurred()) - - // Fetch the Route - route := &routev1.Route{} - err = reconciler.Client.Get(ctx, types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace}, route) - Expect(err).ToNot(HaveOccurred()) - Expect(route.Spec).To(Equal(existingRoute.Spec)) + It("Should not update Route in PVC-mode", func() { + namespace := "route-test-namespace-2-pvc" + instance = createDefaultPVCCustomResource(namespace) + setupAndTestSameRouteCreation(instance, namespace) + }) + It("Should not update Route in DB-mode", func() { + namespace := "route-test-namespace-2-db" + instance = createDefaultDBCustomResource(namespace) + setupAndTestSameRouteCreation(instance, namespace) }) + It("Should not update Route in migration-mode", func() { + namespace := "route-test-namespace-2-migration" + instance = createDefaultMigrationCustomResource(namespace) + setupAndTestSameRouteCreation(instance, namespace) + }) + }) }) diff --git a/controllers/secrets.go b/controllers/secrets.go new file mode 100644 index 00000000..6e2d49b7 --- /dev/null +++ b/controllers/secrets.go @@ -0,0 +1,60 @@ +package controllers + +import ( + "context" + "fmt" + trustyaiopendatahubiov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// findDatabaseSecret finds the DB configuration secret named (specified or default) in the same namespace as the CR +func (r *TrustyAIServiceReconciler) findDatabaseSecret(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService) (*corev1.Secret, error) { + + databaseConfigurationsName := instance.Spec.Storage.DatabaseConfigurations + defaultDatabaseConfigurationsName := instance.Name + dbCredentialsSuffix + + secret := &corev1.Secret{} + + if databaseConfigurationsName != "" { + secret := &corev1.Secret{} + err := r.Get(ctx, client.ObjectKey{Name: databaseConfigurationsName, Namespace: instance.Namespace}, secret) + if err == nil { + return secret, nil + } + if !errors.IsNotFound(err) { + return nil, fmt.Errorf("failed to get secret %s in namespace %s: %w", databaseConfigurationsName, instance.Namespace, err) + } + } else { + // If specified not found, try the default + + err := r.Get(ctx, client.ObjectKey{Name: defaultDatabaseConfigurationsName, Namespace: instance.Namespace}, secret) + if err == nil { + return secret, nil + } + if !errors.IsNotFound(err) { + return nil, fmt.Errorf("failed to get secret %s in namespace %s: %w", defaultDatabaseConfigurationsName, instance.Namespace, err) + } + + } + + return nil, fmt.Errorf("neither secret %s nor %s found in namespace %s", databaseConfigurationsName, defaultDatabaseConfigurationsName, instance.Namespace) +} + +// validateDatabaseSecret validates the DB configuration secret +func (r *TrustyAIServiceReconciler) validateDatabaseSecret(secret *corev1.Secret) error { + + mandatoryKeys := []string{"databaseKind", "databaseUsername", "databasePassword", "databaseService", "databasePort"} + + for _, key := range mandatoryKeys { + value, exists := secret.Data[key] + if !exists { + return fmt.Errorf("mandatory key %s is missing from database configuration", key) + } + if len(value) == 0 { + return fmt.Errorf("mandatory key %s is empty on database configuration", key) + } + } + return nil +} diff --git a/controllers/service_accounts_test.go b/controllers/service_accounts_test.go index 4b64837d..17b9d660 100644 --- a/controllers/service_accounts_test.go +++ b/controllers/service_accounts_test.go @@ -26,7 +26,7 @@ var _ = Describe("Service Accounts", func() { It("Should create SAs, CRBs successfully", func() { namespace1 := "sa-test-namespace-1" - instance1 := createDefaultCR(namespace1) + instance1 := createDefaultPVCCustomResource(namespace1) WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace1) @@ -36,7 +36,7 @@ var _ = Describe("Service Accounts", func() { namespace2 := "sa-test-namespace-2" - instance2 := createDefaultCR(namespace2) + instance2 := createDefaultPVCCustomResource(namespace2) WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace2) diff --git a/controllers/statuses.go b/controllers/statuses.go index b54036ac..b88ba47c 100644 --- a/controllers/statuses.go +++ b/controllers/statuses.go @@ -2,6 +2,7 @@ package controllers import ( "context" + trustyaiopendatahubiov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/v1alpha1" v1 "k8s.io/api/core/v1" "k8s.io/client-go/util/retry" @@ -10,12 +11,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ) -// IsAllReady checks if all the necessary readiness fields are true. -func (rs *AvailabilityStatus) IsAllReady() bool { - return rs.PVCReady && rs.DeploymentReady && rs.RouteReady +// IsAllReady checks if all the necessary readiness fields are true for the specific mode +func (rs *AvailabilityStatus) IsAllReady(mode string) bool { + return (rs.PVCReady && rs.DeploymentReady && rs.RouteReady && mode == STORAGE_PVC) || (rs.DeploymentReady && rs.RouteReady && mode == STORAGE_DATABASE) } -// AvailabilityStatus holds the readiness status of various resources. +// AvailabilityStatus has the readiness status of various resources. type AvailabilityStatus struct { PVCReady bool DeploymentReady bool @@ -44,37 +45,39 @@ func (r *TrustyAIServiceReconciler) updateStatus(ctx context.Context, original * return saved, err } -// reconcileStatuses checks the readiness status of PVC, Deployment, Route and Inference Services +// reconcileStatuses checks the readiness status of required resources func (r *TrustyAIServiceReconciler) reconcileStatuses(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService) (ctrl.Result, error) { var err error status := AvailabilityStatus{} - // Check for PVC readiness - status.PVCReady, err = r.checkPVCReady(ctx, instance) - if err != nil || !status.PVCReady { - // PVC not ready, requeue - return Requeue() + if instance.Spec.Storage.IsStoragePVC() || instance.IsMigration() { + // Check for PVC readiness + status.PVCReady, err = r.checkPVCReady(ctx, instance) + if err != nil || !status.PVCReady { + // PVC not ready, requeue + return RequeueWithDelayMessage(ctx, defaultRequeueDelay, "PVC not ready") + } } // Check for deployment readiness status.DeploymentReady, err = r.checkDeploymentReady(ctx, instance) if err != nil || !status.DeploymentReady { // Deployment not ready, requeue - return Requeue() + return RequeueWithDelayMessage(ctx, defaultRequeueDelay, "Deployment not ready") } // Check for route readiness status.RouteReady, err = r.checkRouteReady(ctx, instance) if err != nil || !status.RouteReady { // Route not ready, requeue - return Requeue() + return RequeueWithDelayMessage(ctx, defaultRequeueDelay, "Route not ready") } // Check if InferenceServices present status.InferenceServiceReady, err = r.checkInferenceServicesPresent(ctx, instance.Namespace) // All checks passed, resources are ready - if status.IsAllReady() { + if status.IsAllReady(instance.Spec.Storage.Format) { _, updateErr := r.updateStatus(ctx, instance, func(saved *trustyaiopendatahubiov1alpha1.TrustyAIService) { if status.InferenceServiceReady { @@ -83,7 +86,9 @@ func (r *TrustyAIServiceReconciler) reconcileStatuses(ctx context.Context, insta UpdateInferenceServiceNotPresent(saved) } - UpdatePVCAvailable(saved) + if instance.Spec.Storage.IsStoragePVC() || instance.IsMigration() { + UpdatePVCAvailable(saved) + } UpdateRouteAvailable(saved) UpdateTrustyAIServiceAvailable(saved) saved.Status.Phase = "Ready" @@ -101,11 +106,14 @@ func (r *TrustyAIServiceReconciler) reconcileStatuses(ctx context.Context, insta UpdateInferenceServiceNotPresent(saved) } - if status.PVCReady { - UpdatePVCAvailable(saved) - } else { - UpdatePVCNotAvailable(saved) + if instance.Spec.Storage.IsStoragePVC() || instance.IsMigration() { + if status.PVCReady { + UpdatePVCAvailable(saved) + } else { + UpdatePVCNotAvailable(saved) + } } + if status.RouteReady { UpdateRouteAvailable(saved) } else { diff --git a/controllers/statuses_test.go b/controllers/statuses_test.go index 6d110816..9e395df4 100644 --- a/controllers/statuses_test.go +++ b/controllers/statuses_test.go @@ -3,6 +3,7 @@ package controllers import ( "context" "fmt" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" trustyaiopendatahubiov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/v1alpha1" @@ -26,6 +27,27 @@ func checkCondition(conditions []trustyaiopendatahubiov1alpha1.Condition, condit return nil, false, fmt.Errorf("%s condition not found", conditionType) } +func setupAndTestStatusNoComponent(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, namespace string) { + WaitFor(func() error { + return createNamespace(ctx, k8sClient, namespace) + }, "failed to create namespace") + + // Call the reconcileStatuses function + _, _ = reconciler.reconcileStatuses(ctx, instance) + + readyCondition, statusMatch, err := checkCondition(instance.Status.Conditions, "Ready", corev1.ConditionTrue, true) + Expect(err).NotTo(HaveOccurred(), "Error checking Ready condition") + if readyCondition != nil { + Expect(statusMatch).To(Equal(corev1.ConditionFalse), "Ready condition should be true") + } + + availableCondition, statusMatch, err := checkCondition(instance.Status.Conditions, StatusTypeAvailable, corev1.ConditionFalse, true) + Expect(err).NotTo(HaveOccurred(), "Error checking Available condition") + if availableCondition != nil { + Expect(statusMatch).To(Equal(corev1.ConditionFalse), "Available condition should be false") + } +} + var _ = Describe("Status and condition tests", func() { BeforeEach(func() { @@ -41,37 +63,163 @@ var _ = Describe("Status and condition tests", func() { Context("When no component exists", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService - It("Should not be available", func() { - namespace := "statuses-test-namespace-1" - instance = createDefaultCR(namespace) + It("Should not be available in PVC-mode", func() { + namespace := "statuses-test-namespace-1-pvc" + instance = createDefaultPVCCustomResource(namespace) + setupAndTestStatusNoComponent(instance, namespace) + }) + It("Should not be available in DB-mode", func() { + namespace := "statuses-test-namespace-1-db" + instance = createDefaultDBCustomResource(namespace) + setupAndTestStatusNoComponent(instance, namespace) + }) + It("Should not be available in migration-mode", func() { + namespace := "statuses-test-namespace-1-migration" + instance = createDefaultMigrationCustomResource(namespace) + setupAndTestStatusNoComponent(instance, namespace) + }) + + }) + Context("When route, deployment and PVC component, but not inference service, exist", func() { + var instance *trustyaiopendatahubiov1alpha1.TrustyAIService + It("Should be available in PVC-mode", func() { + namespace := "statuses-test-namespace-2-pvc" + instance = createDefaultPVCCustomResource(namespace) WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) }, "failed to create namespace") + caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) + + WaitFor(func() error { + return reconciler.reconcileRouteAuth(instance, ctx, reconciler.createRouteObject) + }, "failed to create route") + WaitFor(func() error { + return makeRouteReady(ctx, k8sClient, instance) + }, "failed to make route ready") + WaitFor(func() error { + return reconciler.ensurePVC(ctx, instance) + }, "failed to create PVC") + WaitFor(func() error { + return makePVCReady(ctx, k8sClient, instance) + }, "failed to bind PVC") + WaitFor(func() error { + return reconciler.ensureDeployment(ctx, instance, caBundle, false) + }, "failed to create deployment") + WaitFor(func() error { + return makeDeploymentReady(ctx, k8sClient, instance) + }, "failed to make deployment ready") + WaitFor(func() error { + return k8sClient.Create(ctx, instance) + }, "failed to create TrustyAIService") // Call the reconcileStatuses function - _, _ = reconciler.reconcileStatuses(ctx, instance) + WaitFor(func() error { + _, err := reconciler.reconcileStatuses(ctx, instance) + return err + }, "failed to update statuses") + + // Fetch the updated instance + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + }, instance) + }, "failed to get updated instance") readyCondition, statusMatch, err := checkCondition(instance.Status.Conditions, "Ready", corev1.ConditionTrue, true) Expect(err).NotTo(HaveOccurred(), "Error checking Ready condition") if readyCondition != nil { - Expect(statusMatch).To(Equal(corev1.ConditionFalse), "Ready condition should be true") + Expect(statusMatch).To(Equal(corev1.ConditionTrue), "Ready condition should be true") } - availableCondition, statusMatch, err := checkCondition(instance.Status.Conditions, StatusTypeAvailable, corev1.ConditionFalse, true) + availableCondition, statusMatch, err := checkCondition(instance.Status.Conditions, StatusTypeAvailable, corev1.ConditionTrue, false) Expect(err).NotTo(HaveOccurred(), "Error checking Available condition") - if availableCondition != nil { - Expect(statusMatch).To(Equal(corev1.ConditionFalse), "Available condition should be false") - } + Expect(availableCondition).NotTo(BeNil(), "Available condition should not be null") + Expect(statusMatch).To(Equal(true), "Ready condition should be true") + + routeAvailableCondition, statusMatch, err := checkCondition(instance.Status.Conditions, StatusTypeRouteAvailable, corev1.ConditionTrue, false) + Expect(err).NotTo(HaveOccurred(), "Error checking RouteAvailable condition") + Expect(routeAvailableCondition).NotTo(BeNil(), "RouteAvailable condition should not be null") + Expect(statusMatch).To(Equal(true), "RouteAvailable condition should be true") + + pvcAvailableCondition, statusMatch, err := checkCondition(instance.Status.Conditions, StatusTypePVCAvailable, corev1.ConditionTrue, false) + Expect(err).NotTo(HaveOccurred(), "Error checking PVCAvailable condition") + Expect(pvcAvailableCondition).NotTo(BeNil(), "PVCAvailable condition should not be null") + Expect(statusMatch).To(Equal(true), "PVCAvailable condition should be true") + ISPresentCondition, statusMatch, err := checkCondition(instance.Status.Conditions, StatusTypeInferenceServicesPresent, corev1.ConditionFalse, false) + Expect(err).NotTo(HaveOccurred(), "Error checking InferenceServicePresent condition") + Expect(ISPresentCondition).NotTo(BeNil(), "InferenceServicePresent condition should not be null") + Expect(statusMatch).To(Equal(true), "InferenceServicePresent condition should be false") }) - }) + It("Should be available in DB-mode", func() { + namespace := "statuses-test-namespace-2-db" + instance = createDefaultDBCustomResource(namespace) + WaitFor(func() error { + return createNamespace(ctx, k8sClient, namespace) + }, "failed to create namespace") + caBundle := reconciler.GetCustomCertificatesBundle(ctx, instance) - Context("When route, deployment and PVC component, but not inference service, exist", func() { - var instance *trustyaiopendatahubiov1alpha1.TrustyAIService - It("Should be available", func() { - namespace := "statuses-test-namespace-2" - instance = createDefaultCR(namespace) + WaitFor(func() error { + return reconciler.reconcileRouteAuth(instance, ctx, reconciler.createRouteObject) + }, "failed to create route") + WaitFor(func() error { + return makeRouteReady(ctx, k8sClient, instance) + }, "failed to make route ready") + WaitFor(func() error { + return reconciler.ensureDeployment(ctx, instance, caBundle, false) + }, "failed to create deployment") + WaitFor(func() error { + return makeDeploymentReady(ctx, k8sClient, instance) + }, "failed to make deployment ready") + WaitFor(func() error { + return k8sClient.Create(ctx, instance) + }, "failed to create TrustyAIService") + + // Call the reconcileStatuses function + WaitFor(func() error { + _, err := reconciler.reconcileStatuses(ctx, instance) + return err + }, "failed to update statuses") + + // Fetch the updated instance + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + }, instance) + }, "failed to get updated instance") + + readyCondition, statusMatch, err := checkCondition(instance.Status.Conditions, "Ready", corev1.ConditionTrue, true) + Expect(err).NotTo(HaveOccurred(), "Error checking Ready condition") + if readyCondition != nil { + Expect(statusMatch).To(Equal(corev1.ConditionTrue), "Ready condition should be true") + } + + availableCondition, statusMatch, err := checkCondition(instance.Status.Conditions, StatusTypeAvailable, corev1.ConditionTrue, false) + Expect(err).NotTo(HaveOccurred(), "Error checking Available condition") + Expect(availableCondition).NotTo(BeNil(), "Available condition should not be null") + Expect(statusMatch).To(Equal(true), "Ready condition should be true") + + routeAvailableCondition, statusMatch, err := checkCondition(instance.Status.Conditions, StatusTypeRouteAvailable, corev1.ConditionTrue, false) + Expect(err).NotTo(HaveOccurred(), "Error checking RouteAvailable condition") + Expect(routeAvailableCondition).NotTo(BeNil(), "RouteAvailable condition should not be null") + Expect(statusMatch).To(Equal(true), "RouteAvailable condition should be true") + + pvcAvailableCondition, _, err := checkCondition(instance.Status.Conditions, StatusTypePVCAvailable, corev1.ConditionTrue, false) + Expect(err).To(HaveOccurred(), "Error checking PVCAvailable condition") + Expect(pvcAvailableCondition).To(BeNil(), "PVCAvailable condition should be null") + + ISPresentCondition, statusMatch, err := checkCondition(instance.Status.Conditions, StatusTypeInferenceServicesPresent, corev1.ConditionFalse, false) + Expect(err).NotTo(HaveOccurred(), "Error checking InferenceServicePresent condition") + Expect(ISPresentCondition).NotTo(BeNil(), "InferenceServicePresent condition should not be null") + Expect(statusMatch).To(Equal(true), "InferenceServicePresent condition should be false") + + }) + It("Should be available in migration-mode", func() { + namespace := "statuses-test-namespace-2-migration" + instance = createDefaultMigrationCustomResource(namespace) WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) }, "failed to create namespace") @@ -90,7 +238,7 @@ var _ = Describe("Status and condition tests", func() { return makePVCReady(ctx, k8sClient, instance) }, "failed to bind PVC") WaitFor(func() error { - return reconciler.ensureDeployment(ctx, instance, caBundle) + return reconciler.ensureDeployment(ctx, instance, caBundle, false) }, "failed to create deployment") WaitFor(func() error { return makeDeploymentReady(ctx, k8sClient, instance) @@ -138,6 +286,7 @@ var _ = Describe("Status and condition tests", func() { Expect(err).NotTo(HaveOccurred(), "Error checking InferenceServicePresent condition") Expect(ISPresentCondition).NotTo(BeNil(), "InferenceServicePresent condition should not be null") Expect(statusMatch).To(Equal(true), "InferenceServicePresent condition should be false") + }) }) @@ -145,7 +294,7 @@ var _ = Describe("Status and condition tests", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService It("Should be available", func() { namespace := "statuses-test-namespace-2" - instance = createDefaultCR(namespace) + instance = createDefaultPVCCustomResource(namespace) WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) }, "failed to create namespace") @@ -164,7 +313,7 @@ var _ = Describe("Status and condition tests", func() { return makePVCReady(ctx, k8sClient, instance) }, "failed to bind PVC") WaitFor(func() error { - return reconciler.ensureDeployment(ctx, instance, caBundle) + return reconciler.ensureDeployment(ctx, instance, caBundle, false) }, "failed to create deployment") WaitFor(func() error { return makeDeploymentReady(ctx, k8sClient, instance) diff --git a/controllers/storage_test.go b/controllers/storage_test.go index f11e6394..c5799035 100644 --- a/controllers/storage_test.go +++ b/controllers/storage_test.go @@ -31,7 +31,7 @@ var _ = Describe("PVC Reconciliation", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService It("should create a new PVC and emit an event", func() { namespace := "pvc-test-namespace-1" - instance = createDefaultCR(namespace) + instance = createDefaultPVCCustomResource(namespace) WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) }, "failed to create namespace") @@ -54,7 +54,7 @@ var _ = Describe("PVC Reconciliation", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService It("should not attempt to create the PVC", func() { namespace := "pvc-test-namespace-2" - instance = createDefaultCR(namespace) + instance = createDefaultPVCCustomResource(namespace) WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) }, "failed to create namespace") @@ -77,4 +77,16 @@ var _ = Describe("PVC Reconciliation", func() { }) }) + Context("when a migration CR is made", func() { + var instance *trustyaiopendatahubiov1alpha1.TrustyAIService + It("Check all fields are correct", func() { + namespace := "pvc-test-namespace-3" + instance = createDefaultMigrationCustomResource(namespace) + + Expect(instance.IsMigration()).To(BeTrue()) + Expect(instance.Spec.Storage.IsStoragePVC()).To(BeFalse()) + Expect(instance.Spec.Storage.IsStorageDatabase()).To(BeTrue()) + }) + }) + }) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index ec35b5f9..7ee75a93 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -65,8 +65,9 @@ var ( ) const ( - defaultServiceName = "example-trustyai-service" - operatorNamespace = "system" + defaultServiceName = "example-trustyai-service" + defaultDatabaseConfigurationName = defaultServiceName + "-db-credentials" + operatorNamespace = "system" ) const ( @@ -86,8 +87,8 @@ func WaitFor(operation func() error, errorMsg string) { Eventually(operation, defaultTimeout, defaultPolling).Should(Succeed(), errorMsg) } -// createDefaultCR creates a TrustyAIService instance with default values -func createDefaultCR(namespaceCurrent string) *trustyaiopendatahubiov1alpha1.TrustyAIService { +// createDefaultPVCCustomResource creates a TrustyAIService instance with default values and PVC backend +func createDefaultPVCCustomResource(namespaceCurrent string) *trustyaiopendatahubiov1alpha1.TrustyAIService { service := trustyaiopendatahubiov1alpha1.TrustyAIService{ ObjectMeta: metav1.ObjectMeta{ Name: defaultServiceName, @@ -96,7 +97,7 @@ func createDefaultCR(namespaceCurrent string) *trustyaiopendatahubiov1alpha1.Tru }, Spec: trustyaiopendatahubiov1alpha1.TrustyAIServiceSpec{ Storage: trustyaiopendatahubiov1alpha1.StorageSpec{ - Format: "PVC", + Format: STORAGE_PVC, Folder: "/data", Size: "1Gi", }, @@ -112,6 +113,54 @@ func createDefaultCR(namespaceCurrent string) *trustyaiopendatahubiov1alpha1.Tru return &service } +// createDefaultDBCustomResource creates a TrustyAIService instance with default values and DB backend +func createDefaultDBCustomResource(namespaceCurrent string) *trustyaiopendatahubiov1alpha1.TrustyAIService { + service := trustyaiopendatahubiov1alpha1.TrustyAIService{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultServiceName, + Namespace: namespaceCurrent, + UID: types.UID(uuid.New().String()), + }, + Spec: trustyaiopendatahubiov1alpha1.TrustyAIServiceSpec{ + Storage: trustyaiopendatahubiov1alpha1.StorageSpec{ + Format: STORAGE_DATABASE, + DatabaseConfigurations: defaultDatabaseConfigurationName, + }, + Metrics: trustyaiopendatahubiov1alpha1.MetricsSpec{ + Schedule: "5s", + }, + }, + } + return &service +} + +// createDefaultMigrationCustomResource creates a TrustyAIService instance with default values and both PVC and DB backend +func createDefaultMigrationCustomResource(namespaceCurrent string) *trustyaiopendatahubiov1alpha1.TrustyAIService { + service := trustyaiopendatahubiov1alpha1.TrustyAIService{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultServiceName, + Namespace: namespaceCurrent, + UID: types.UID(uuid.New().String()), + }, + Spec: trustyaiopendatahubiov1alpha1.TrustyAIServiceSpec{ + Storage: trustyaiopendatahubiov1alpha1.StorageSpec{ + Format: STORAGE_DATABASE, + DatabaseConfigurations: defaultDatabaseConfigurationName, + Folder: "/data", + Size: "1Gi", + }, + Data: trustyaiopendatahubiov1alpha1.DataSpec{ + Filename: "data.csv", + Format: "CSV", + }, + Metrics: trustyaiopendatahubiov1alpha1.MetricsSpec{ + Schedule: "5s", + }, + }, + } + return &service +} + // createNamespace creates a new namespace func createNamespace(ctx context.Context, k8sClient client.Client, namespace string) error { ns := &corev1.Namespace{ @@ -148,6 +197,34 @@ func createConfigMap(namespace string, oauthImage string, trustyaiServiceImage s } } +// createSecret creates a secret in the specified namespace +func createSecret(namespace string, secretName string, data map[string]string) *corev1.Secret { + // Convert the data map values from string to byte array + byteData := make(map[string][]byte) + for key, value := range data { + byteData[key] = []byte(value) + } + + // Define the Secret with the necessary data + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: namespace, + }, + Data: byteData, + } +} + +func createDatabaseConfiguration(namespace string, name string, dbKind string) *corev1.Secret { + return createSecret(namespace, name, map[string]string{ + "databaseKind": dbKind, + "databaseUsername": "foo", + "databasePassword": "bar", + "databaseService": "mariadb-service", + "databasePort": "3306", + }) +} + // createTrustedCABundleConfigMap creates a ConfigMap in the specified namespace // with the label to inject the trusted CA bundle by OpenShift func createTrustedCABundleConfigMap(namespace string) *corev1.ConfigMap { diff --git a/controllers/templates/service/deployment.tmpl.yaml b/controllers/templates/service/deployment.tmpl.yaml index b9f2392e..840421f0 100644 --- a/controllers/templates/service/deployment.tmpl.yaml +++ b/controllers/templates/service/deployment.tmpl.yaml @@ -12,6 +12,11 @@ metadata: app.kubernetes.io/part-of: trustyai app.kubernetes.io/version: {{ .Version }} spec: + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 replicas: 1 selector: matchLabels: @@ -38,25 +43,85 @@ spec: - name: trustyai-service image: {{ .ServiceImage }} env: - - name: STORAGE_DATA_FILENAME - value: {{ .Instance.Spec.Data.Filename }} - name: SERVICE_STORAGE_FORMAT value: {{ .Instance.Spec.Storage.Format }} + {{ if eq .Instance.Spec.Storage.Format "PVC" }} + - name: STORAGE_DATA_FILENAME + value: {{ .Instance.Spec.Data.Filename }} + - name: STORAGE_DATA_FOLDER + value: {{ .Instance.Spec.Storage.Folder }} + - name: SERVICE_DATA_FORMAT + value: {{ .Instance.Spec.Data.Format }} + - name: QUARKUS_HIBERNATE_ORM_ACTIVE + value: false + {{ end }} + {{ if .Instance.IsMigration }} + - name: STORAGE_DATA_FILENAME + value: {{ .Instance.Spec.Data.Filename }} - name: STORAGE_DATA_FOLDER value: {{ .Instance.Spec.Storage.Folder }} - name: SERVICE_DATA_FORMAT value: {{ .Instance.Spec.Data.Format }} + {{ end }} + {{ if eq .Instance.Spec.Storage.Format "DATABASE" }} + - name: QUARKUS_HIBERNATE_ORM_ACTIVE + value: true + - name: QUARKUS_DATASOURCE_DB_KIND + valueFrom: + secretKeyRef: + name: {{ .Instance.Spec.Storage.DatabaseConfigurations }} + key: databaseKind + - name: QUARKUS_DATASOURCE_JDBC_MAX_SIZE + value: 16 + - name: QUARKUS_DATASOURCE_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Instance.Spec.Storage.DatabaseConfigurations }} + key: databaseUsername + - name: QUARKUS_DATASOURCE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Instance.Spec.Storage.DatabaseConfigurations }} + key: databasePassword + - name: DATABASE_SERVICE + valueFrom: + secretKeyRef: + name: {{ .Instance.Spec.Storage.DatabaseConfigurations }} + key: databaseService + - name: DATABASE_PORT + valueFrom: + secretKeyRef: + name: {{ .Instance.Spec.Storage.DatabaseConfigurations }} + key: databasePort + - name: QUARKUS_DATASOURCE_JDBC_URL + value: "jdbc:${QUARKUS_DATASOURCE_DB_KIND}://${DATABASE_SERVICE}:${DATABASE_PORT}/trustyai_database" + - name: SERVICE_DATA_FORMAT + value: "HIBERNATE" + - name: QUARKUS_DATASOURCE_GENERATION + valueFrom: + secretKeyRef: + name: {{ .Instance.Spec.Storage.DatabaseConfigurations }} + key: databaseGeneration + {{ end }} - name: SERVICE_METRICS_SCHEDULE value: {{ .Instance.Spec.Metrics.Schedule }} - name: SERVICE_BATCH_SIZE - value: {{ .Schedule }} + value: {{ .BatchSize }} + {{ if .Instance.IsMigration }} + - name: STORAGE_MIGRATION_CONFIG_FROM_FOLDER + value: {{ .Instance.Spec.Storage.Folder }} + - name: STORAGE_MIGRATION_CONFIG_FROM_FILENAME + value: {{ .Instance.Spec.Data.Filename }} + {{ end }} volumeMounts: - name: {{ .Instance.Name }}-internal readOnly: false mountPath: /etc/tls/internal + {{ if or (eq .Instance.Spec.Storage.Format "PVC") (.Instance.IsMigration) }} - name: {{ .VolumeMountName }} mountPath: {{ .Instance.Spec.Storage.Folder }} readOnly: false + {{ end }} - resources: limits: cpu: 100m @@ -125,9 +190,11 @@ spec: "pods", "verb": "get"}} serviceAccount: {{ .Instance.Name }}-proxy volumes: + {{ if or (eq .Instance.Spec.Storage.Format "PVC") ( .Instance.IsMigration) }} - name: volume persistentVolumeClaim: claimName: {{ .PVCClaimName }} + {{ end }} {{ if .CustomCertificatesBundle.IsDefined }} - name: {{ .CustomCertificatesBundle.VolumeName}} configMap: diff --git a/controllers/trustyaiservice_controller.go b/controllers/trustyaiservice_controller.go index 00600da7..d37fb267 100644 --- a/controllers/trustyaiservice_controller.go +++ b/controllers/trustyaiservice_controller.go @@ -136,25 +136,57 @@ func (r *TrustyAIServiceReconciler) Reconcile(ctx context.Context, req ctrl.Requ return RequeueWithDelayMessage(ctx, time.Minute, "Not all replicas are ready, requeue the reconcile request") } - // Ensure PVC - err = r.ensurePVC(ctx, instance) - if err != nil { - // PVC not found condition - log.FromContext(ctx).Error(err, "Error creating PVC storage.") - _, updateErr := r.updateStatus(ctx, instance, UpdatePVCNotAvailable) - if updateErr != nil { - return RequeueWithErrorMessage(ctx, err, "Failed to update status") - } + if instance.Spec.Storage.IsStoragePVC() || instance.IsMigration() { + // Ensure PVC + err = r.ensurePVC(ctx, instance) + if err != nil { + // PVC not found condition + log.FromContext(ctx).Error(err, "Error creating PVC storage.") + _, updateErr := r.updateStatus(ctx, instance, UpdatePVCNotAvailable) + if updateErr != nil { + return RequeueWithErrorMessage(ctx, err, "Failed to update status") + } - // If there was an error finding the PV, requeue the request - return RequeueWithErrorMessage(ctx, err, "Could not find requested PersistentVolumeClaim.") + // If there was an error finding the PV, requeue the request + return RequeueWithErrorMessage(ctx, err, "Could not find requested PersistentVolumeClaim.") + } + } + if instance.Spec.Storage.IsStorageDatabase() { + // Get database configuration + secret, err := r.findDatabaseSecret(ctx, instance) + if err != nil { + return RequeueWithErrorMessage(ctx, err, "Service configured to use database storage but no database configuration found.") + } + err = r.validateDatabaseSecret(secret) + if err != nil { + return RequeueWithErrorMessage(ctx, err, "Database configuration contains errors.") + } } - // Ensure Deployment object - err = r.ensureDeployment(ctx, instance, caBundle) - if err != nil { - return RequeueWithError(err) + // Check for migration annotation + if _, ok := instance.Annotations[migrationAnnotationKey]; ok { + log.FromContext(ctx).Info("Found migration annotation. Migrating.") + err = r.ensureDeployment(ctx, instance, caBundle, true) + //err = r.redeployForMigration(ctx, instance) + + if err != nil { + return RequeueWithErrorMessage(ctx, err, "Retrying to restart deployment during migration.") + } + + // Remove the migration annotation after processing to avoid restarts + delete(instance.Annotations, migrationAnnotationKey) + log.FromContext(ctx).Info("Deleting annotation") + if err := r.Update(ctx, instance); err != nil { + return RequeueWithErrorMessage(ctx, err, "Failed to remove migration annotation.") + } + } else { + // Ensure Deployment object + err = r.ensureDeployment(ctx, instance, caBundle, false) + log.FromContext(ctx).Info("No annotation found") + if err != nil { + return RequeueWithError(err) + } } // Fetch the TrustyAIService instance @@ -235,6 +267,7 @@ func (r *TrustyAIServiceReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&trustyaiopendatahubiov1alpha1.TrustyAIService{}). + Owns(&appsv1.Deployment{}). Watches(&source.Kind{Type: &kservev1beta1.InferenceService{}}, &handler.EnqueueRequestForObject{}). Watches(&source.Kind{Type: &kservev1alpha1.ServingRuntime{}}, &handler.EnqueueRequestForObject{}). Complete(r)