diff --git a/internal/controller/install/armadaserver_controller.go b/internal/controller/install/armadaserver_controller.go index c622228..3dd642b 100644 --- a/internal/controller/install/armadaserver_controller.go +++ b/internal/controller/install/armadaserver_controller.go @@ -449,6 +449,8 @@ func createArmadaServerDeployment( volumeMounts := createVolumeMounts(GetConfigFilename(as.Name), as.Spec.AdditionalVolumeMounts) volumeMounts = append(volumeMounts, createPulsarVolumeMounts(pulsarConfig)...) + readinessProbe, livenessProbe := CreateProbesWithScheme(GetServerScheme(commonConfig.GRPC.TLS)) + deployment := appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: as.Name, @@ -483,6 +485,8 @@ func createArmadaServerDeployment( Env: env, VolumeMounts: volumeMounts, SecurityContext: as.Spec.SecurityContext, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, }}, Volumes: volumes, }, diff --git a/internal/controller/install/armadaserver_controller_test.go b/internal/controller/install/armadaserver_controller_test.go index e94f12e..bd0319c 100644 --- a/internal/controller/install/armadaserver_controller_test.go +++ b/internal/controller/install/armadaserver_controller_test.go @@ -5,6 +5,10 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" + "google.golang.org/protobuf/testing/protocmp" + "k8s.io/apimachinery/pkg/util/intstr" + "github.com/armadaproject/armada-operator/internal/controller/builders" "k8s.io/utils/ptr" @@ -429,12 +433,12 @@ func TestSchedulerReconciler_createIngress(t *testing.T) { input := v1alpha1.ArmadaServer{ TypeMeta: metav1.TypeMeta{ - Kind: "Lookout", + Kind: "ArmadaServer", APIVersion: "install.armadaproject.io/v1alpha1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: "default", - Name: "lookout", + Name: "armadaserver", }, Spec: v1alpha1.ArmadaServerSpec{ Replicas: ptr.To[int32](2), @@ -459,3 +463,210 @@ func TestSchedulerReconciler_createIngress(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, ingress) } + +func TestArmadaServerReconciler_CreateDeployment(t *testing.T) { + t.Parallel() + + commonConfig := &builders.CommonApplicationConfig{ + HTTPPort: 8080, + GRPCPort: 5051, + MetricsPort: 9000, + Profiling: builders.ProfilingConfig{ + Port: 1337, + }, + } + + armadaServer := &installv1alpha1.ArmadaServer{ + TypeMeta: metav1.TypeMeta{ + Kind: "ArmadaServer", + APIVersion: "install.armadaproject.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "armadaserver"}, + Spec: installv1alpha1.ArmadaServerSpec{ + PulsarInit: true, + CommonSpecBase: installv1alpha1.CommonSpecBase{ + Labels: map[string]string{"test": "hello"}, + Image: installv1alpha1.Image{ + Repository: "testrepo", + Tag: "1.0.0", + }, + ApplicationConfig: runtime.RawExtension{}, + Resources: &corev1.ResourceRequirements{}, + Prometheus: &installv1alpha1.PrometheusConfig{Enabled: true}, + }, + ClusterIssuer: "test", + HostNames: []string{"localhost"}, + Ingress: &installv1alpha1.IngressConfig{ + IngressClass: "nginx", + Labels: map[string]string{"test": "hello"}, + Annotations: map[string]string{"test": "hello"}, + }, + }, + } + + deployment, err := createArmadaServerDeployment(armadaServer, "armadaserver", commonConfig) + assert.NoError(t, err) + + expectedDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "armadaserver", + Namespace: "default", + Labels: map[string]string{ + "app": "armadaserver", + "release": "armadaserver", + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To[int32](1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "armadaserver", + }, + }, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxUnavailable: &intstr.IntOrString{ + IntVal: 1, + }, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "armadaserver", + Namespace: "default", + Labels: map[string]string{ + "app": "armadaserver", + "release": "armadaserver", + }, + Annotations: map[string]string{ + "checksum/config": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + PodAffinity: &corev1.PodAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{ + { + Weight: 100, + PodAffinityTerm: corev1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "app", + Operator: metav1.LabelSelectorOpIn, + Values: []string{ + "armadaserver", + }, + }, + }, + }, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "user-config", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "armadaserver", + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Args: []string{ + "--config", + "/config/application_config.yaml", + }, + Env: []corev1.EnvVar{ + { + Name: "SERVICE_ACCOUNT", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.serviceAccountName", + }, + }, + }, + { + Name: "POD_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + }, + Image: "testrepo:1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromString("http"), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 10, + TimeoutSeconds: 10, + FailureThreshold: 3, + }, + Name: "armadaserver", + Ports: []corev1.ContainerPort{ + { + Name: "grpc", + ContainerPort: 5051, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "http", + ContainerPort: 8080, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "metrics", + ContainerPort: 9000, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "profiling", + ContainerPort: 1337, + Protocol: corev1.ProtocolTCP, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromString("http"), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 5, + TimeoutSeconds: 5, + FailureThreshold: 2, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "user-config", + ReadOnly: true, + MountPath: appConfigFilepath, + SubPath: "armadaserver-config.yaml", + }, + }, + }, + }, + ServiceAccountName: "armadaserver", + }, + }, + }, + } + + if !cmp.Equal(expectedDeployment, deployment, protocmp.Transform()) { + t.Fatalf("deployment is not the same %s", cmp.Diff(expectedDeployment, deployment, protocmp.Transform())) + } +} diff --git a/internal/controller/install/binoculars_controller.go b/internal/controller/install/binoculars_controller.go index d7c80a0..95b328c 100644 --- a/internal/controller/install/binoculars_controller.go +++ b/internal/controller/install/binoculars_controller.go @@ -227,6 +227,8 @@ func createBinocularsDeployment( env := createEnv(binoculars.Spec.Environment) volumes := createVolumes(binoculars.Name, binoculars.Spec.AdditionalVolumes) volumeMounts := createVolumeMounts(GetConfigFilename(secret.Name), binoculars.Spec.AdditionalVolumeMounts) + readinessProbe, livenessProbe := CreateProbesWithScheme(GetServerScheme(commonConfig.GRPC.TLS)) + containers := []corev1.Container{{ Name: "binoculars", ImagePullPolicy: corev1.PullIfNotPresent, @@ -236,6 +238,8 @@ func createBinocularsDeployment( Env: env, VolumeMounts: volumeMounts, SecurityContext: binoculars.Spec.SecurityContext, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, }} deployment := appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Name: binoculars.Name, Namespace: binoculars.Namespace, Labels: AllLabels(binoculars.Name, binoculars.Labels)}, diff --git a/internal/controller/install/binoculars_controller_test.go b/internal/controller/install/binoculars_controller_test.go index 1049608..651c984 100644 --- a/internal/controller/install/binoculars_controller_test.go +++ b/internal/controller/install/binoculars_controller_test.go @@ -5,6 +5,10 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" + "google.golang.org/protobuf/testing/protocmp" + "k8s.io/apimachinery/pkg/util/intstr" + "github.com/armadaproject/armada-operator/internal/controller/builders" "k8s.io/utils/ptr" @@ -14,6 +18,7 @@ import ( "github.com/armadaproject/armada-operator/test/k8sclient" "github.com/golang/mock/gomock" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" @@ -476,12 +481,12 @@ func TestSchedulerReconciler_createBinocularsIngress(t *testing.T) { input := v1alpha1.Binoculars{ TypeMeta: metav1.TypeMeta{ - Kind: "Lookout", + Kind: "Binoculars", APIVersion: "install.armadaproject.io/v1alpha1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: "default", - Name: "lookout", + Name: "binoculars", }, Spec: v1alpha1.BinocularsSpec{ Replicas: ptr.To[int32](2), @@ -506,3 +511,208 @@ func TestSchedulerReconciler_createBinocularsIngress(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, ingress) } + +func TestBinocularsReconciler_CreateDeployment(t *testing.T) { + t.Parallel() + + commonConfig := &builders.CommonApplicationConfig{ + HTTPPort: 8080, + GRPCPort: 5051, + MetricsPort: 9000, + Profiling: builders.ProfilingConfig{ + Port: 1337, + }, + } + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "binoculars", + }, + } + + binoculars := &v1alpha1.Binoculars{ + TypeMeta: metav1.TypeMeta{ + Kind: "Binoculars", + APIVersion: "install.armadaproject.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "binoculars"}, + Spec: v1alpha1.BinocularsSpec{ + CommonSpecBase: installv1alpha1.CommonSpecBase{ + Labels: map[string]string{"test": "hello"}, + Image: installv1alpha1.Image{ + Repository: "testrepo", + Tag: "1.0.0", + }, + ApplicationConfig: runtime.RawExtension{}, + Resources: &corev1.ResourceRequirements{}, + }, + + Replicas: ptr.To[int32](2), + HostNames: []string{"localhost"}, + ClusterIssuer: "test", + Ingress: &installv1alpha1.IngressConfig{ + IngressClass: "nginx", + Labels: map[string]string{"test": "hello"}, + Annotations: map[string]string{"test": "hello"}, + }, + }, + } + + deployment, err := createBinocularsDeployment(binoculars, secret, "binoculars", commonConfig) + assert.NoError(t, err) + + expectedDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "binoculars", + Namespace: "default", + Labels: map[string]string{ + "app": "binoculars", + "release": "binoculars", + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To[int32](2), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "binoculars", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "binoculars", + Namespace: "default", + Labels: map[string]string{ + "app": "binoculars", + "release": "binoculars", + }, + Annotations: map[string]string{ + "checksum/config": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + PodAffinity: &corev1.PodAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{ + { + Weight: 100, + PodAffinityTerm: corev1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "app", + Operator: metav1.LabelSelectorOpIn, + Values: []string{ + "binoculars", + }, + }, + }, + }, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "user-config", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "binoculars", + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Args: []string{ + "--config", + "/config/application_config.yaml", + }, + Env: []corev1.EnvVar{ + { + Name: "SERVICE_ACCOUNT", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.serviceAccountName", + }, + }, + }, + { + Name: "POD_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + }, + Image: "testrepo:1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromString("http"), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 10, + TimeoutSeconds: 10, + FailureThreshold: 3, + }, + Name: "binoculars", + Ports: []corev1.ContainerPort{ + { + Name: "grpc", + ContainerPort: 5051, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "http", + ContainerPort: 8080, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "metrics", + ContainerPort: 9000, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "profiling", + ContainerPort: 1337, + Protocol: corev1.ProtocolTCP, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromString("http"), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 5, + TimeoutSeconds: 5, + FailureThreshold: 2, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "user-config", + ReadOnly: true, + MountPath: appConfigFilepath, + SubPath: "binoculars-config.yaml", + }, + }, + }, + }, + ServiceAccountName: "binoculars", + }, + }, + }, + } + + if !cmp.Equal(expectedDeployment, deployment, protocmp.Transform()) { + t.Fatalf("deployment is not the same %s", cmp.Diff(expectedDeployment, deployment, protocmp.Transform())) + } +} diff --git a/internal/controller/install/common_helpers.go b/internal/controller/install/common_helpers.go index 8002b4f..0067c34 100644 --- a/internal/controller/install/common_helpers.go +++ b/internal/controller/install/common_helpers.go @@ -320,6 +320,15 @@ func ExtractPulsarConfig(config runtime.RawExtension) (PulsarConfig, error) { return asConfig.Pulsar, nil } +// GetServerScheme returns the URI scheme for the grpc server +func GetServerScheme(tlsConfig builders.TLSConfig) corev1.URIScheme { + if tlsConfig.Enabled { + return corev1.URISchemeHTTPS + } + + return corev1.URISchemeHTTP +} + // waitForJob waits for the Job to reach a terminal state (complete or failed). func waitForJob(ctx context.Context, c client.Client, job *batchv1.Job, pollInterval, timeout time.Duration) error { return wait.PollUntilContextTimeout( @@ -371,6 +380,36 @@ func createEnv(crdEnv []corev1.EnvVar) []corev1.EnvVar { return envVars } +func CreateProbesWithScheme(scheme corev1.URIScheme) (*corev1.Probe, *corev1.Probe) { + readinessProbe := &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromString("http"), + Scheme: scheme, + }, + }, + InitialDelaySeconds: 5, + TimeoutSeconds: 5, + FailureThreshold: 2, + } + + livenessProbe := &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromString("http"), + Scheme: scheme, + }, + }, + InitialDelaySeconds: 10, + TimeoutSeconds: 10, + FailureThreshold: 3, + } + + return readinessProbe, livenessProbe +} + // createVolumes creates the default appconfig Volume and appends the CRD AdditionalVolumes func createVolumes(configVolumeSecretName string, crdVolumes []corev1.Volume) []corev1.Volume { volumes := []corev1.Volume{{ @@ -754,15 +793,6 @@ func newContainerPortsHTTPWithMetrics(config *builders.CommonApplicationConfig) return ports } -// newContainerPortsGRPCWithMetrics creates container ports for grpc and metrics server and optional port for profiling server. -func newContainerPortsGRPCWithMetrics(config *builders.CommonApplicationConfig) []corev1.ContainerPort { - ports := []corev1.ContainerPort{newContainerPortGRPC(config), newContainerPortMetrics(config)} - if config.Profiling.Port > 0 { - ports = append(ports, newContainerPortProfiling(config)) - } - return ports -} - // newContainerPortsMetrics creates container ports for metrics server and optional port for profiling server. func newContainerPortsMetrics(config *builders.CommonApplicationConfig) []corev1.ContainerPort { ports := []corev1.ContainerPort{newContainerPortMetrics(config)} diff --git a/internal/controller/install/executor_controller.go b/internal/controller/install/executor_controller.go index 26d0644..3e79c0e 100644 --- a/internal/controller/install/executor_controller.go +++ b/internal/controller/install/executor_controller.go @@ -180,7 +180,7 @@ func (r *ExecutorReconciler) generateExecutorInstallComponents( } serviceAccountName = serviceAccount.Name } - deployment := r.createDeployment(executor, serviceAccountName, config) + deployment := createExecutorDeployment(executor, serviceAccountName, config) if err = controllerutil.SetOwnerReference(executor, deployment, scheme); err != nil { return nil, errors.WithStack(err) } @@ -241,7 +241,7 @@ func (r *ExecutorReconciler) generateExecutorInstallComponents( return components, nil } -func (r *ExecutorReconciler) createDeployment( +func createExecutorDeployment( executor *installv1alpha1.Executor, serviceAccountName string, config *builders.CommonApplicationConfig, @@ -249,6 +249,7 @@ func (r *ExecutorReconciler) createDeployment( var replicas int32 = 1 volumes := createVolumes(executor.Name, executor.Spec.AdditionalVolumes) volumeMounts := createVolumeMounts(GetConfigFilename(executor.Name), executor.Spec.AdditionalVolumeMounts) + readinessProbe, livenessProbe := CreateProbesWithScheme(corev1.URISchemeHTTP) env := []corev1.EnvVar{ { @@ -274,10 +275,12 @@ func (r *ExecutorReconciler) createDeployment( ImagePullPolicy: corev1.PullIfNotPresent, Image: ImageString(executor.Spec.Image), Args: []string{appConfigFlag, appConfigFilepath}, - Ports: newContainerPortsMetrics(config), + Ports: newContainerPortsHTTPWithMetrics(config), Env: env, VolumeMounts: volumeMounts, SecurityContext: executor.Spec.SecurityContext, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, }} deployment := appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Name: executor.Name, Namespace: executor.Namespace, Labels: AllLabels(executor.Name, executor.Labels)}, diff --git a/internal/controller/install/executor_controller_test.go b/internal/controller/install/executor_controller_test.go index 0d9a480..830d4ca 100644 --- a/internal/controller/install/executor_controller_test.go +++ b/internal/controller/install/executor_controller_test.go @@ -5,6 +5,13 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" + "google.golang.org/protobuf/testing/protocmp" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + + "github.com/armadaproject/armada-operator/internal/controller/builders" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "github.com/armadaproject/armada-operator/test/k8sclient" @@ -394,3 +401,165 @@ func TestExecutorReconciler_generateAdditionalClusterRoles(t *testing.T) { } assert.Equal(t, expectedClusterRoleBinding2, *bindings[1]) } + +func TestExecutorReconciler_CreateDeployment(t *testing.T) { + t.Parallel() + + commonConfig := &builders.CommonApplicationConfig{ + HTTPPort: 8080, + GRPCPort: 5051, + MetricsPort: 9000, + Profiling: builders.ProfilingConfig{ + Port: 1337, + }, + } + + executor := &installv1alpha1.Executor{ + TypeMeta: metav1.TypeMeta{ + Kind: "Executor", + APIVersion: "install.armadaproject.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "executor"}, + Spec: installv1alpha1.ExecutorSpec{ + CommonSpecBase: installv1alpha1.CommonSpecBase{ + Labels: nil, + Image: installv1alpha1.Image{ + Repository: "testrepo", + Tag: "1.0.0", + }, + ApplicationConfig: runtime.RawExtension{}, + Resources: &corev1.ResourceRequirements{}, + Prometheus: &installv1alpha1.PrometheusConfig{Enabled: true, ScrapeInterval: &metav1.Duration{Duration: 1 * time.Second}}, + }, + }, + } + + deployment := createExecutorDeployment(executor, "executor", commonConfig) + + expectedDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "executor", + Namespace: "default", + Labels: map[string]string{ + "app": "executor", + "release": "executor", + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To[int32](1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "executor", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "executor", + Namespace: "default", + Labels: map[string]string{ + "app": "executor", + "release": "executor", + }, + Annotations: map[string]string{ + "checksum/config": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }, + }, + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + { + Name: "user-config", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "executor", + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Args: []string{ + "--config", + "/config/application_config.yaml", + }, + Env: []corev1.EnvVar{ + { + Name: "SERVICE_ACCOUNT", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.serviceAccountName", + }, + }, + }, + { + Name: "POD_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + }, + Image: "testrepo:1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromString("http"), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 10, + TimeoutSeconds: 10, + FailureThreshold: 3, + }, + Name: "executor", + Ports: []corev1.ContainerPort{ + { + Name: "http", + ContainerPort: 8080, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "metrics", + ContainerPort: 9000, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "profiling", + ContainerPort: 1337, + Protocol: corev1.ProtocolTCP, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromString("http"), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 5, + TimeoutSeconds: 5, + FailureThreshold: 2, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "user-config", + ReadOnly: true, + MountPath: appConfigFilepath, + SubPath: "executor-config.yaml", + }, + }, + }, + }, + ServiceAccountName: "executor", + }, + }, + }, + } + + if !cmp.Equal(expectedDeployment, deployment, protocmp.Transform()) { + t.Fatalf("deployment is not the same %s", cmp.Diff(expectedDeployment, deployment, protocmp.Transform())) + } +} diff --git a/internal/controller/install/lookout_controller.go b/internal/controller/install/lookout_controller.go index 6777c9f..3dd6ebf 100644 --- a/internal/controller/install/lookout_controller.go +++ b/internal/controller/install/lookout_controller.go @@ -273,6 +273,7 @@ func createLookoutDeployment(lookout *installv1alpha1.Lookout, serviceAccountNam env := createEnv(lookout.Spec.Environment) volumes := createVolumes(lookout.Name, lookout.Spec.AdditionalVolumes) volumeMounts := createVolumeMounts(GetConfigFilename(lookout.Name), lookout.Spec.AdditionalVolumeMounts) + readinessProbe, livenessProbe := CreateProbesWithScheme(GetServerScheme(config.GRPC.TLS)) deployment := appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Name: lookout.Name, Namespace: lookout.Namespace, Labels: AllLabels(lookout.Name, lookout.Labels)}, @@ -302,6 +303,8 @@ func createLookoutDeployment(lookout *installv1alpha1.Lookout, serviceAccountNam Env: env, VolumeMounts: volumeMounts, SecurityContext: lookout.Spec.SecurityContext, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, }}, Volumes: volumes, }, @@ -461,12 +464,7 @@ func createLookoutCronJob(lookout *installv1alpha1.Lookout, serviceAccountName s dbPruningSchedule = *lookout.Spec.DbPruningSchedule } - appConfig, err := builders.ConvertRawExtensionToYaml(lookout.Spec.ApplicationConfig) - if err != nil { - return nil, err - } - var lookoutConfig LookoutConfig - err = yaml.Unmarshal([]byte(appConfig), &lookoutConfig) + lookoutConfig, err := extractLookoutConfig(lookout.Spec.ApplicationConfig) if err != nil { return nil, err } @@ -558,3 +556,17 @@ func (r *LookoutReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&installv1alpha1.Lookout{}). Complete(r) } + +// extractLookoutConfig will unmarshal the appconfig and return the LookoutConfig portion +func extractLookoutConfig(config runtime.RawExtension) (LookoutConfig, error) { + appConfig, err := builders.ConvertRawExtensionToYaml(config) + if err != nil { + return LookoutConfig{}, err + } + var lookoutConfig LookoutConfig + err = yaml.Unmarshal([]byte(appConfig), &lookoutConfig) + if err != nil { + return LookoutConfig{}, err + } + return lookoutConfig, err +} diff --git a/internal/controller/install/lookout_controller_test.go b/internal/controller/install/lookout_controller_test.go index cb3e046..113b9a2 100644 --- a/internal/controller/install/lookout_controller_test.go +++ b/internal/controller/install/lookout_controller_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "k8s.io/apimachinery/pkg/util/intstr" + "github.com/google/go-cmp/cmp" "google.golang.org/protobuf/testing/protocmp" @@ -529,6 +531,201 @@ func TestLookoutReconciler_CreateCronJobErrorDueToApplicationConfig(t *testing.T assert.Equal(t, "yaml: line 1: did not find expected ',' or '}'", err.Error()) } +func TestLookoutReconciler_CreateDeployment(t *testing.T) { + t.Parallel() + + commonConfig := &builders.CommonApplicationConfig{ + HTTPPort: 8080, + GRPCPort: 5051, + MetricsPort: 9000, + Profiling: builders.ProfilingConfig{ + Port: 1337, + }, + } + + lookout := &v1alpha1.Lookout{ + TypeMeta: metav1.TypeMeta{ + Kind: "Lookout", + APIVersion: "install.armadaproject.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "lookout", + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + Finalizers: []string{operatorFinalizer}, + }, + Spec: v1alpha1.LookoutSpec{ + CommonSpecBase: installv1alpha1.CommonSpecBase{ + Labels: nil, + Image: v1alpha1.Image{ + Repository: "testrepo", + Tag: "1.0.0", + }, + ApplicationConfig: runtime.RawExtension{Raw: []byte(`{}`)}, + Resources: &corev1.ResourceRequirements{}, + }, + Replicas: ptr.To[int32](2), + ClusterIssuer: "test", + Ingress: &v1alpha1.IngressConfig{ + IngressClass: "nginx", + }, + }, + } + + deployment, err := createLookoutDeployment(lookout, "lookout", commonConfig) + assert.NoError(t, err) + + expectedDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "lookout", + Namespace: "default", + Labels: map[string]string{ + "app": "lookout", + "release": "lookout", + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To[int32](2), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "lookout", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "lookout", + Namespace: "default", + Labels: map[string]string{ + "app": "lookout", + "release": "lookout", + }, + Annotations: map[string]string{ + "checksum/config": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + }, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + PodAffinity: &corev1.PodAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{ + { + Weight: 100, + PodAffinityTerm: corev1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "app", + Operator: metav1.LabelSelectorOpIn, + Values: []string{ + "lookout", + }, + }, + }, + }, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "user-config", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "lookout", + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Args: []string{ + "--config", + "/config/application_config.yaml", + }, + Env: []corev1.EnvVar{ + { + Name: "SERVICE_ACCOUNT", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.serviceAccountName", + }, + }, + }, + { + Name: "POD_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + }, + Image: "testrepo:1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromString("http"), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 10, + TimeoutSeconds: 10, + FailureThreshold: 3, + }, + Name: "lookout", + Ports: []corev1.ContainerPort{ + { + Name: "http", + ContainerPort: 8080, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "metrics", + ContainerPort: 9000, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "profiling", + ContainerPort: 1337, + Protocol: corev1.ProtocolTCP, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromString("http"), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 5, + TimeoutSeconds: 5, + FailureThreshold: 2, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "user-config", + ReadOnly: true, + MountPath: appConfigFilepath, + SubPath: "lookout-config.yaml", + }, + }, + }, + }, + ServiceAccountName: "lookout", + }, + }, + }, + } + + if !cmp.Equal(expectedDeployment, deployment, protocmp.Transform()) { + t.Fatalf("deployment is not the same %s", cmp.Diff(expectedDeployment, deployment, protocmp.Transform())) + } +} + func TestLookoutReconciler_CreateCronJob(t *testing.T) { t.Parallel() diff --git a/internal/controller/install/scheduler_controller.go b/internal/controller/install/scheduler_controller.go index 19403f8..810efc5 100644 --- a/internal/controller/install/scheduler_controller.go +++ b/internal/controller/install/scheduler_controller.go @@ -296,6 +296,7 @@ func newSchedulerDeployment( volumes = append(volumes, createPulsarVolumes(pulsarConfig)...) volumeMounts := createVolumeMounts(GetConfigFilename(scheduler.Name), scheduler.Spec.AdditionalVolumeMounts) volumeMounts = append(volumeMounts, createPulsarVolumeMounts(pulsarConfig)...) + readinessProbe, livenessProbe := CreateProbesWithScheme(GetServerScheme(config.GRPC.TLS)) deployment := appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Name: scheduler.Name, Namespace: scheduler.Namespace, Labels: AllLabels(scheduler.Name, scheduler.Labels)}, @@ -321,10 +322,12 @@ func newSchedulerDeployment( ImagePullPolicy: corev1.PullIfNotPresent, Image: ImageString(scheduler.Spec.Image), Args: []string{"run", appConfigFlag, appConfigFilepath}, - Ports: newContainerPortsGRPCWithMetrics(config), + Ports: newContainerPortsAll(config), Env: env, VolumeMounts: volumeMounts, SecurityContext: scheduler.Spec.SecurityContext, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, }}, Volumes: volumes, }, @@ -374,12 +377,7 @@ func newSchedulerMigrationJob(scheduler *installv1alpha1.Scheduler, serviceAccou volumes := createVolumes(scheduler.Name, scheduler.Spec.AdditionalVolumes) volumeMounts := createVolumeMounts(GetConfigFilename(scheduler.Name), scheduler.Spec.AdditionalVolumeMounts) - appConfig, err := builders.ConvertRawExtensionToYaml(scheduler.Spec.ApplicationConfig) - if err != nil { - return nil, err - } - var schedulerConfig SchedulerConfig - err = yaml.Unmarshal([]byte(appConfig), &schedulerConfig) + schedulerConfig, err := extractSchedulerConfig(scheduler.Spec.ApplicationConfig) if err != nil { return nil, err } @@ -753,3 +751,17 @@ func (r *SchedulerReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&installv1alpha1.Scheduler{}). Complete(r) } + +// extractSchedulerConfig will unmarshal the appconfig and return the SchedulerConfig portion +func extractSchedulerConfig(config runtime.RawExtension) (SchedulerConfig, error) { + appConfig, err := builders.ConvertRawExtensionToYaml(config) + if err != nil { + return SchedulerConfig{}, err + } + var schedulerConfig SchedulerConfig + err = yaml.Unmarshal([]byte(appConfig), &schedulerConfig) + if err != nil { + return SchedulerConfig{}, err + } + return schedulerConfig, err +} diff --git a/internal/controller/install/scheduler_controller_test.go b/internal/controller/install/scheduler_controller_test.go index 20ec586..d716173 100644 --- a/internal/controller/install/scheduler_controller_test.go +++ b/internal/controller/install/scheduler_controller_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "k8s.io/apimachinery/pkg/util/intstr" + "github.com/google/go-cmp/cmp" "google.golang.org/protobuf/testing/protocmp" @@ -702,6 +704,206 @@ func TestSchedulerReconciler_ReconcileMissingResources(t *testing.T) { } } +func TestSchedulerReconciler_CreateDeployment(t *testing.T) { + t.Parallel() + + commonConfig := &builders.CommonApplicationConfig{ + HTTPPort: 8080, + GRPCPort: 5051, + MetricsPort: 9000, + Profiling: builders.ProfilingConfig{ + Port: 1337, + }, + } + + scheduler := &v1alpha1.Scheduler{ + TypeMeta: metav1.TypeMeta{ + Kind: "Scheduler", + APIVersion: "install.armadaproject.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "scheduler"}, + Spec: v1alpha1.SchedulerSpec{ + Replicas: ptr.To[int32](2), + CommonSpecBase: installv1alpha1.CommonSpecBase{ + Labels: nil, + Image: v1alpha1.Image{ + Repository: "testrepo", + Tag: "1.0.0", + }, + ApplicationConfig: runtime.RawExtension{}, + Prometheus: &installv1alpha1.PrometheusConfig{Enabled: true, ScrapeInterval: &metav1.Duration{Duration: 1 * time.Second}}, + TerminationGracePeriodSeconds: ptr.To(int64(20)), + }, + ClusterIssuer: "test", + HostNames: []string{"localhost"}, + Ingress: &installv1alpha1.IngressConfig{ + IngressClass: "nginx", + Labels: map[string]string{"test": "hello"}, + Annotations: map[string]string{"test": "hello"}, + }, + }, + } + + deployment, err := newSchedulerDeployment(scheduler, "scheduler", commonConfig) + assert.NoError(t, err) + + expectedDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "scheduler", + Namespace: "default", + Labels: map[string]string{ + "app": "scheduler", + "release": "scheduler", + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To[int32](2), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "scheduler", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "scheduler", + Namespace: "default", + Labels: map[string]string{ + "app": "scheduler", + "release": "scheduler", + }, + Annotations: map[string]string{ + "checksum/config": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + PodAffinity: &corev1.PodAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{ + { + Weight: 100, + PodAffinityTerm: corev1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "app", + Operator: metav1.LabelSelectorOpIn, + Values: []string{ + "scheduler", + }, + }, + }, + }, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "user-config", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "scheduler", + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Args: []string{ + "run", + "--config", + "/config/application_config.yaml", + }, + Env: []corev1.EnvVar{ + { + Name: "SERVICE_ACCOUNT", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.serviceAccountName", + }, + }, + }, + { + Name: "POD_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + }, + Image: "testrepo:1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromString("http"), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 10, + TimeoutSeconds: 10, + FailureThreshold: 3, + }, + Name: "scheduler", + Ports: []corev1.ContainerPort{ + { + Name: "grpc", + ContainerPort: 5051, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "http", + ContainerPort: 8080, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "metrics", + ContainerPort: 9000, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "profiling", + ContainerPort: 1337, + Protocol: corev1.ProtocolTCP, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromString("http"), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 5, + TimeoutSeconds: 5, + FailureThreshold: 2, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "user-config", + ReadOnly: true, + MountPath: appConfigFilepath, + SubPath: "scheduler-config.yaml", + }, + }, + }, + }, + ServiceAccountName: "scheduler", + }, + }, + }, + } + + if !cmp.Equal(expectedDeployment, deployment, protocmp.Transform()) { + t.Fatalf("deployment is not the same %s", cmp.Diff(expectedDeployment, deployment, protocmp.Transform())) + } +} + func TestSchedulerReconciler_createSchedulerCronJob(t *testing.T) { t.Parallel()