Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Add liveness and readiness probes #339

Merged
merged 3 commits into from
Nov 26, 2024
Merged

fix: Add liveness and readiness probes #339

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6ec02d6
Add liveness and readiness probes
Sovietaced Nov 21, 2024
d0d1a6c
Address comments, fix issues, more tests
Sovietaced Nov 23, 2024
90e9d45
Address comments
Sovietaced Nov 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions internal/controller/install/armadaserver_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,13 @@ func createArmadaServerDeployment(
volumeMounts := createVolumeMounts(GetConfigFilename(as.Name), as.Spec.AdditionalVolumeMounts)
volumeMounts = append(volumeMounts, createPulsarVolumeMounts(pulsarConfig)...)

asConfig, err := extractArmadaServerConfig(as.Spec.ApplicationConfig)
if err != nil {
return nil, err
}

readinessProbe, livenessProbe := CreateProbesWithScheme(GetServerScheme(asConfig.Grpc.Tls))

deployment := appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: as.Name,
Expand Down Expand Up @@ -483,6 +490,8 @@ func createArmadaServerDeployment(
Env: env,
VolumeMounts: volumeMounts,
SecurityContext: as.Spec.SecurityContext,
ReadinessProbe: readinessProbe,
LivenessProbe: livenessProbe,
}},
Volumes: volumes,
},
Expand Down Expand Up @@ -678,3 +687,21 @@ func createServerPrometheusRule(server *installv1alpha1.ArmadaServer) *monitorin
},
}
}

type ArmadaServerConfig struct {
Sovietaced marked this conversation as resolved.
Show resolved Hide resolved
Grpc GrpcConfig
}

// extractArmadaServerConfig will unmarshal the appconfig and return the AramadaServerConfig portion
func extractArmadaServerConfig(config runtime.RawExtension) (ArmadaServerConfig, error) {
appConfig, err := builders.ConvertRawExtensionToYaml(config)
if err != nil {
return ArmadaServerConfig{}, err
}
var asConfig ArmadaServerConfig
err = yaml.Unmarshal([]byte(appConfig), &asConfig)
if err != nil {
return ArmadaServerConfig{}, err
}
return asConfig, err
}
29 changes: 29 additions & 0 deletions internal/controller/install/binoculars_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ import (
"context"
"time"

"sigs.k8s.io/yaml"

"github.com/pkg/errors"

installv1alpha1 "github.com/armadaproject/armada-operator/api/install/v1alpha1"
Expand Down Expand Up @@ -227,6 +229,13 @@ func createBinocularsDeployment(
env := createEnv(binoculars.Spec.Environment)
volumes := createVolumes(binoculars.Name, binoculars.Spec.AdditionalVolumes)
volumeMounts := createVolumeMounts(GetConfigFilename(secret.Name), binoculars.Spec.AdditionalVolumeMounts)

binocularsConfig, err := extractBinocularsConfig(binoculars.Spec.ApplicationConfig)
if err != nil {
return nil, err
}
readinessProbe, livenessProbe := CreateProbesWithScheme(GetServerScheme(binocularsConfig.Grpc.Tls))

containers := []corev1.Container{{
Name: "binoculars",
ImagePullPolicy: corev1.PullIfNotPresent,
Expand All @@ -236,6 +245,8 @@ func createBinocularsDeployment(
Env: env,
VolumeMounts: volumeMounts,
SecurityContext: binoculars.Spec.SecurityContext,
ReadinessProbe: readinessProbe,
LivenessProbe: livenessProbe,
}}
deployment := appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{Name: binoculars.Name, Namespace: binoculars.Namespace, Labels: AllLabels(binoculars.Name, binoculars.Labels)},
Expand Down Expand Up @@ -367,3 +378,21 @@ func (r *BinocularsReconciler) SetupWithManager(mgr ctrl.Manager) error {
For(&installv1alpha1.Binoculars{}).
Complete(r)
}

type BinocularsConfig struct {
Grpc GrpcConfig
}

// extractBinocularsConfig will unmarshal the appconfig and return the BinocularsConfig portion
func extractBinocularsConfig(config runtime.RawExtension) (BinocularsConfig, error) {
appConfig, err := builders.ConvertRawExtensionToYaml(config)
if err != nil {
return BinocularsConfig{}, err
}
var binocularsConfig BinocularsConfig
err = yaml.Unmarshal([]byte(appConfig), &binocularsConfig)
if err != nil {
return BinocularsConfig{}, err
}
return binocularsConfig, err
}
58 changes: 48 additions & 10 deletions internal/controller/install/common_helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,14 @@ type PulsarConfig struct {
Cacert string
}

type GrpcConfig struct {
Tls TlsConfig
}

type TlsConfig struct {
Enabled bool
}

// ArmadaInit used to initialize pulsar
type ArmadaInit struct {
Enabled bool
Expand Down Expand Up @@ -317,7 +325,16 @@ func ExtractPulsarConfig(config runtime.RawExtension) (PulsarConfig, error) {
if err != nil {
return PulsarConfig{}, err
}
return asConfig.Pulsar, nil
return asConfig.Pulsar, err
Sovietaced marked this conversation as resolved.
Show resolved Hide resolved
}

// GetServerScheme returns the URI scheme for the grpc server
func GetServerScheme(tlsConfig TlsConfig) corev1.URIScheme {
if tlsConfig.Enabled {
return corev1.URISchemeHTTPS
}

return corev1.URISchemeHTTP
}

// waitForJob waits for the Job to reach a terminal state (complete or failed).
Expand Down Expand Up @@ -371,6 +388,36 @@ func createEnv(crdEnv []corev1.EnvVar) []corev1.EnvVar {
return envVars
}

func CreateProbesWithScheme(scheme corev1.URIScheme) (*corev1.Probe, *corev1.Probe) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The values here are pretty standardized for now but we might want to support customization later.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, seems good enough for starters.

readinessProbe := &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/health",
Port: intstr.FromString("http"),
Scheme: scheme,
},
},
InitialDelaySeconds: 5,
TimeoutSeconds: 5,
FailureThreshold: 2,
}

livenessProbe := &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/health",
Port: intstr.FromString("http"),
Scheme: scheme,
},
},
InitialDelaySeconds: 10,
TimeoutSeconds: 10,
FailureThreshold: 3,
}

return readinessProbe, livenessProbe
}

// createVolumes creates the default appconfig Volume and appends the CRD AdditionalVolumes
func createVolumes(configVolumeSecretName string, crdVolumes []corev1.Volume) []corev1.Volume {
volumes := []corev1.Volume{{
Expand Down Expand Up @@ -733,15 +780,6 @@ func newContainerPortsHTTPWithMetrics(config *builders.CommonApplicationConfig)
return ports
}

// newContainerPortsGRPCWithMetrics creates container ports for grpc and metrics server and optional port for profiling server.
func newContainerPortsGRPCWithMetrics(config *builders.CommonApplicationConfig) []corev1.ContainerPort {
ports := []corev1.ContainerPort{newContainerPortGRPC(config), newContainerPortMetrics(config)}
if config.Profiling.Port > 0 {
ports = append(ports, newContainerPortProfiling(config))
}
return ports
}

// newContainerPortsMetrics creates container ports for metrics server and optional port for profiling server.
func newContainerPortsMetrics(config *builders.CommonApplicationConfig) []corev1.ContainerPort {
ports := []corev1.ContainerPort{newContainerPortMetrics(config)}
Expand Down
4 changes: 4 additions & 0 deletions internal/controller/install/executor_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,8 @@ func (r *ExecutorReconciler) createDeployment(
volumes := createVolumes(executor.Name, executor.Spec.AdditionalVolumes)
volumeMounts := createVolumeMounts(GetConfigFilename(executor.Name), executor.Spec.AdditionalVolumeMounts)

readinessProbe, livenessProbe := CreateProbesWithScheme(corev1.URISchemeHTTP)

env := []corev1.EnvVar{
{
Name: "SERVICE_ACCOUNT",
Expand Down Expand Up @@ -278,6 +280,8 @@ func (r *ExecutorReconciler) createDeployment(
Env: env,
VolumeMounts: volumeMounts,
SecurityContext: executor.Spec.SecurityContext,
ReadinessProbe: readinessProbe,
LivenessProbe: livenessProbe,
}}
deployment := appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{Name: executor.Name, Namespace: executor.Namespace, Labels: AllLabels(executor.Name, executor.Labels)},
Expand Down
31 changes: 25 additions & 6 deletions internal/controller/install/lookout_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ func (r *LookoutReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct

type LookoutConfig struct {
Postgres PostgresConfig
Tls TlsConfig
}

func generateLookoutInstallComponents(
Expand Down Expand Up @@ -271,6 +272,13 @@ func createLookoutDeployment(lookout *installv1alpha1.Lookout, serviceAccountNam
volumes := createVolumes(lookout.Name, lookout.Spec.AdditionalVolumes)
volumeMounts := createVolumeMounts(GetConfigFilename(lookout.Name), lookout.Spec.AdditionalVolumeMounts)

lookoutConfig, err := extractLookoutConfig(lookout.Spec.ApplicationConfig)
if err != nil {
return nil, err
}

readinessProbe, livenessProbe := CreateProbesWithScheme(GetServerScheme(lookoutConfig.Tls))

deployment := appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{Name: lookout.Name, Namespace: lookout.Namespace, Labels: AllLabels(lookout.Name, lookout.Labels)},
Spec: appsv1.DeploymentSpec{
Expand Down Expand Up @@ -299,6 +307,8 @@ func createLookoutDeployment(lookout *installv1alpha1.Lookout, serviceAccountNam
Env: env,
VolumeMounts: volumeMounts,
SecurityContext: lookout.Spec.SecurityContext,
ReadinessProbe: readinessProbe,
LivenessProbe: livenessProbe,
}},
Volumes: volumes,
},
Expand Down Expand Up @@ -458,12 +468,7 @@ func createLookoutCronJob(lookout *installv1alpha1.Lookout, serviceAccountName s
dbPruningSchedule = *lookout.Spec.DbPruningSchedule
}

appConfig, err := builders.ConvertRawExtensionToYaml(lookout.Spec.ApplicationConfig)
if err != nil {
return nil, err
}
var lookoutConfig LookoutConfig
err = yaml.Unmarshal([]byte(appConfig), &lookoutConfig)
lookoutConfig, err := extractLookoutConfig(lookout.Spec.ApplicationConfig)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -555,3 +560,17 @@ func (r *LookoutReconciler) SetupWithManager(mgr ctrl.Manager) error {
For(&installv1alpha1.Lookout{}).
Complete(r)
}

// extractLookoutConfig will unmarshal the appconfig and return the LookoutConfig portion
func extractLookoutConfig(config runtime.RawExtension) (LookoutConfig, error) {
appConfig, err := builders.ConvertRawExtensionToYaml(config)
if err != nil {
return LookoutConfig{}, err
}
var lookoutConfig LookoutConfig
err = yaml.Unmarshal([]byte(appConfig), &lookoutConfig)
if err != nil {
return LookoutConfig{}, err
}
return lookoutConfig, err
}
Loading
Loading