add support for cluster downsize

we do this by way of adding backoffLimitPerIndex and setting to 0, meaning that a pod (follower broker) cannot be recreated when the pod is killed. We might want to do this for autoscaling. See the examples/elasticity/downsize for details. Signed-off-by: vsoch <[email protected]>
flux-framework · Jan 10, 2024 · 1c01f9d · 1c01f9d
1 parent 0a119ba
commit 1c01f9d
Show file tree

Hide file tree

Showing 22 changed files with 480 additions and 1,112 deletions.
diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml
@@ -13,10 +13,10 @@ jobs:
     name: Prepare chart
     steps:
     - name: Checkout Repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - uses: actions/setup-go@v3
       with:
-        go-version: ^1.18.1
+        go-version: ^1.20
     - name: GHCR Login
       if: (github.event_name != 'pull_request')
       uses: docker/login-action@v2
@@ -44,4 +44,4 @@ jobs:
         PKG_RESPONSE=$(helm package ./chart)
         echo "$PKG_RESPONSE"
         CHART_TAR_GZ=$(basename "$PKG_RESPONSE") 
-        helm push "$CHART_TAR_GZ" oci://ghcr.io/flux-framework/flux-operator-helm
+        helm push "$CHART_TAR_GZ" oci://ghcr.io/flux-framework/flux-operator-helm
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -92,7 +92,7 @@ jobs:
       run: sudo apt-get update && sudo apt-get install -y libsodium-dev libzmq3-dev libczmq-dev
 
     - name: Start minikube
-      uses: medyagh/setup-minikube@697f2b7aaed5f70bf2a94ee21a4ec3dde7b12f92 # v0.0.9
+      uses: medyagh/setup-minikube@master
 
     - name: Pull Docker Containers to MiniKube
       env:

diff --git a/Makefile b/Makefile
@@ -342,6 +342,7 @@ $(KUSTOMIZE): $(LOCALBIN)
 controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary.
 $(CONTROLLER_GEN): $(LOCALBIN)
 	GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION)
+	go mod tidy
 
 # Build the latest openapi-gen from source
 .PHONY: openapi-gen

diff --git a/api/v1alpha2/minicluster_types.go b/api/v1alpha2/minicluster_types.go
@@ -66,6 +66,12 @@ type MiniClusterSpec struct {
 	// +optional
 	ShareProcessNamespace bool `json:"shareProcessNamespace"`
 
+	// Restart failed workers (defaults to true)
+	// This is setting backoffLimitPerIndex to 0 on the backend
+	// This requires an additional feature gate to be enabled.
+	// +optional
+	SuspendWorkers bool `json:"suspendWorkers"`
+
 	// Cleanup the pods and storage when the index broker pod is complete
 	// +kubebuilder:default=false
 	// +default=false

diff --git a/api/v1alpha2/swagger.json b/api/v1alpha2/swagger.json
@@ -622,6 +622,11 @@
           "format": "int32",
           "default": 1
         },
+        "suspendWorkers": {
+          "description": "Restart failed workers (defaults to true) This is setting backoffLimitPerIndex to 0 on the backend This requires an additional feature gate to be enabled.",
+          "type": "boolean",
+          "default": false
+        },
         "tasks": {
           "description": "Total number of CPUs being run across entire cluster",
           "type": "integer",

diff --git a/api/v1alpha2/zz_generated.openapi.go b/api/v1alpha2/zz_generated.openapi.go
diff --git a/chart/templates/minicluster-crd.yaml b/chart/templates/minicluster-crd.yaml
@@ -689,6 +689,11 @@ spec:
                   pods) This is also the minimum number required to start Flux
                 format: int32
                 type: integer
+              suspendWorkers:
+                description: Restart failed workers (defaults to true) This is setting
+                  backoffLimitPerIndex to 0 on the backend This requires an additional
+                  feature gate to be enabled.
+                type: boolean
               tasks:
                 default: 1
                 description: Total number of CPUs being run across entire cluster

diff --git a/config/crd/bases/flux-framework.org_miniclusters.yaml b/config/crd/bases/flux-framework.org_miniclusters.yaml
@@ -695,6 +695,11 @@ spec:
                   in pods) This is also the minimum number required to start Flux
                 format: int32
                 type: integer
+              suspendWorkers:
+                description: Restart failed workers (defaults to true) This is setting
+                  backoffLimitPerIndex to 0 on the backend This requires an additional
+                  feature gate to be enabled.
+                type: boolean
               tasks:
                 default: 1
                 description: Total number of CPUs being run across entire cluster

diff --git a/controllers/flux/job.go b/controllers/flux/job.go
@@ -77,6 +77,12 @@ func NewMiniClusterJob(cluster *api.MiniCluster) (*batchv1.Job, error) {
 		},
 	}
 
+	// Does the user want backoff limit per index?
+	if cluster.Spec.SuspendWorkers {
+		var backoffLimitPerIndex int32 = 0
+		job.Spec.BackoffLimitPerIndex = &backoffLimitPerIndex
+	}
+
 	// Add Affinity to map one pod / node only if the user hasn't disbaled it
 	if !cluster.Spec.Network.DisableAffinity {
 		job.Spec.Template.Spec.Affinity = getAffinity(cluster)

diff --git a/controllers/flux/suite_test.go b/controllers/flux/suite_test.go
diff --git a/docs/getting_started/custom-resource-definition.md b/docs/getting_started/custom-resource-definition.md
@@ -92,6 +92,19 @@ This would be equivalent to giving a start command of `sleep infinity` however o
 (e.g., if there is a flux shutdown from within the Flux instance) the sleep command would
 not exit with a failed code.
 
+### suspendWorkers
+
+By default, when a pod fails it is attempted to restart. When you use the [JobBackoffPerIndex](https://kubernetes.io/blog/2023/08/21/kubernetes-1-28-jobapi-update/#backoff-limit-per-index) feature gate (Kubernetes 1.28) you can set this to an explicit number of failures allowed. For example, a value of 0 will mean the pod is only allowed to fail once (and not recreated).
+Since we want this primarily to be a case of "restart the workers" or "don't restart them" we expose this as a boolean.
+Setting `suspendWorkers` to true indicates that on a failure, we do not restart.
+
+```yaml
+spec:
+  suspendWorkers: true
+```
+
+This can be useful for cases of autoscaling in the down direction when you need to drain a node, and then delete the pod.
+
 ### launcher
 
 If you are using an executor that launches Flux Jobs (e.g., workflow managers such as Snakemake and Nextflow do!)

diff --git a/examples/dist/flux-operator-arm.yaml b/examples/dist/flux-operator-arm.yaml
@@ -701,6 +701,11 @@ spec:
                   in pods) This is also the minimum number required to start Flux
                 format: int32
                 type: integer
+              suspendWorkers:
+                description: Restart failed workers (defaults to true) This is setting
+                  backoffLimitPerIndex to 0 on the backend This requires an additional
+                  feature gate to be enabled.
+                type: boolean
               tasks:
                 default: 1
                 description: Total number of CPUs being run across entire cluster

diff --git a/examples/dist/flux-operator.yaml b/examples/dist/flux-operator.yaml
@@ -701,6 +701,11 @@ spec:
                   in pods) This is also the minimum number required to start Flux
                 format: int32
                 type: integer
+              suspendWorkers:
+                description: Restart failed workers (defaults to true) This is setting
+                  backoffLimitPerIndex to 0 on the backend This requires an additional
+                  feature gate to be enabled.
+                type: boolean
               tasks:
                 default: 1
                 description: Total number of CPUs being run across entire cluster