fix: delete slurm jobs if pod's state is succeeded or failed

d-hayashi · Aug 27, 2024 · 710b4b9 · 710b4b9
1 parent 5e0cd2b
commit 710b4b9
Showing 1 changed file with 8 additions and 0 deletions.
diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go
@@ -169,6 +169,10 @@ func (w *watcher) fetchJobStateOnKubernetes() error {
 	}
 
 	for _, pod := range pods.Items {
+		// Filter out pods with state `Succeeded` or `Failed`
+		if pod.Status.Phase == "Succeeded" || pod.Status.Phase == "Failed" {
+			continue
+		}
 		annotations := pod.GetAnnotations()
 		if UUID, uuidExists := annotations["k8s-slurm-injector/uuid"]; uuidExists {
 			if jobId, jobIdExists := annotations["k8s-slurm-injector/jobid"]; jobIdExists {
@@ -201,6 +205,10 @@ func (w *watcher) fetchJobStateOnKubernetes() error {
 			if UUID, uuidExists := annotations["k8s-slurm-injector/uuid"]; uuidExists {
 				podExists := false
 				for _, pod := range pods.Items {
+					// Filter out pods with state `Succeeded` or `Failed`
+					if pod.Status.Phase == "Succeeded" || pod.Status.Phase == "Failed" {
+						continue
+					}
 					if podUUID, podUUIDExists := pod.GetAnnotations()["k8s-slurm-injector/uuid"]; podUUIDExists {
 						if podUUID == UUID {
 							podExists = true