Backoff on etcd errors (#5710)

Signed-off-by: Haytham Abuelfutuh <[email protected]>
flyteorg · Sep 23, 2024 · 0c3782b · 0c3782b
1 parent 28f65b3
commit 0c3782b
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/flytepropeller/pkg/controller/nodes/task/backoff/handler.go b/flytepropeller/pkg/controller/nodes/task/backoff/handler.go
@@ -195,8 +195,12 @@ func IsResourceQuotaExceeded(err error) bool {
 	return apiErrors.IsForbidden(err) && strings.Contains(err.Error(), "exceeded quota")
 }
 
+func IsEtcdError(err error) bool {
+	return apiErrors.IsForbidden(err) && strings.Contains(err.Error(), "etcdserver:")
+}
+
 func IsBackOffError(err error) bool {
-	return IsResourceQuotaExceeded(err) || apiErrors.IsTooManyRequests(err) || apiErrors.IsServerTimeout(err)
+	return IsResourceQuotaExceeded(err) || apiErrors.IsTooManyRequests(err) || apiErrors.IsServerTimeout(err) || IsEtcdError(err)
 }
 
 func GetComputeResourceAndQuantity(err error, resourceRegex *regexp.Regexp) v1.ResourceList {