feat: implement backoff when deleting artifacts. Fixes argoproj#9294. (…

…argoproj#10088)

Signed-off-by: Dillen Padhiar <[email protected]>

cmd/argoexec/commands/artifact/delete.go

@@ -16,6 +16,8 @@ import (
 	"github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
 	workflow "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned"
 	wfv1alpha1 "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
+	"github.com/argoproj/argo-workflows/v3/util/retry"
+	waitutil "github.com/argoproj/argo-workflows/v3/util/wait"
 	executor "github.com/argoproj/argo-workflows/v3/workflow/artifacts"
 	"github.com/argoproj/argo-workflows/v3/workflow/common"
 )
@@ -82,13 +84,16 @@ func deleteArtifacts(labelSelector string, ctx context.Context, artifactGCTaskIn
 					return err
 				}
 
-				err = drv.Delete(&artifact)
-				if err != nil {
-					errString := err.Error()
-					artResultNodeStatus.ArtifactResults[artifact.Name] = v1alpha1.ArtifactResult{Name: artifact.Name, Success: false, Error: &errString}
-				} else {
+				err = waitutil.Backoff(retry.DefaultRetry, func() (bool, error) {
+					err = drv.Delete(&artifact)
+					if err != nil {
+						errString := err.Error()
+						artResultNodeStatus.ArtifactResults[artifact.Name] = v1alpha1.ArtifactResult{Name: artifact.Name, Success: false, Error: &errString}
+						return false, err
+					}
 					artResultNodeStatus.ArtifactResults[artifact.Name] = v1alpha1.ArtifactResult{Name: artifact.Name, Success: true, Error: nil}
-				}
+					return true, err
+				})
 			}
 
 			task.Status.ArtifactResultsByNode[nodeName] = artResultNodeStatus