fix: terminate sync if in progress (#4946)

* terminate operation before sync

* using method for checking manual sync

* terminating only in case of error

* handling termination error

* updating sync function

client/argocdServer/ArgoClientWrapperService.go

@@ -12,6 +12,7 @@ import (
 	"github.com/devtron-labs/devtron/client/argocdServer/application"
 	"github.com/devtron-labs/devtron/client/argocdServer/bean"
 	"github.com/devtron-labs/devtron/client/argocdServer/repository"
+	"github.com/devtron-labs/devtron/internal/util"
 	"github.com/devtron-labs/devtron/pkg/deployment/gitOps/config"
 	"github.com/devtron-labs/devtron/pkg/deployment/gitOps/git"
 	"github.com/devtron-labs/devtron/util/retryFunc"
@@ -25,6 +26,14 @@ type ACDConfig struct {
 	ArgoCDAutoSyncEnabled bool `env:"ARGO_AUTO_SYNC_ENABLED" envDefault:"true"` // will gradually switch this flag to false in enterprise
 }
 
+func (config *ACDConfig) IsManualSyncEnabled() bool {
+	return config.ArgoCDAutoSyncEnabled == false
+}
+
+func (config *ACDConfig) IsAutoSyncEnabled() bool {
+	return config.ArgoCDAutoSyncEnabled == true
+}
+
 func GetACDDeploymentConfig() (*ACDConfig, error) {
 	cfg := &ACDConfig{}
 	err := env.Parse(cfg)
@@ -34,6 +43,10 @@ func GetACDDeploymentConfig() (*ACDConfig, error) {
 	return cfg, err
 }
 
+const (
+	ErrorOperationAlreadyInProgress = "another operation is already in progress" // this string is returned from argocd
+)
+
 type ArgoClientWrapperService interface {
 
 	// GetArgoAppWithNormalRefresh - refresh app at argocd side
@@ -96,17 +109,45 @@ func (impl *ArgoClientWrapperServiceImpl) GetArgoAppWithNormalRefresh(context co
 }
 
 func (impl *ArgoClientWrapperServiceImpl) SyncArgoCDApplicationIfNeededAndRefresh(context context.Context, argoAppName string) error {
+
 	impl.logger.Info("argocd manual sync for app started", "argoAppName", argoAppName)
-	if !impl.ACDConfig.ArgoCDAutoSyncEnabled {
+	if impl.ACDConfig.IsManualSyncEnabled() {
+
 		impl.logger.Debugw("syncing argocd app as manual sync is enabled", "argoAppName", argoAppName)
 		revision := "master"
 		pruneResources := true
-		_, syncErr := impl.acdClient.Sync(context, &application2.ApplicationSyncRequest{Name: &argoAppName, Revision: &revision, Prune: &pruneResources})
+		_, syncErr := impl.acdClient.Sync(context, &application2.ApplicationSyncRequest{Name: &argoAppName,
+			Revision: &revision,
+			Prune:    &pruneResources,
+		})
 		if syncErr != nil {
-			impl.logger.Errorw("cannot get application with refresh", "app", argoAppName)
-			return syncErr
+			impl.logger.Errorw("error in syncing argoCD app", "app", argoAppName, "err", syncErr)
+			statusCode, msg := util.GetClientDetailedError(syncErr)
+			if statusCode.IsFailedPreconditionCode() && msg == ErrorOperationAlreadyInProgress {
+				impl.logger.Info("terminating ongoing sync operation and retrying manual sync", "argoAppName", argoAppName)
+				_, terminationErr := impl.acdClient.TerminateOperation(context, &application2.OperationTerminateRequest{
+					Name: &argoAppName,
+				})
+				if terminationErr != nil {
+					impl.logger.Errorw("error in terminating sync operation")
+					return fmt.Errorf("error in terminating existing sync, err: %w", terminationErr)
+				}
+				_, syncErr = impl.acdClient.Sync(context, &application2.ApplicationSyncRequest{Name: &argoAppName,
+					Revision: &revision,
+					Prune:    &pruneResources,
+					RetryStrategy: &v1alpha1.RetryStrategy{
+						Limit: 1,
+					},
+				})
+				if syncErr != nil {
+					impl.logger.Errorw("error in syncing argoCD app", "app", argoAppName, "err", syncErr)
+					return syncErr
+				}
+			} else {
+				return syncErr
+			}
 		}
-		impl.logger.Debugw("argocd sync completed", "argoAppName", argoAppName)
+		impl.logger.Infow("argocd sync completed", "argoAppName", argoAppName)
 	}
 	refreshErr := impl.GetArgoAppWithNormalRefresh(context, argoAppName)
 	if refreshErr != nil {
@@ -129,10 +170,9 @@ func (impl *ArgoClientWrapperServiceImpl) UpdateArgoCDSyncModeIfNeeded(ctx conte
 }
 
 func (impl *ArgoClientWrapperServiceImpl) isArgoAppSyncModeMigrationNeeded(argoApplication *v1alpha1.Application) bool {
-	if !impl.ACDConfig.ArgoCDAutoSyncEnabled && argoApplication.Spec.SyncPolicy.Automated != nil {
+	if impl.ACDConfig.IsManualSyncEnabled() && argoApplication.Spec.SyncPolicy.Automated != nil {
 		return true
-	}
-	if impl.ACDConfig.ArgoCDAutoSyncEnabled && argoApplication.Spec.SyncPolicy.Automated == nil {
+	} else if impl.ACDConfig.IsAutoSyncEnabled() && argoApplication.Spec.SyncPolicy.Automated == nil {
 		return true
 	}
 	return false
@@ -141,7 +181,7 @@ func (impl *ArgoClientWrapperServiceImpl) isArgoAppSyncModeMigrationNeeded(argoA
 func (impl *ArgoClientWrapperServiceImpl) CreateRequestForArgoCDSyncModeUpdateRequest(argoApplication *v1alpha1.Application) *v1alpha1.Application {
 	// set automated field in update request
 	var automated *v1alpha1.SyncPolicyAutomated
-	if impl.ACDConfig.ArgoCDAutoSyncEnabled {
+	if impl.ACDConfig.IsAutoSyncEnabled() {
 		automated = &v1alpha1.SyncPolicyAutomated{
 			Prune: true,
 		}

client/argocdServer/application/Application.go

@@ -51,6 +51,8 @@ type ServiceClient interface {
 
 	// Delete deletes an application
 	Delete(ctx context.Context, query *application.ApplicationDeleteRequest) (*application.ApplicationResponse, error)
+
+	TerminateOperation(ctx context.Context, query *application.OperationTerminateRequest) (*application.OperationTerminateResponse, error)
 }
 
 type ServiceClientImpl struct {
@@ -349,3 +351,17 @@ func (c ServiceClientImpl) buildPodMetadata(resp *v1alpha1.ApplicationTree, resp
 	}
 	return
 }
+
+func (c ServiceClientImpl) TerminateOperation(ctxt context.Context, query *application.OperationTerminateRequest) (*application.OperationTerminateResponse, error) {
+	ctx, cancel := context.WithTimeout(ctxt, argoApplication.TimeoutFast)
+	defer cancel()
+	token, ok := ctxt.Value("token").(string)
+	if !ok {
+		return nil, argoApplication.NewErrUnauthorized("Unauthorized")
+	}
+	conn := c.argoCDConnectionManager.GetConnection(token)
+	defer util.Close(conn, c.logger)
+	asc := application.NewApplicationServiceClient(conn)
+	resp, err := asc.TerminateOperation(ctx, query)
+	return resp, err
+}

pkg/app/AppService.go

@@ -442,7 +442,7 @@ func (impl *AppServiceImpl) CheckIfPipelineUpdateEventIsValidForAppStore(gitOpsA
 		// drop event
 		return isValid, installedAppVersionHistory, appId, envId, nil
 	}
-	if !impl.acdConfig.ArgoCDAutoSyncEnabled {
+	if impl.acdConfig.IsManualSyncEnabled() {
 		isArgoAppSynced := impl.pipelineStatusTimelineService.GetArgoAppSyncStatusForAppStore(installedAppVersionHistory.Id)
 		if !isArgoAppSynced {
 			return isValid, installedAppVersionHistory, appId, envId, nil
@@ -491,7 +491,7 @@ func (impl *AppServiceImpl) CheckIfPipelineUpdateEventIsValid(argoAppName, gitHa
 		// drop event
 		return isValid, pipeline, cdWfr, pipelineOverride, nil
 	}
-	if !impl.acdConfig.ArgoCDAutoSyncEnabled {
+	if impl.acdConfig.IsManualSyncEnabled() {
 		// if manual sync, proceed only if ARGOCD_SYNC_COMPLETED timeline is created
 		isArgoAppSynced := impl.pipelineStatusTimelineService.GetArgoAppSyncStatus(cdWfr.Id)
 		if !isArgoAppSynced {

pkg/app/ManifestPushService.go

@@ -148,7 +148,7 @@ func (impl *GitOpsManifestPushServiceImpl) PushChart(manifestPushTemplate *bean.
 	gitCommitTimeline := impl.pipelineStatusTimelineService.GetTimelineDbObjectByTimelineStatusAndTimelineDescription(manifestPushTemplate.WorkflowRunnerId, 0, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, "Git commit done successfully.", manifestPushTemplate.UserId, time.Now())
 
 	timelines := []*pipelineConfig.PipelineStatusTimeline{gitCommitTimeline}
-	if !impl.acdConfig.ArgoCDAutoSyncEnabled {
+	if impl.acdConfig.IsManualSyncEnabled() {
 		// if manual sync is enabled, add ARGOCD_SYNC_INITIATED_TIMELINE
 		argoCDSyncInitiatedTimeline := impl.pipelineStatusTimelineService.GetTimelineDbObjectByTimelineStatusAndTimelineDescription(manifestPushTemplate.WorkflowRunnerId, 0, pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED, "argocd sync initiated.", manifestPushTemplate.UserId, time.Now())
 		timelines = append(timelines, argoCDSyncInitiatedTimeline)

pkg/appStore/chartGroup/ChartGroupService.go

@@ -993,7 +993,7 @@ func (impl *ChartGroupServiceImpl) performDeployStageOnAcd(installedAppVersion *
 			GetTimelineDbObjectByTimelineStatusAndTimelineDescription(0, installedAppVersion.InstalledAppVersionHistoryId, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, "Git commit done successfully.", installedAppVersion.UserId, time.Now())
 
 		timelines := []*pipelineConfig.PipelineStatusTimeline{GitCommitSuccessTimeline}
-		if !impl.acdConfig.ArgoCDAutoSyncEnabled {
+		if impl.acdConfig.IsManualSyncEnabled() {
 			ArgocdSyncInitiatedTimeline := impl.pipelineStatusTimelineService.
 				GetTimelineDbObjectByTimelineStatusAndTimelineDescription(0, installedAppVersion.InstalledAppVersionHistoryId, pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED, "ArgoCD sync initiated.", installedAppVersion.UserId, time.Now())
 

pkg/appStore/installedApp/service/AppStoreDeploymentService.go

@@ -164,7 +164,7 @@ func (impl *AppStoreDeploymentServiceImpl) InstallApp(installAppVersionRequest *
 		}
 		if util.IsAcdApp(installAppVersionRequest.DeploymentAppType) {
 			_ = impl.fullModeDeploymentService.SaveTimelineForHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, "Git commit done successfully.", time.Now(), tx)
-			if !impl.aCDConfig.ArgoCDAutoSyncEnabled {
+			if impl.aCDConfig.IsManualSyncEnabled() {
 				_ = impl.fullModeDeploymentService.SaveTimelineForHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED, "argocd sync initiated.", time.Now(), tx)
 			}
 		}
@@ -695,7 +695,7 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex
 
 		upgradeAppRequest.GitHash = gitOpsResponse.GitHash
 		_ = impl.fullModeDeploymentService.SaveTimelineForHelmApps(upgradeAppRequest, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, "Git commit done successfully.", time.Now(), tx)
-		if !impl.aCDConfig.ArgoCDAutoSyncEnabled {
+		if impl.aCDConfig.IsManualSyncEnabled() {
 			_ = impl.fullModeDeploymentService.SaveTimelineForHelmApps(upgradeAppRequest, pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED, "Argocd sync initiated", time.Now(), tx)
 		}
 		installedAppVersionHistory.GitHash = gitOpsResponse.GitHash

pkg/appStore/installedApp/service/FullMode/deployment/FullModeDeploymentService.go

@@ -156,7 +156,7 @@ func (impl *FullModeDeploymentServiceImpl) InstallApp(installAppVersionRequest *
 		impl.Logger.Errorw("error in getting the argo application with normal refresh", "err", err)
 		return nil, err
 	}
-	if !impl.acdConfig.ArgoCDAutoSyncEnabled {
+	if impl.acdConfig.IsManualSyncEnabled() {
 		timeline := &pipelineConfig.PipelineStatusTimeline{
 			InstalledAppVersionHistoryId: installAppVersionRequest.InstalledAppVersionHistoryId,
 			Status:                       pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED,
@@ -304,7 +304,7 @@ func (impl *FullModeDeploymentServiceImpl) RollbackRelease(ctx context.Context,
 		return installedApp, false, err
 	}
 
-	isManualSync := !impl.acdConfig.ArgoCDAutoSyncEnabled
+	isManualSync := impl.acdConfig.IsManualSyncEnabled()
 
 	GitCommitSuccessTimeline := impl.pipelineStatusTimelineService.
 		GetTimelineDbObjectByTimelineStatusAndTimelineDescription(0, installedApp.InstalledAppVersionHistoryId, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, "Git commit done successfully.", installedApp.UserId, time.Now())

pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppArgoCdService.go

@@ -127,7 +127,7 @@ func (impl *FullModeDeploymentServiceImpl) UpdateAndSyncACDApps(installAppVersio
 		}
 		return err
 	}
-	if !impl.acdConfig.ArgoCDAutoSyncEnabled {
+	if impl.acdConfig.IsManualSyncEnabled() {
 		err = impl.SaveTimelineForHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED, "argocd sync completed", syncTime, tx)
 		if err != nil {
 			impl.Logger.Errorw("error in saving timeline for acd helm apps", "err", err)

pkg/deployment/trigger/devtronApps/TriggerService.go

@@ -1041,7 +1041,7 @@ func (impl *TriggerServiceImpl) deployArgocdApp(overrideRequest *bean3.ValuesOve
 		impl.logger.Errorw("error in getting argo application with normal refresh", "argoAppName", valuesOverrideResponse.Pipeline.DeploymentAppName)
 		return fmt.Errorf("%s. err: %s", bean.ARGOCD_SYNC_ERROR, err.Error())
 	}
-	if !impl.ACDConfig.ArgoCDAutoSyncEnabled {
+	if impl.ACDConfig.IsManualSyncEnabled() {
 		timeline := &pipelineConfig.PipelineStatusTimeline{
 			CdWorkflowRunnerId: overrideRequest.WfrId,
 			StatusTime:         syncTime,

pkg/workflow/status/WorkflowStatusService.go

@@ -193,7 +193,7 @@ func (impl *WorkflowStatusServiceImpl) UpdatePipelineTimelineAndStatusByLiveAppl
 			return nil, isTimelineUpdated
 		}
 
-		if !impl.acdConfig.ArgoCDAutoSyncEnabled {
+		if impl.acdConfig.IsManualSyncEnabled() {
 			// if manual sync check for application sync status
 			isArgoAppSynced := impl.pipelineStatusTimelineService.GetArgoAppSyncStatus(cdWfr.Id)
 			if !isArgoAppSynced {
@@ -277,7 +277,7 @@ func (impl *WorkflowStatusServiceImpl) UpdatePipelineTimelineAndStatusByLiveAppl
 			// drop event
 			return nil, isTimelineUpdated
 		}
-		if !impl.acdConfig.ArgoCDAutoSyncEnabled {
+		if impl.acdConfig.IsManualSyncEnabled() {
 			isArgoAppSynced := impl.pipelineStatusTimelineService.GetArgoAppSyncStatusForAppStore(installedAppVersionHistory.Id)
 			if !isArgoAppSynced {
 				return nil, isTimelineUpdated
@@ -469,7 +469,7 @@ func (impl *WorkflowStatusServiceImpl) CheckArgoPipelineTimelineStatusPeriodical
 }
 
 func (impl *WorkflowStatusServiceImpl) syncACDDevtronApps(deployedBeforeMinutes int, pipelineId int) error {
-	if impl.acdConfig.ArgoCDAutoSyncEnabled {
+	if impl.acdConfig.IsAutoSyncEnabled() {
 		// don't check for apps if auto sync is enabled
 		return nil
 	}
@@ -528,7 +528,7 @@ func (impl *WorkflowStatusServiceImpl) syncACDDevtronApps(deployedBeforeMinutes
 }
 
 func (impl *WorkflowStatusServiceImpl) syncACDHelmApps(deployedBeforeMinutes int, installedAppVersionId int) error {
-	if impl.acdConfig.ArgoCDAutoSyncEnabled {
+	if impl.acdConfig.IsAutoSyncEnabled() {
 		// don't check for apps if auto sync is enabled
 		return nil
 	}