Always make use of the no_wait flag in the exclude controller to redu…

…ce the noise from exclusions that timeout (#2175)

controllers/exclude_processes.go

@@ -189,7 +189,11 @@ func (e excludeProcesses) reconcile(ctx context.Context, r *FoundationDBClusterR
 	}
 
 	r.Recorder.Event(cluster, corev1.EventTypeNormal, "ExcludingProcesses", fmt.Sprintf("Excluding %v", fdbProcessesToExclude))
-	err = adminClient.ExcludeProcesses(fdbProcessesToExclude)
+	// We use the no_wait exclusion here to trigger the exclusion without waiting for the data movement to complete.
+	// There is no need to wait for the data movement to complete in this call as later calls will verify that the
+	// data is moved and the processes are fully excluded. Using the no_wait flag here will reduce the timeout errors
+	// as those are hit most of the time if at least one storage process is included in the exclusion list.
+	err = adminClient.ExcludeProcessesWithNoWait(fdbProcessesToExclude, true)
 	if err != nil {
 		return &requeue{curError: err, delayedRequeue: true}
 	}
@@ -206,7 +210,7 @@ func (e excludeProcesses) reconcile(ctx context.Context, r *FoundationDBClusterR
 		}
 	}
 
-	// Reset the SecondsSinceLastRecovered sine the operator just excluded some processes, which will cause a recovery.
+	// Reset the SecondsSinceLastRecovered since the operator just excluded some processes, which will cause a recovery.
 	status.Cluster.RecoveryState.SecondsSinceLastRecovered = 0.0
 
 	return nil

fdbclient/admin_client.go

@@ -377,6 +377,12 @@ func (client *cliAdminClient) ResetMaintenanceMode() error {
 
 // ExcludeProcesses starts evacuating processes so that they can be removed from the database.
 func (client *cliAdminClient) ExcludeProcesses(addresses []fdbv1beta2.ProcessAddress) error {
+	return client.ExcludeProcessesWithNoWait(addresses, client.Cluster.GetUseNonBlockingExcludes())
+}
+
+// ExcludeProcessesWithNoWait starts evacuating processes so that they can be removed from the database. If noWait is
+// set to true, the exclude command will not block until all data is moved away from the processes.
+func (client *cliAdminClient) ExcludeProcessesWithNoWait(addresses []fdbv1beta2.ProcessAddress, noWait bool) error {
 	if len(addresses) == 0 {
 		return nil
 	}
@@ -388,7 +394,7 @@ func (client *cliAdminClient) ExcludeProcesses(addresses []fdbv1beta2.ProcessAdd
 
 	var excludeCommand strings.Builder
 	excludeCommand.WriteString("exclude ")
-	if version.HasNonBlockingExcludes(client.Cluster.GetUseNonBlockingExcludes()) {
+	if version.HasNonBlockingExcludes(noWait) {
 		excludeCommand.WriteString("no_wait ")
 	}
 

pkg/fdbadminclient/admin_client.go

@@ -38,6 +38,10 @@ type AdminClient interface {
 	// from the database.
 	ExcludeProcesses(addresses []fdbv1beta2.ProcessAddress) error
 
+	// ExcludeProcessesWithNoWait starts evacuating processes so that they can be removed from the database. If noWait is
+	// set to true, the exclude command will not block until all data is moved away from the processes.
+	ExcludeProcessesWithNoWait(addresses []fdbv1beta2.ProcessAddress, noWait bool) error
+
 	// IncludeProcesses removes processes from the exclusion list and allows
 	// them to take on roles again.
 	IncludeProcesses(addresses []fdbv1beta2.ProcessAddress) error

pkg/fdbadminclient/mock/admin_client_mock.go

@@ -583,6 +583,12 @@ func (client *AdminClient) ExcludeProcesses(addresses []fdbv1beta2.ProcessAddres
 	return nil
 }
 
+// ExcludeProcessesWithNoWait starts evacuating processes so that they can be removed from the database. If noWait is
+// set to true, the exclude command will not block until all data is moved away from the processes.
+func (client *AdminClient) ExcludeProcessesWithNoWait(addresses []fdbv1beta2.ProcessAddress, _ bool) error {
+	return client.ExcludeProcesses(addresses)
+}
+
 // IncludeProcesses removes processes from the exclusion list and allows
 // them to take on roles again.
 func (client *AdminClient) IncludeProcesses(addresses []fdbv1beta2.ProcessAddress) error {