OCPEDGE-2088, OCPEDGE-1885: Updated state transitions & tests for TNF setup job

- Setup job now sets Available condition only when ExternalEtcd hasn't transitioned yet
- Fixed jobcontroller & test setup to handle variable operator status conditions
- Added test coverage for updated expectations for job controller condition transitions (Progressing, Degraded, Available)

Depends on #1484
Depends on #1483

Author: jaypoulz

pkg/operator/ceohelpers/external_etcd_status.go

@@ -123,7 +123,7 @@ func GetExternalEtcdClusterStatus(ctx context.Context,
 	// Check readiness for transition
 	externalEtcdStatus.IsReadyForEtcdTransition = v1helpers.IsOperatorConditionTrue(opStatus.Conditions, etcd.OperatorConditionExternalEtcdReadyForTransition)
 
-	// Check if etcd has completed transition
+	// Check if etcd is running externally
 	externalEtcdStatus.HasExternalEtcdCompletedTransition = v1helpers.IsOperatorConditionTrue(opStatus.Conditions, etcd.OperatorConditionExternalEtcdHasCompletedTransition)
 
 	if externalEtcdStatus.IsEtcdRunningInCluster {

pkg/operator/externaletcdsupportcontroller/external_etcd_support_controller_test.go

@@ -53,9 +53,9 @@ func TestExternalEtcdSupportController(t *testing.T) {
 			name: "ExternalEtcd cluster but bootstrap not completed",
 			staticPodStatus: testutils.StaticPodOperatorStatus(
 				testutils.WithLatestRevision(3),
-				testutils.WithNodeStatusAtCurrentRevision(2),
-				testutils.WithNodeStatusAtCurrentRevision(2),
-				testutils.WithNodeStatusAtCurrentRevision(2),
+				testutils.WithNodeStatusAtCurrentRevision(3),
+				testutils.WithNodeStatusAtCurrentRevision(3),
+				testutils.WithNodeStatusAtCurrentRevision(3),
 			),
 			expectedConfigMapExists: false,
 			expectedErr:             nil,
@@ -65,9 +65,9 @@ func TestExternalEtcdSupportController(t *testing.T) {
 			name: "ExternalEtcd cluster and bootstrap completed",
 			staticPodStatus: testutils.StaticPodOperatorStatus(
 				testutils.WithLatestRevision(3),
-				testutils.WithNodeStatusAtCurrentRevision(2),
-				testutils.WithNodeStatusAtCurrentRevision(2),
-				testutils.WithNodeStatusAtCurrentRevision(2),
+				testutils.WithNodeStatusAtCurrentRevision(3),
+				testutils.WithNodeStatusAtCurrentRevision(3),
+				testutils.WithNodeStatusAtCurrentRevision(3),
 				testutils.WithOperatorCondition("EtcdRunningInCluster", operatorv1.ConditionTrue),
 			),
 			expectedConfigMapExists: true,

pkg/operator/targetconfigcontroller/targetconfigcontroller_test.go

@@ -75,7 +75,6 @@ func TestTargetConfigController(t *testing.T) {
 		},
 		{
 			name: "BackupVar Test HappyPath",
-
 			staticPodStatus: u.StaticPodOperatorStatus(
 				u.WithLatestRevision(3),
 				u.WithNodeStatusAtCurrentRevision(3),

pkg/tnf/operator/starter.go

@@ -172,11 +172,23 @@ func handleNodes(
 
 	// the order of job creation does not matter, the jobs wait on each other as needed
 	for _, node := range nodeList {
-		runJobController(ctx, tools.JobTypeAuth, &node.Name, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces)
-		runJobController(ctx, tools.JobTypeAfterSetup, &node.Name, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces)
+		runJobController(ctx, tools.JobTypeAuth, &node.Name, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions)
+		runJobController(ctx, tools.JobTypeAfterSetup, &node.Name, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions)
 	}
-	runJobController(ctx, tools.JobTypeSetup, nil, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces)
-	runJobController(ctx, tools.JobTypeFencing, nil, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces)
+
+	hasExternalEtcdCompletedTransition, err := ceohelpers.HasExternalEtcdCompletedTransition(ctx, operatorClient)
+	if err != nil {
+		klog.Errorf("failed to get external etcd status: %v", err)
+		return
+	}
+
+	setupConditions := jobs.DefaultConditions
+	if !hasExternalEtcdCompletedTransition {
+		setupConditions = jobs.AllConditions
+	}
+
+	runJobController(ctx, tools.JobTypeSetup, nil, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, setupConditions)
+	runJobController(ctx, tools.JobTypeFencing, nil, controllerContext, operatorClient, kubeClient, kubeInformersForNamespaces, jobs.DefaultConditions)
 }
 
 func waitForEtcdBootstrapCompleted(ctx context.Context, operatorClient v1helpers.StaticPodOperatorClient) error {
@@ -246,7 +258,7 @@ func runTnfResourceController(ctx context.Context, controllerContext *controller
 	go tnfResourceController.Run(ctx, 1)
 }
 
-func runJobController(ctx context.Context, jobType tools.JobType, nodeName *string, controllerContext *controllercmd.ControllerContext, operatorClient v1helpers.StaticPodOperatorClient, kubeClient kubernetes.Interface, kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces) {
+func runJobController(ctx context.Context, jobType tools.JobType, nodeName *string, controllerContext *controllercmd.ControllerContext, operatorClient v1helpers.StaticPodOperatorClient, kubeClient kubernetes.Interface, kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces, conditions []string) {
 	nodeNameForLogs := "n/a"
 	if nodeName != nil {
 		nodeNameForLogs = *nodeName
@@ -259,6 +271,7 @@ func runJobController(ctx context.Context, jobType tools.JobType, nodeName *stri
 		operatorClient,
 		kubeClient,
 		kubeInformersForNamespaces.InformersFor(operatorclient.TargetNamespace).Batch().V1().Jobs(),
+		conditions,
 		[]factory.Informer{},
 		[]jobs.JobHookFunc{
 			func(_ *operatorv1.OperatorSpec, job *batchv1.Job) error {

pkg/tnf/operator/starter_test.go

@@ -37,6 +37,8 @@ import (
 	"github.com/openshift/cluster-etcd-operator/pkg/operator/ceohelpers"
 	"github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient"
 	u "github.com/openshift/cluster-etcd-operator/pkg/testutils"
+	"github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/etcd"
+	"github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/jobs"
 )
 
 type args struct {
@@ -111,6 +113,70 @@ func TestHandleDualReplicaClusters(t *testing.T) {
 	}
 }
 
+func TestSetupJobConditionsBasedOnExternalEtcd(t *testing.T) {
+	tests := []struct {
+		name                     string
+		isReadyForEtcdTransition bool
+		expectedAvailableInSetup bool
+	}{
+		{
+			name:                     "Job sets the Available condition when not ready for etcd transition",
+			isReadyForEtcdTransition: false,
+			expectedAvailableInSetup: true,
+		},
+		{
+			name:                     "Job does not set the Available condition when ready for etcd transition",
+			isReadyForEtcdTransition: true,
+			expectedAvailableInSetup: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Set up operator status with the appropriate condition
+			operatorStatus := &operatorv1.StaticPodOperatorStatus{}
+			if tt.isReadyForEtcdTransition {
+				operatorStatus.Conditions = []operatorv1.OperatorCondition{
+					{
+						Type:   etcd.OperatorConditionExternalEtcdHasCompletedTransition,
+						Status: operatorv1.ConditionTrue,
+					},
+				}
+			}
+
+			fakeOperatorClient := v1helpers.NewFakeStaticPodOperatorClient(
+				&operatorv1.StaticPodOperatorSpec{},
+				operatorStatus,
+				nil,
+				nil,
+			)
+
+			hasExternalEtcdCompletedTransition, err := ceohelpers.HasExternalEtcdCompletedTransition(context.Background(), fakeOperatorClient)
+			if err != nil {
+				t.Errorf("failed to get external etcd status: %v", err)
+			}
+
+			// Determine setup conditions based on the etcd transition status
+			setupConditions := jobs.DefaultConditions
+			if !hasExternalEtcdCompletedTransition {
+				setupConditions = jobs.AllConditions
+			}
+
+			hasAvailableCondition := false
+			for _, condition := range setupConditions {
+				if condition == operatorv1.OperatorStatusTypeAvailable {
+					hasAvailableCondition = true
+					break
+				}
+			}
+
+			require.Equalf(t, tt.expectedAvailableInSetup, hasAvailableCondition,
+				"Setup job should have Available condition: %v, but got: %v",
+				tt.expectedAvailableInSetup, hasAvailableCondition)
+		})
+	}
+}
+
 func getArgs(t *testing.T, dualReplicaControlPlaneEnabled bool) args {
 
 	fakeKubeClient := fake.NewSimpleClientset()

pkg/tnf/pkg/jobs/jobcontroller.go

@@ -26,6 +26,12 @@ import (
 	"github.com/openshift/cluster-etcd-operator/pkg/tnf/pkg/tools"
 )
 
+// By default, we only set the progressing and degraded conditions for TNF setup.
+// This is because the operator is only unavailable during the transition of etcd
+// ownership between CEO and pacemaker.
+var DefaultConditions = []string{opv1.OperatorStatusTypeProgressing, opv1.OperatorStatusTypeDegraded}
+var AllConditions = []string{opv1.OperatorStatusTypeAvailable, opv1.OperatorStatusTypeProgressing, opv1.OperatorStatusTypeDegraded}
+
 // TODO This based on DeploymentController in openshift/library-go
 // TODO should be moved there once it proved to be useful
 
@@ -71,6 +77,7 @@ func NewJobController(
 	operatorClient v1helpers.OperatorClient,
 	kubeClient kubernetes.Interface,
 	jobInformer batchinformersv1.JobInformer,
+	conditions []string,
 	optionalInformers []factory.Informer,
 	optionalJobHooks ...JobHookFunc,
 ) factory.Controller {
@@ -82,9 +89,7 @@ func NewJobController(
 		kubeClient,
 		jobInformer,
 	).WithConditions(
-		opv1.OperatorStatusTypeAvailable,
-		opv1.OperatorStatusTypeProgressing,
-		opv1.OperatorStatusTypeDegraded,
+		conditions...,
 	).WithExtraInformers(
 		optionalInformers...,
 	).WithJobHooks(