From 5175652a8e29261a9111874113544a8d72723cd0 Mon Sep 17 00:00:00 2001
From: Marek Siarkowicz <siarkowicz@google.com>
Date: Sun, 3 Dec 2023 17:24:53 +0100
Subject: [PATCH] Abort if failpoint injecton failed

If one of nodes is unhealthy the test would never finish as watchers
would never reach max revision.

Signed-off-by: Marek Siarkowicz <siarkowicz@google.com>
---
 tests/robustness/failpoint/failpoint.go | 13 +++++--------
 tests/robustness/main_test.go           | 13 +++++++++++--
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/tests/robustness/failpoint/failpoint.go b/tests/robustness/failpoint/failpoint.go
index 4109973f422..de5bfe53e9f 100644
--- a/tests/robustness/failpoint/failpoint.go
+++ b/tests/robustness/failpoint/failpoint.go
@@ -78,7 +78,7 @@ func Validate(clus *e2e.EtcdProcessCluster, failpoint Failpoint) error {
 	return nil
 }
 
-func Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, failpoint Failpoint) {
+func Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, failpoint Failpoint) error {
 	ctx, cancel := context.WithTimeout(ctx, triggerTimeout)
 	defer cancel()
 	var err error
@@ -89,8 +89,7 @@ func Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdPro
 
 		lg.Info("Verifying cluster health before failpoint", zap.String("failpoint", failpoint.Name()))
 		if err = verifyClusterHealth(ctx, t, clus); err != nil {
-			t.Errorf("failed to verify cluster health before failpoint injection, err: %v", err)
-			return
+			return fmt.Errorf("failed to verify cluster health before failpoint injection, err: %v", err)
 		}
 
 		lg.Info("Triggering failpoint", zap.String("failpoint", failpoint.Name()))
@@ -98,8 +97,7 @@ func Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdPro
 		if err != nil {
 			select {
 			case <-ctx.Done():
-				t.Errorf("Triggering failpoints timed out, err: %v", ctx.Err())
-				return
+				return fmt.Errorf("Triggering failpoints timed out, err: %v", ctx.Err())
 			default:
 			}
 			lg.Info("Failed to trigger failpoint", zap.String("failpoint", failpoint.Name()), zap.Error(err))
@@ -109,8 +107,7 @@ func Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdPro
 
 		lg.Info("Verifying cluster health after failpoint", zap.String("failpoint", failpoint.Name()))
 		if err = verifyClusterHealth(ctx, t, clus); err != nil {
-			t.Errorf("failed to verify cluster health after failpoint injection, err: %v", err)
-			return
+			return fmt.Errorf("failed to verify cluster health after failpoint injection, err: %v", err)
 		}
 
 		successes++
@@ -119,7 +116,7 @@ func Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdPro
 		t.Errorf("failed to trigger failpoints enough times, err: %v", err)
 	}
 
-	return
+	return nil
 }
 
 func verifyClusterHealth(ctx context.Context, _ *testing.T, clus *e2e.EtcdProcessCluster) error {
diff --git a/tests/robustness/main_test.go b/tests/robustness/main_test.go
index 82f074518af..b674c3ed750 100644
--- a/tests/robustness/main_test.go
+++ b/tests/robustness/main_test.go
@@ -88,6 +88,8 @@ func testRobustness(ctx context.Context, t *testing.T, lg *zap.Logger, s testSce
 }
 
 func (s testScenario) run(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster) (reports []report.ClientReport) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	g := errgroup.Group{}
 	var operationReport, watchReport []report.ClientReport
 	finishTraffic := make(chan struct{})
@@ -98,15 +100,22 @@ func (s testScenario) run(ctx context.Context, t *testing.T, lg *zap.Logger, clu
 	ids := identity.NewIdProvider()
 	g.Go(func() error {
 		defer close(finishTraffic)
-		failpoint.Inject(ctx, t, lg, clus, s.failpoint)
+		err := failpoint.Inject(ctx, t, lg, clus, s.failpoint)
+		if err != nil {
+			t.Error(err)
+			cancel()
+		}
 		time.Sleep(time.Second)
+		lg.Info("Finished injecting failures")
 		return nil
 	})
 	maxRevisionChan := make(chan int64, 1)
 	g.Go(func() error {
 		defer close(maxRevisionChan)
 		operationReport = traffic.SimulateTraffic(ctx, t, lg, clus, s.profile, s.traffic, finishTraffic, baseTime, ids)
-		maxRevisionChan <- operationsMaxRevision(operationReport)
+		maxRevision := operationsMaxRevision(operationReport)
+		maxRevisionChan <- maxRevision
+		lg.Info("Finished simulating traffic", zap.Int64("max-revision", maxRevision))
 		return nil
 	})
 	g.Go(func() error {