Merge pull request #17055 from serathius/abort-failpoint-failed

Abort if failpoint injecton failed
etcd-io · Dec 3, 2023 · 21704b8 · 21704b8
2 parents 9d18fb0 + 5175652
commit 21704b8
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 10 deletions.
diff --git a/tests/robustness/failpoint/failpoint.go b/tests/robustness/failpoint/failpoint.go
@@ -78,7 +78,7 @@ func Validate(clus *e2e.EtcdProcessCluster, failpoint Failpoint) error {
 	return nil
 }
 
-func Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, failpoint Failpoint) {
+func Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, failpoint Failpoint) error {
 	ctx, cancel := context.WithTimeout(ctx, triggerTimeout)
 	defer cancel()
 	var err error
@@ -89,17 +89,15 @@ func Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdPro
 
 		lg.Info("Verifying cluster health before failpoint", zap.String("failpoint", failpoint.Name()))
 		if err = verifyClusterHealth(ctx, t, clus); err != nil {
-			t.Errorf("failed to verify cluster health before failpoint injection, err: %v", err)
-			return
+			return fmt.Errorf("failed to verify cluster health before failpoint injection, err: %v", err)
 		}
 
 		lg.Info("Triggering failpoint", zap.String("failpoint", failpoint.Name()))
 		err = failpoint.Inject(ctx, t, lg, clus)
 		if err != nil {
 			select {
 			case <-ctx.Done():
-				t.Errorf("Triggering failpoints timed out, err: %v", ctx.Err())
-				return
+				return fmt.Errorf("Triggering failpoints timed out, err: %v", ctx.Err())
 			default:
 			}
 			lg.Info("Failed to trigger failpoint", zap.String("failpoint", failpoint.Name()), zap.Error(err))
@@ -109,8 +107,7 @@ func Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdPro
 
 		lg.Info("Verifying cluster health after failpoint", zap.String("failpoint", failpoint.Name()))
 		if err = verifyClusterHealth(ctx, t, clus); err != nil {
-			t.Errorf("failed to verify cluster health after failpoint injection, err: %v", err)
-			return
+			return fmt.Errorf("failed to verify cluster health after failpoint injection, err: %v", err)
 		}
 
 		successes++
@@ -119,7 +116,7 @@ func Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdPro
 		t.Errorf("failed to trigger failpoints enough times, err: %v", err)
 	}
 
-	return
+	return nil
 }
 
 func verifyClusterHealth(ctx context.Context, _ *testing.T, clus *e2e.EtcdProcessCluster) error {

diff --git a/tests/robustness/main_test.go b/tests/robustness/main_test.go
@@ -88,6 +88,8 @@ func testRobustness(ctx context.Context, t *testing.T, lg *zap.Logger, s testSce
 }
 
 func (s testScenario) run(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster) (reports []report.ClientReport) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	g := errgroup.Group{}
 	var operationReport, watchReport []report.ClientReport
 	finishTraffic := make(chan struct{})
@@ -98,15 +100,22 @@ func (s testScenario) run(ctx context.Context, t *testing.T, lg *zap.Logger, clu
 	ids := identity.NewIdProvider()
 	g.Go(func() error {
 		defer close(finishTraffic)
-		failpoint.Inject(ctx, t, lg, clus, s.failpoint)
+		err := failpoint.Inject(ctx, t, lg, clus, s.failpoint)
+		if err != nil {
+			t.Error(err)
+			cancel()
+		}
 		time.Sleep(time.Second)
+		lg.Info("Finished injecting failures")
 		return nil
 	})
 	maxRevisionChan := make(chan int64, 1)
 	g.Go(func() error {
 		defer close(maxRevisionChan)
 		operationReport = traffic.SimulateTraffic(ctx, t, lg, clus, s.profile, s.traffic, finishTraffic, baseTime, ids)
-		maxRevisionChan <- operationsMaxRevision(operationReport)
+		maxRevision := operationsMaxRevision(operationReport)
+		maxRevisionChan <- maxRevision
+		lg.Info("Finished simulating traffic", zap.Int64("max-revision", maxRevision))
 		return nil
 	})
 	g.Go(func() error {