feat: report triggered rules for every test

Adds list of triggered rules by stage to the JSON output.

Fixes #429

check/base.go

@@ -112,6 +112,13 @@ func (c *FTWCheck) SetEndMarker(marker []byte) {
 	c.log.WithEndMarker(marker)
 }
 
+func (c *FTWCheck) GetTriggeredRules() []uint {
+	if c.CloudMode() {
+		return nil
+	}
+	return c.log.TriggeredRules()
+}
+
 func (c *FTWCheck) Close() error {
 	return c.log.Cleanup()
 }

cmd/run.go

@@ -49,6 +49,7 @@ const (
 	waitForInsecureSkipTlsVerifyFlag = "wait-for-insecure-skip-tls-verify"
 	waitForNoRedirectFlag            = "wait-for-no-redirect"
 	waitForTimeoutFlag               = "wait-for-timeout"
+	reportTriggeredRulesFlag         = "report-triggered-rules"
 )
 
 // NewRunCmd represents the run command
@@ -86,6 +87,7 @@ func NewRunCommand() *cobra.Command {
 	runCmd.Flags().Bool(waitForNoRedirectFlag, http.DefaultNoRedirect, "Do not follow HTTP 3xx redirects.")
 	runCmd.Flags().DurationP(rateLimitFlag, "r", 0, "Limit the request rate to the server to 1 request per specified duration. 0 is the default, and disables rate limiting.")
 	runCmd.Flags().Bool(failFastFlag, false, "Fail on first failed test")
+	runCmd.Flags().Bool(reportTriggeredRulesFlag, false, "Report triggered rules for each test")
 
 	return runCmd
 }

runner/run.go

@@ -88,6 +88,8 @@ func RunTest(runContext *TestRunContext, ftwTest *test.FTWTest) error {
 			runContext.Stats.addResultToStats(Skipped, &testCase)
 			continue
 		}
+		runContext.StartTest()
+
 		test.ApplyPlatformOverrides(runContext.Config, &testCase)
 		// this is just for printing once the next test
 		if changed && !runContext.ShowOnlyFailed {
@@ -116,7 +118,7 @@ func RunTest(runContext *TestRunContext, ftwTest *test.FTWTest) error {
 				}
 			}
 		}
-		runContext.Stats.addResultToStats(runContext.Result, &testCase)
+		runContext.EndTest(&testCase)
 		if runContext.RunnerConfig.FailFast && runContext.Stats.TotalFailed() > 0 {
 			break
 		}
@@ -133,7 +135,7 @@ func RunTest(runContext *TestRunContext, ftwTest *test.FTWTest) error {
 //
 //gocyclo:ignore
 func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase schema.Test, stage schema.Stage) error {
-	stageStartTime := time.Now()
+	runContext.StartStage()
 	stageId := uuid.NewString()
 	// Apply global overrides initially
 	testInput := (test.Input)(stage.Input)
@@ -152,7 +154,7 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase sch
 	// Do not even run test if result is overridden. Directly set and display the overridden result.
 	if overridden := overriddenTestResult(ftwCheck, &testCase); overridden != Failed {
 		runContext.Result = overridden
-		displayResult(&testCase, runContext, overridden, time.Duration(0), time.Duration(0))
+		displayResult(&testCase, runContext, overridden, time.Duration(0))
 		return nil
 	}
 
@@ -208,14 +210,11 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase sch
 	}
 
 	roundTripTime := runContext.Client.GetRoundTripTime().RoundTripDuration()
-	stageTime := time.Since(stageStartTime)
 
-	runContext.Result = testResult
+	runContext.EndStage(&testCase, testResult, ftwCheck.GetTriggeredRules())
 
 	// show the result unless quiet was passed in the command line
-	displayResult(&testCase, runContext, testResult, roundTripTime, stageTime)
-
-	runContext.Stats.addStageResultToStats(&testCase, stageTime)
+	displayResult(&testCase, runContext, testResult, roundTripTime)
 
 	return nil
 }
@@ -325,14 +324,14 @@ func checkTestSanity(stage *schema.Stage) error {
 	return nil
 }
 
-func displayResult(testCase *schema.Test, rc *TestRunContext, result TestResult, roundTripTime time.Duration, stageTime time.Duration) {
+func displayResult(testCase *schema.Test, rc *TestRunContext, result TestResult, roundTripTime time.Duration) {
 	switch result {
 	case Success:
 		if !rc.ShowOnlyFailed {
-			rc.Output.Println(rc.Output.Message("+ passed in %s (RTT %s)"), stageTime, roundTripTime)
+			rc.Output.Println(rc.Output.Message("+ passed in %s (RTT %s)"), rc.CurrentStageDuration, roundTripTime)
 		}
 	case Failed:
-		rc.Output.Println(rc.Output.Message("- %s failed in %s (RTT %s)"), testCase.IdString(), stageTime, roundTripTime)
+		rc.Output.Println(rc.Output.Message("- %s failed in %s (RTT %s)"), testCase.IdString(), rc.CurrentStageDuration, roundTripTime)
 	case Ignored:
 		if !rc.ShowOnlyFailed {
 			rc.Output.Println(rc.Output.Message(":information:test ignored"))

runner/run_test.go

@@ -602,3 +602,27 @@ func (s *runTestSuite) TestVirtualHostMode_True() {
 
 	s.Equal("not-localhost_virtual-host", request.Headers().Get("Host"))
 }
+
+func (s *runTestSuite) TestTriggeredRules() {
+	res, err := Run(s.cfg, s.ftwTests, &RunnerConfig{}, s.out)
+	s.Require().NoError(err)
+	triggeredRules := map[string][][]uint{
+		"123456-1": {{
+			920210,
+			920300,
+			949110,
+			980130,
+		}},
+		"123456-2": {{
+			920210,
+			920300,
+			949110,
+			980130,
+		}, {
+			920210,
+			920300,
+			949110,
+			980130,
+		}}}
+	s.Equal(triggeredRules, res.Stats.TriggeredRules, "Oops, triggered rules don't match expectation")
+}

runner/stats.go

@@ -43,20 +43,27 @@ type RunStats struct {
 	RunTime map[string]time.Duration `json:"runtime"`
 	// TotalTime is the duration over all runs, the sum of all individual run times.
 	TotalTime time.Duration
+	// TriggeredRules maps triggered rules to stages of tests
+	TriggeredRules map[string][][]uint `json:"triggered-rules"`
 }
 
+// type rulesByStage struct {
+// 	Stages map[uint][]uint `json:"stages"`
+// }
+
 // NewRunStats creates and initializes a new Stats struct.
 func NewRunStats() *RunStats {
 	return &RunStats{
-		Run:        0,
-		Success:    []string{},
-		Failed:     []string{},
-		Skipped:    []string{},
-		Ignored:    []string{},
-		ForcedPass: []string{},
-		ForcedFail: []string{},
-		RunTime:    make(map[string]time.Duration),
-		TotalTime:  0,
+		Run:            0,
+		Success:        []string{},
+		Failed:         []string{},
+		Skipped:        []string{},
+		Ignored:        []string{},
+		ForcedPass:     []string{},
+		ForcedFail:     []string{},
+		RunTime:        make(map[string]time.Duration),
+		TotalTime:      0,
+		TriggeredRules: make(map[string][][]uint),
 	}
 }
 
@@ -86,8 +93,10 @@ func (stats *RunStats) addResultToStats(result TestResult, testCase *schema.Test
 	}
 }
 
-func (stats *RunStats) addStageResultToStats(testCase *schema.Test, stageTime time.Duration) {
+func (stats *RunStats) addStageResultToStats(testCase *schema.Test, stageTime time.Duration, triggeredRules []uint) {
 	stats.RunTime[testCase.IdString()] += stageTime
+	byStage := stats.TriggeredRules[testCase.IdString()]
+	stats.TriggeredRules[testCase.IdString()] = append(byStage, triggeredRules)
 	stats.TotalTime += stageTime
 }
 

runner/testdata/TestTriggeredRules.yaml

@@ -0,0 +1,39 @@
+---
+meta:
+  author: "tester"
+  description: "Example Test"
+rule_id: 123456
+tests:
+  - test_id: 1
+    description: "tests for verifying recording of triggered rules"
+    stages:
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "none.host"
+        output:
+          status: 413
+  - test_id: 2
+    description: "access real external site"
+    stages:
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "{{ .TestAddr }}"
+        output:
+          expect_error: False
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "{{ .TestAddr }}"
+        output:
+          expect_error: False

runner/types.go

@@ -7,6 +7,7 @@ import (
 	"regexp"
 	"time"
 
+	schema "github.com/coreruleset/ftw-tests-schema/v2/types"
 	"github.com/coreruleset/go-ftw/config"
 	"github.com/coreruleset/go-ftw/ftwhttp"
 	"github.com/coreruleset/go-ftw/output"
@@ -41,17 +42,37 @@ type RunnerConfig struct {
 // This includes configuration information as well as statistics
 // and results.
 type TestRunContext struct {
-	Config         *config.FTWConfiguration
-	RunnerConfig   *RunnerConfig
-	Include        *regexp.Regexp
-	Exclude        *regexp.Regexp
-	IncludeTags    *regexp.Regexp
-	ShowTime       bool
-	ShowOnlyFailed bool
-	Output         *output.Output
-	Stats          *RunStats
-	Result         TestResult
-	Duration       time.Duration
-	Client         *ftwhttp.Client
-	LogLines       *waflog.FTWLogLines
+	Config                *config.FTWConfiguration
+	RunnerConfig          *RunnerConfig
+	Include               *regexp.Regexp
+	Exclude               *regexp.Regexp
+	IncludeTags           *regexp.Regexp
+	ShowTime              bool
+	ShowOnlyFailed        bool
+	Output                *output.Output
+	Stats                 *RunStats
+	Result                TestResult
+	Duration              time.Duration
+	Client                *ftwhttp.Client
+	LogLines              *waflog.FTWLogLines
+	CurrentStageDuration  time.Duration
+	currentStageStartTime time.Time
+}
+
+func (t *TestRunContext) StartTest() {
+}
+
+func (t *TestRunContext) EndTest(testCase *schema.Test) {
+	t.Stats.addResultToStats(t.Result, testCase)
+}
+
+func (t *TestRunContext) StartStage() {
+	t.currentStageStartTime = time.Now()
+	t.CurrentStageDuration = time.Duration(0)
+}
+
+func (t *TestRunContext) EndStage(testCase *schema.Test, testResult TestResult, triggeredRules []uint) {
+	t.CurrentStageDuration = time.Since(t.currentStageStartTime)
+	t.Result = testResult
+	t.Stats.addStageResultToStats(testCase, t.CurrentStageDuration, triggeredRules)
 }

waflog/read.go

@@ -63,6 +63,7 @@ func (ll *FTWLogLines) TriggeredRules() []uint {
 	for ruleId := range ruleIdsSet {
 		ruleIds = append(ruleIds, ruleId)
 	}
+	slices.Sort(ruleIds)
 	ll.triggeredRules = ruleIds
 	// Reset map for next use
 	for key := range ruleIdsSet {