diff --git a/check/base.go b/check/base.go index e49d0a6..9be19bc 100644 --- a/check/base.go +++ b/check/base.go @@ -112,6 +112,13 @@ func (c *FTWCheck) SetEndMarker(marker []byte) { c.log.WithEndMarker(marker) } +func (c *FTWCheck) GetTriggeredRules() []uint { + if c.CloudMode() { + return nil + } + return c.log.TriggeredRules() +} + func (c *FTWCheck) Close() error { return c.log.Cleanup() } diff --git a/cmd/run.go b/cmd/run.go index dd97532..4ff9c58 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -49,6 +49,7 @@ const ( waitForInsecureSkipTlsVerifyFlag = "wait-for-insecure-skip-tls-verify" waitForNoRedirectFlag = "wait-for-no-redirect" waitForTimeoutFlag = "wait-for-timeout" + reportTriggeredRulesFlag = "report-triggered-rules" ) // NewRunCmd represents the run command @@ -86,6 +87,7 @@ func NewRunCommand() *cobra.Command { runCmd.Flags().Bool(waitForNoRedirectFlag, http.DefaultNoRedirect, "Do not follow HTTP 3xx redirects.") runCmd.Flags().DurationP(rateLimitFlag, "r", 0, "Limit the request rate to the server to 1 request per specified duration. 0 is the default, and disables rate limiting.") runCmd.Flags().Bool(failFastFlag, false, "Fail on first failed test") + runCmd.Flags().Bool(reportTriggeredRulesFlag, false, "Report triggered rules for each test") return runCmd } diff --git a/runner/run.go b/runner/run.go index 4ddb5b5..0787b00 100644 --- a/runner/run.go +++ b/runner/run.go @@ -88,6 +88,8 @@ func RunTest(runContext *TestRunContext, ftwTest *test.FTWTest) error { runContext.Stats.addResultToStats(Skipped, &testCase) continue } + runContext.StartTest() + test.ApplyPlatformOverrides(runContext.Config, &testCase) // this is just for printing once the next test if changed && !runContext.ShowOnlyFailed { @@ -116,7 +118,7 @@ func RunTest(runContext *TestRunContext, ftwTest *test.FTWTest) error { } } } - runContext.Stats.addResultToStats(runContext.Result, &testCase) + runContext.EndTest(&testCase) if runContext.RunnerConfig.FailFast && runContext.Stats.TotalFailed() > 0 { break } @@ -133,7 +135,7 @@ func RunTest(runContext *TestRunContext, ftwTest *test.FTWTest) error { // //gocyclo:ignore func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase schema.Test, stage schema.Stage) error { - stageStartTime := time.Now() + runContext.StartStage() stageId := uuid.NewString() // Apply global overrides initially testInput := (test.Input)(stage.Input) @@ -152,7 +154,7 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase sch // Do not even run test if result is overridden. Directly set and display the overridden result. if overridden := overriddenTestResult(ftwCheck, &testCase); overridden != Failed { runContext.Result = overridden - displayResult(&testCase, runContext, overridden, time.Duration(0), time.Duration(0)) + displayResult(&testCase, runContext, overridden, time.Duration(0)) return nil } @@ -208,14 +210,11 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase sch } roundTripTime := runContext.Client.GetRoundTripTime().RoundTripDuration() - stageTime := time.Since(stageStartTime) - runContext.Result = testResult + runContext.EndStage(&testCase, testResult, ftwCheck.GetTriggeredRules()) // show the result unless quiet was passed in the command line - displayResult(&testCase, runContext, testResult, roundTripTime, stageTime) - - runContext.Stats.addStageResultToStats(&testCase, stageTime) + displayResult(&testCase, runContext, testResult, roundTripTime) return nil } @@ -325,14 +324,14 @@ func checkTestSanity(stage *schema.Stage) error { return nil } -func displayResult(testCase *schema.Test, rc *TestRunContext, result TestResult, roundTripTime time.Duration, stageTime time.Duration) { +func displayResult(testCase *schema.Test, rc *TestRunContext, result TestResult, roundTripTime time.Duration) { switch result { case Success: if !rc.ShowOnlyFailed { - rc.Output.Println(rc.Output.Message("+ passed in %s (RTT %s)"), stageTime, roundTripTime) + rc.Output.Println(rc.Output.Message("+ passed in %s (RTT %s)"), rc.CurrentStageDuration, roundTripTime) } case Failed: - rc.Output.Println(rc.Output.Message("- %s failed in %s (RTT %s)"), testCase.IdString(), stageTime, roundTripTime) + rc.Output.Println(rc.Output.Message("- %s failed in %s (RTT %s)"), testCase.IdString(), rc.CurrentStageDuration, roundTripTime) case Ignored: if !rc.ShowOnlyFailed { rc.Output.Println(rc.Output.Message(":information:test ignored")) diff --git a/runner/run_test.go b/runner/run_test.go index 72f3bad..235a0d6 100644 --- a/runner/run_test.go +++ b/runner/run_test.go @@ -602,3 +602,27 @@ func (s *runTestSuite) TestVirtualHostMode_True() { s.Equal("not-localhost_virtual-host", request.Headers().Get("Host")) } + +func (s *runTestSuite) TestTriggeredRules() { + res, err := Run(s.cfg, s.ftwTests, &RunnerConfig{}, s.out) + s.Require().NoError(err) + triggeredRules := map[string][][]uint{ + "123456-1": {{ + 920210, + 920300, + 949110, + 980130, + }}, + "123456-2": {{ + 920210, + 920300, + 949110, + 980130, + }, { + 920210, + 920300, + 949110, + 980130, + }}} + s.Equal(triggeredRules, res.Stats.TriggeredRules, "Oops, triggered rules don't match expectation") +} diff --git a/runner/stats.go b/runner/stats.go index 7436f09..28a1d96 100644 --- a/runner/stats.go +++ b/runner/stats.go @@ -43,20 +43,27 @@ type RunStats struct { RunTime map[string]time.Duration `json:"runtime"` // TotalTime is the duration over all runs, the sum of all individual run times. TotalTime time.Duration + // TriggeredRules maps triggered rules to stages of tests + TriggeredRules map[string][][]uint `json:"triggered-rules"` } +// type rulesByStage struct { +// Stages map[uint][]uint `json:"stages"` +// } + // NewRunStats creates and initializes a new Stats struct. func NewRunStats() *RunStats { return &RunStats{ - Run: 0, - Success: []string{}, - Failed: []string{}, - Skipped: []string{}, - Ignored: []string{}, - ForcedPass: []string{}, - ForcedFail: []string{}, - RunTime: make(map[string]time.Duration), - TotalTime: 0, + Run: 0, + Success: []string{}, + Failed: []string{}, + Skipped: []string{}, + Ignored: []string{}, + ForcedPass: []string{}, + ForcedFail: []string{}, + RunTime: make(map[string]time.Duration), + TotalTime: 0, + TriggeredRules: make(map[string][][]uint), } } @@ -86,8 +93,10 @@ func (stats *RunStats) addResultToStats(result TestResult, testCase *schema.Test } } -func (stats *RunStats) addStageResultToStats(testCase *schema.Test, stageTime time.Duration) { +func (stats *RunStats) addStageResultToStats(testCase *schema.Test, stageTime time.Duration, triggeredRules []uint) { stats.RunTime[testCase.IdString()] += stageTime + byStage := stats.TriggeredRules[testCase.IdString()] + stats.TriggeredRules[testCase.IdString()] = append(byStage, triggeredRules) stats.TotalTime += stageTime } diff --git a/runner/testdata/TestTriggeredRules.yaml b/runner/testdata/TestTriggeredRules.yaml new file mode 100644 index 0000000..b697565 --- /dev/null +++ b/runner/testdata/TestTriggeredRules.yaml @@ -0,0 +1,39 @@ +--- +meta: + author: "tester" + description: "Example Test" +rule_id: 123456 +tests: + - test_id: 1 + description: "tests for verifying recording of triggered rules" + stages: + - input: + dest_addr: "{{ .TestAddr }}" + port: {{ .TestPort }} + headers: + User-Agent: "ModSecurity CRS 3 Tests" + Accept: "*/*" + Host: "none.host" + output: + status: 413 + - test_id: 2 + description: "access real external site" + stages: + - input: + dest_addr: "{{ .TestAddr }}" + port: {{ .TestPort }} + headers: + User-Agent: "ModSecurity CRS 3 Tests" + Accept: "*/*" + Host: "{{ .TestAddr }}" + output: + expect_error: False + - input: + dest_addr: "{{ .TestAddr }}" + port: {{ .TestPort }} + headers: + User-Agent: "ModSecurity CRS 3 Tests" + Accept: "*/*" + Host: "{{ .TestAddr }}" + output: + expect_error: False diff --git a/runner/types.go b/runner/types.go index 2a41c59..808d4a6 100644 --- a/runner/types.go +++ b/runner/types.go @@ -7,6 +7,7 @@ import ( "regexp" "time" + schema "github.com/coreruleset/ftw-tests-schema/v2/types" "github.com/coreruleset/go-ftw/config" "github.com/coreruleset/go-ftw/ftwhttp" "github.com/coreruleset/go-ftw/output" @@ -41,17 +42,37 @@ type RunnerConfig struct { // This includes configuration information as well as statistics // and results. type TestRunContext struct { - Config *config.FTWConfiguration - RunnerConfig *RunnerConfig - Include *regexp.Regexp - Exclude *regexp.Regexp - IncludeTags *regexp.Regexp - ShowTime bool - ShowOnlyFailed bool - Output *output.Output - Stats *RunStats - Result TestResult - Duration time.Duration - Client *ftwhttp.Client - LogLines *waflog.FTWLogLines + Config *config.FTWConfiguration + RunnerConfig *RunnerConfig + Include *regexp.Regexp + Exclude *regexp.Regexp + IncludeTags *regexp.Regexp + ShowTime bool + ShowOnlyFailed bool + Output *output.Output + Stats *RunStats + Result TestResult + Duration time.Duration + Client *ftwhttp.Client + LogLines *waflog.FTWLogLines + CurrentStageDuration time.Duration + currentStageStartTime time.Time +} + +func (t *TestRunContext) StartTest() { +} + +func (t *TestRunContext) EndTest(testCase *schema.Test) { + t.Stats.addResultToStats(t.Result, testCase) +} + +func (t *TestRunContext) StartStage() { + t.currentStageStartTime = time.Now() + t.CurrentStageDuration = time.Duration(0) +} + +func (t *TestRunContext) EndStage(testCase *schema.Test, testResult TestResult, triggeredRules []uint) { + t.CurrentStageDuration = time.Since(t.currentStageStartTime) + t.Result = testResult + t.Stats.addStageResultToStats(testCase, t.CurrentStageDuration, triggeredRules) } diff --git a/waflog/read.go b/waflog/read.go index dc07569..88dfede 100644 --- a/waflog/read.go +++ b/waflog/read.go @@ -63,6 +63,7 @@ func (ll *FTWLogLines) TriggeredRules() []uint { for ruleId := range ruleIdsSet { ruleIds = append(ruleIds, ruleId) } + slices.Sort(ruleIds) ll.triggeredRules = ruleIds // Reset map for next use for key := range ruleIdsSet {