From 159b2529e85c7a963a7bf173f7a4f114eae569b9 Mon Sep 17 00:00:00 2001 From: Timo Stamm Date: Thu, 19 Dec 2024 16:05:16 +0100 Subject: [PATCH] Add all conformance runner flags to the documentation --- docs/conformance.md | 26 ++++++++++++----------- tools/protovalidate-conformance/config.go | 2 +- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/docs/conformance.md b/docs/conformance.md index 8fda2573..6178f284 100644 --- a/docs/conformance.md +++ b/docs/conformance.md @@ -20,17 +20,19 @@ make conformance You can customize the behavior of the tests using the following flags: -| Flag | Description | Default Value | -|------------------------|------------------------------------------------------|---------------| -| `--suite ` | Filter suites using the provided regular expression. | None | -| `--case ` | Filter cases using the provided regular expression. | None | -| `--timeout ` | Set a per-suite timeout. | 5 seconds | -| `--verbose`, `-v` | Enable verbose output. | `false` | -| `--strict` | Enable strict mode. | `false` | -| `--json` | Return results as JSON to stdout. | `false` | -| `--proto` | Return results as binary serialized proto to stdout. | `false` | -| `--dump` | Output the expected results, without a command. | `false` | -| `--expected_failures` | `.yaml` file containing list of expected failures | None | +| Flag | Description | Default Value | +|------------------------|-----------------------------------------------------------------------------------------------------|---------------| +| `--suite ` | Filter suites using the provided regular expression. | None | +| `--case ` | Filter cases using the provided regular expression. | None | +| `--timeout ` | Set a per-suite timeout. | 5 seconds | +| `--verbose`, `-v` | Enable verbose output. | `false` | +| `--strict` | Enable strict mode. Require that the violation type is an exact match. | `false` | +| `--strict_message` | Require that violation messages match the expected message exactly. | `false` | +| `--strict_error` | Require that compile-time errors are distinguished from runtime errors. | `false` | +| `--json` | Return results as JSON serialized message `buf.validate.conformance.harness.ResultSet` to stdout. | `false` | +| `--proto` | Return results as binary serialized message `buf.validate.conformance.harness.ResultSet` to stdout. | `false` | +| `--dump` | Output the expected results, without a command. | `false` | +| `--expected_failures` | `.yaml` file containing list of expected failures | None | ## Components @@ -59,7 +61,7 @@ For a reference implementation, see [Go's conformance test executor][pv-go]. The test messages are defined in [`.proto` files][cases-proto]. These are utilized to make concrete message _test cases_ which are organized into groups known as [_test suites_][suites]. Which tests are passed to the executor can be -controlled through the `-suite` and `-case` flags mentioned above. +controlled through the `--suite` and `--case` flags mentioned above. ### Expected Failures diff --git a/tools/protovalidate-conformance/config.go b/tools/protovalidate-conformance/config.go index d594e4fa..bbf095a8 100644 --- a/tools/protovalidate-conformance/config.go +++ b/tools/protovalidate-conformance/config.go @@ -63,7 +63,7 @@ func parseFlags() (*config, error) { flag.StringVar(&cfg.caseFilterPattern, "case", cfg.caseFilterPattern, "regex to filter cases") flag.DurationVar(&cfg.suiteTimeout, "timeout", cfg.suiteTimeout, "per-suite timeout") flag.BoolVarP(&cfg.verbose, "verbose", "v", cfg.verbose, "verbose output") - flag.BoolVar(&cfg.strict, "strict", cfg.strict, "strict mode") + flag.BoolVar(&cfg.strict, "strict", cfg.strict, "require that the violation type is an exact match") flag.BoolVar(&cfg.strictMessage, "strict_message", cfg.strictMessage, "require that violation messages match the expected message exactly") flag.BoolVar(&cfg.strictError, "strict_error", cfg.strictError, "require that compile-time errors are distinguished from runtime errors") flag.BoolVar(&cfg.json, "json", cfg.json, "return results as JSON to stdout")