From 79f6e21881ae566036a22bbf0f1879454bf5784a Mon Sep 17 00:00:00 2001 From: Calum Sieppert Date: Tue, 10 Sep 2024 12:19:55 -0600 Subject: [PATCH 1/5] Add option to skip printing skipped tests (#453) (cherry picked from commit 3fde247636cda16cb40bc5b1869b75a34b8481dd) --- Expecto/Expecto.Impl.fs | 12 ++++++++++-- Expecto/Expecto.fs | 3 +++ RELEASE_NOTES.md | 3 +++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Expecto/Expecto.Impl.fs b/Expecto/Expecto.Impl.fs index 7987a07d..a8b384df 100644 --- a/Expecto/Expecto.Impl.fs +++ b/Expecto/Expecto.Impl.fs @@ -498,6 +498,8 @@ module Impl = listStates : FocusState list /// Allows the test printer to be parametised to your liking. printer : TestPrinters + /// Whether to show skipped tests in the output. + printSkippedTests : bool /// Verbosity level (default: Info). verbosity : LogLevel /// Process name to log under (default: "Expecto") @@ -536,6 +538,7 @@ module Impl = TestPrinters.defaultPrinter else TestPrinters.teamCityPrinter TestPrinters.defaultPrinter + printSkippedTests = true verbosity = Info logName = None locate = fun _ -> SourceLocation.empty @@ -648,13 +651,18 @@ module Impl = let beforeEach (test:FlatTest) = let name = config.joinWith.format test.name - config.printer.beforeEach name + if config.printSkippedTests || test.shouldSkipEvaluation.IsNone then + config.printer.beforeEach name + else + async.Zero() async { let! beforeAsync = beforeEach test |> Async.StartChild let! result = execTestAsync ct config test do! beforeAsync - do! TestPrinters.printResult config test result + + if config.printSkippedTests || test.shouldSkipEvaluation.IsNone then + do! TestPrinters.printResult config test result if progressStarted && Option.isNone test.shouldSkipEvaluation then Fraction (Interlocked.Increment testsCompleted, testLength) diff --git a/Expecto/Expecto.fs b/Expecto/Expecto.fs index 524748f4..08013637 100644 --- a/Expecto/Expecto.fs +++ b/Expecto/Expecto.fs @@ -438,6 +438,8 @@ module Tests = | Colours of int /// Adds a test printer. | Printer of TestPrinters + /// Whether to show skipped tests in the output. + | PrintSkippedTests of bool /// Sets the verbosity level. | Verbosity of LogLevel /// Append a summary handler. @@ -529,6 +531,7 @@ module Tests = | _ -> JoinWith.Dot } | Printer p -> fun o -> { o with printer = p } + | PrintSkippedTests b -> fun o -> { o with printSkippedTests = b } | Verbosity l -> fun o -> { o with verbosity = l } | Append_Summary_Handler (SummaryHandler h) -> fun o -> o.appendSummaryHandler h diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 4497a01f..1b24e1a1 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,6 @@ +### 11.0.0-alpha3 - 2024-09-10 +* Add option to not print skipped tests + ### 11.0.0-alpha2 - 2024-08-18 * Breaking change: Update BenchmarkDotNet in Expecto.BenchmarkDotNet to 0.14.0 (#502) * 'BenchmarkConfig' has new 'eventProcessors' and 'categoryDiscoverer' properties. From f7d1690ffffe69e2f8df2a47d72415b5090e67a0 Mon Sep 17 00:00:00 2001 From: Calum Sieppert Date: Tue, 10 Sep 2024 15:35:39 -0600 Subject: [PATCH 2/5] Fix replay for fscheck3 --- Expecto.FsCheck/FsCheck.fs | 4 ++-- Expecto.FsCheck3/FsCheck3.fs | 4 ++-- Expecto.Tests.FsCheck3/FsCheck3Tests.fs | 24 ++++++++++++------------ Expecto.Tests/FsCheckTests.fs | 8 ++++---- Expecto/Model.fs | 2 +- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Expecto.FsCheck/FsCheck.fs b/Expecto.FsCheck/FsCheck.fs index 3758cd20..cc08c729 100644 --- a/Expecto.FsCheck/FsCheck.fs +++ b/Expecto.FsCheck/FsCheck.fs @@ -71,7 +71,7 @@ module ExpectoFsCheck = (String.concat " " data.Labels) let focus = - sprintf "Focus on error:\n\t%s (%A, %A) \"%s\"" methodName (uint64 std) (uint64 gen) name + sprintf "Focus on error:\n\t%s (%A, %A, 0) \"%s\"" methodName (uint64 std) (uint64 gen) name sprintf "Failed after %s. %s%s\nResult:\n\t%A\n%s%s%s" (numTests data.NumberOfTests) parameters shrunk @@ -92,7 +92,7 @@ module ExpectoFsCheck = MaxFail = 1000 // We're converting uint64s to a smaller type, but it shouldn't be an issue because users are only using the // values given in the test output, which are only ints when running FsCheck 2 - Replay = Option.map Random.StdGen (config.replay |> Option.map (fun (seed, gamma) -> int seed, int gamma)) + Replay = Option.map Random.StdGen (config.replay |> Option.map (fun (seed, gamma, _) -> int seed, int gamma)) Name = name StartSize = config.startSize EndSize = config.endSize diff --git a/Expecto.FsCheck3/FsCheck3.fs b/Expecto.FsCheck3/FsCheck3.fs index 55c8e4c4..613905ba 100644 --- a/Expecto.FsCheck3/FsCheck3.fs +++ b/Expecto.FsCheck3/FsCheck3.fs @@ -71,7 +71,7 @@ module ExpectoFsCheck = (String.concat " " data.Labels) let focus = - sprintf "Focus on error:\n\t%s (%A, %A) \"%s\"" methodName originalSeed.Seed originalSeed.Gamma name + sprintf "Focus on error:\n\t%s (%A, %A, %A) \"%s\"" methodName originalSeed.Seed originalSeed.Gamma size name sprintf "Failed after %s. %s%s\nResult:\n\t%A\n%s%s%s" (numTests data.NumberOfTests) parameters shrunk @@ -91,7 +91,7 @@ module ExpectoFsCheck = let config = Config.Default .WithMaxTest(config.maxTest) - .WithReplay(Option.map (fun (seed,gamma) -> {Rnd = Rnd(seed, gamma); Size = None}) config.replay) + .WithReplay(Option.map (fun (seed,gamma,size) -> {Rnd = Rnd(seed, gamma); Size = Some(size)}) config.replay) .WithName(name) .WithStartSize(config.startSize) .WithEndSize(config.endSize) diff --git a/Expecto.Tests.FsCheck3/FsCheck3Tests.fs b/Expecto.Tests.FsCheck3/FsCheck3Tests.fs index f1bbeb2b..5c854861 100644 --- a/Expecto.Tests.FsCheck3/FsCheck3Tests.fs +++ b/Expecto.Tests.FsCheck3/FsCheck3Tests.fs @@ -60,7 +60,7 @@ let focused = testList "FsCheck focused" [ testCase "ignore me" <| ignore - etestProperty (1UL,3UL) "Deliberately failing test" <| + etestProperty (1UL,3UL,50) "Deliberately failing test" <| fun a b c -> // wrong on purpose to test failures a * (b + c) = a * a + a * c @@ -87,14 +87,14 @@ let runFsCheckFocusedTests = match (getResult ["FsCheck focused";"Deliberately failing test"]).result with | TestResult.Failed actual -> let expected = " -Failed after 3 tests. Parameters: - -1 1 2 -Shrunk 2 times to: - -1 0 0 +Failed after 1 test. Parameters: + 47 43 -38 +Shrunk 9 times to: + 1 0 0 Result: Failed System.Exception: Expected true, got false. Focus on error: - etestProperty (1UL, 3UL) \"Deliberately failing test\"" + etestProperty (1UL, 3UL, 50) \"Deliberately failing test\"" Expect.equal actual expected "It should fail with the right message" | x -> failtestf "Expected Failed, actual was: %A" x @@ -104,7 +104,7 @@ let config = testList "FsCheck config" [ testCase "ignore me" ignore - etestPropertyWithConfig (1UL,3UL) FsCheckConfig.defaultConfig + etestPropertyWithConfig (1UL,3UL,50) FsCheckConfig.defaultConfig "Deliberately failing test" <| fun a b c -> // wrong on purpose to test failures @@ -132,14 +132,14 @@ let runFsCheckConfigTests = match (getResult ["FsCheck config";"Deliberately failing test"]).result with | TestResult.Failed actual -> let expected = " -Failed after 3 tests. Parameters: - -1 1 2 -Shrunk 2 times to: - -1 0 0 +Failed after 1 test. Parameters: + 47 43 -38 +Shrunk 9 times to: + 1 0 0 Result: Failed System.Exception: Expected true, got false. Focus on error: - etestPropertyWithConfig (1UL, 3UL) \"Deliberately failing test\"" + etestPropertyWithConfig (1UL, 3UL, 50) \"Deliberately failing test\"" Expect.equal actual expected "It should fail with the right message." | x -> diff --git a/Expecto.Tests/FsCheckTests.fs b/Expecto.Tests/FsCheckTests.fs index a6059aca..ee1fa4bd 100644 --- a/Expecto.Tests/FsCheckTests.fs +++ b/Expecto.Tests/FsCheckTests.fs @@ -60,7 +60,7 @@ let focused = testList "FsCheck focused" [ testCase "ignore me" <| ignore - etestProperty (1UL,2UL) "Deliberately failing test" <| + etestProperty (1UL,2UL,0) "Deliberately failing test" <| fun a b c -> // wrong on purpose to test failures a * (b + c) = a * a + a * c @@ -94,7 +94,7 @@ Shrunk 4 times to: Result: False Focus on error: - etestProperty (1UL, 2UL) \"Deliberately failing test\"" + etestProperty (1UL, 2UL, 0) \"Deliberately failing test\"" Expect.equal actual expected "It should fail with the right message" | x -> failtestf "Expected Failed, actual was: %A" x @@ -104,7 +104,7 @@ let config = testList "FsCheck config" [ testCase "ignore me" ignore - etestPropertyWithConfig (1UL,2UL) FsCheckConfig.defaultConfig + etestPropertyWithConfig (1UL,2UL,0) FsCheckConfig.defaultConfig "Deliberately failing test" <| fun a b c -> // wrong on purpose to test failures @@ -139,7 +139,7 @@ Shrunk 4 times to: Result: False Focus on error: - etestPropertyWithConfig (1UL, 2UL) \"Deliberately failing test\"" + etestPropertyWithConfig (1UL, 2UL, 0) \"Deliberately failing test\"" Expect.equal actual expected "It should fail with the right message." | x -> diff --git a/Expecto/Model.fs b/Expecto/Model.fs index aeb3c7dd..1c52c4ce 100644 --- a/Expecto/Model.fs +++ b/Expecto/Model.fs @@ -16,7 +16,7 @@ type FsCheckConfig = /// The size to use for the last test, when all the tests are passing. The size increases linearly between Start- and EndSize. endSize: int /// If set, the seed to use to start testing. Allows reproduction of previous runs. - replay: (uint64 * uint64) option + replay: (uint64 * uint64 * int) option /// The Arbitrary instances on this class will be merged in back to front order, i.e. instances for the same generated type at the front /// of the list will override those at the back. The instances on Arb.Default are always known, and are at the back (so they can always be /// overridden) From e5943f6e72ffb7eb3a1ed821bbda2d44d6d276c1 Mon Sep 17 00:00:00 2001 From: Calum Sieppert Date: Tue, 10 Sep 2024 15:41:07 -0600 Subject: [PATCH 3/5] Update docs --- README.md | 2 +- RELEASE_NOTES.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f1b33efd..043192b2 100644 --- a/README.md +++ b/README.md @@ -1372,5 +1372,5 @@ This might be due to how terminals/the locking thereof work: try running your te ## Migration notes ### 11.0.0 -- Any usages of the `replay` (a.k.a `stdGen` with `etestProperty*` functions) config with FsCheck tests will need to be updated to use `uint64` by appending `UL` to the literals, e.g. from `(1865288075, 296281834)` to `(1865288075UL, 296281834UL)`. +- Any usages of the `replay` (a.k.a `stdGen` with `etestProperty*` functions) config with FsCheck tests will need to be updated to use `uint64` by appending `UL` to the literals. They will also now require a third item indicating the size. E.g. from `(1865288075, 296281834)` to `(1865288075UL, 296281834UL, 3)`. - FsCheck 2 is no longer supported, so we're switching Expecto.FsCheck to use FsCheck 3 by default, even though FsCheck 3 is still in release candidate state. If you still want FsCheck2, we will continue to release FsCheck2 support for the time being using a version suffix, e.g. [11.0.0-fscheck2](https://www.nuget.org/packages/Expecto.FsCheck/11.0.0-alpha1-fscheck2) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 1b24e1a1..7da32e9d 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,9 @@ ### 11.0.0-alpha3 - 2024-09-10 * Add option to not print skipped tests +* Breaking change: Add third item to `FsCheckConfig.replay` indicating the size + * Fixes issue where the replay seed without the size was playing all tests leading up to the failure, making debugging + more difficult + * Existing FsCheck 2 users can enter any number for the size, because it is ignored. ### 11.0.0-alpha2 - 2024-08-18 * Breaking change: Update BenchmarkDotNet in Expecto.BenchmarkDotNet to 0.14.0 (#502) From e69fb64fb88f38e272bef7c8f6700f96562724e3 Mon Sep 17 00:00:00 2001 From: Calum Sieppert Date: Thu, 12 Sep 2024 12:08:35 -0600 Subject: [PATCH 4/5] Fix to show the final seed instead of the original seed for fscheck3 Using the original seed has the same problem --- Expecto.FsCheck3/FsCheck3.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Expecto.FsCheck3/FsCheck3.fs b/Expecto.FsCheck3/FsCheck3.fs index 613905ba..aa470e89 100644 --- a/Expecto.FsCheck3/FsCheck3.fs +++ b/Expecto.FsCheck3/FsCheck3.fs @@ -47,7 +47,7 @@ module ExpectoFsCheck = | TestResult.Failed (_,_,_, Outcome.Failed (:? IgnoreException as e),_,_,_) -> raise e - | TestResult.Failed (data, original, shrunk, outcome,originalSeed,_finalSeed,size) -> + | TestResult.Failed (data, original, shrunk, outcome,_originalSeed,finalSeed,size) -> let parameters = original |> List.map (sprintf "%A") @@ -71,7 +71,7 @@ module ExpectoFsCheck = (String.concat " " data.Labels) let focus = - sprintf "Focus on error:\n\t%s (%A, %A, %A) \"%s\"" methodName originalSeed.Seed originalSeed.Gamma size name + sprintf "Focus on error:\n\t%s (%A, %A, %A) \"%s\"" methodName finalSeed.Seed finalSeed.Gamma size name sprintf "Failed after %s. %s%s\nResult:\n\t%A\n%s%s%s" (numTests data.NumberOfTests) parameters shrunk From 701746a1b3a2773ab9a011b221ce15e2f9345d0e Mon Sep 17 00:00:00 2001 From: Calum Sieppert Date: Thu, 17 Oct 2024 09:04:20 -0600 Subject: [PATCH 5/5] Add back original seed to fscheck3 output --- Expecto.FsCheck3/FsCheck3.fs | 8 +++++--- Expecto.Tests.FsCheck3/FsCheck3Tests.fs | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Expecto.FsCheck3/FsCheck3.fs b/Expecto.FsCheck3/FsCheck3.fs index aa470e89..56e66d47 100644 --- a/Expecto.FsCheck3/FsCheck3.fs +++ b/Expecto.FsCheck3/FsCheck3.fs @@ -47,7 +47,7 @@ module ExpectoFsCheck = | TestResult.Failed (_,_,_, Outcome.Failed (:? IgnoreException as e),_,_,_) -> raise e - | TestResult.Failed (data, original, shrunk, outcome,_originalSeed,finalSeed,size) -> + | TestResult.Failed (data, original, shrunk, outcome,originalSeed,finalSeed,size) -> let parameters = original |> List.map (sprintf "%A") @@ -70,12 +70,14 @@ module ExpectoFsCheck = | _ -> sprintf "Labels of failing property (one or more is failing): %s\n" (String.concat " " data.Labels) + let original = + sprintf "Original seed: (%A, %A)" originalSeed.Seed originalSeed.Gamma let focus = sprintf "Focus on error:\n\t%s (%A, %A, %A) \"%s\"" methodName finalSeed.Seed finalSeed.Gamma size name - sprintf "Failed after %s. %s%s\nResult:\n\t%A\n%s%s%s" + sprintf "Failed after %s. %s%s\nResult:\n\t%A\n%s%s%s\n%s" (numTests data.NumberOfTests) parameters shrunk - outcome labels (stampsToString data.Stamps) focus + outcome labels (stampsToString data.Stamps) original focus |> FailedException |> raise diff --git a/Expecto.Tests.FsCheck3/FsCheck3Tests.fs b/Expecto.Tests.FsCheck3/FsCheck3Tests.fs index 5c854861..5cee077a 100644 --- a/Expecto.Tests.FsCheck3/FsCheck3Tests.fs +++ b/Expecto.Tests.FsCheck3/FsCheck3Tests.fs @@ -93,6 +93,7 @@ Shrunk 9 times to: 1 0 0 Result: Failed System.Exception: Expected true, got false. +Original seed: (1UL, 3UL) Focus on error: etestProperty (1UL, 3UL, 50) \"Deliberately failing test\"" Expect.equal actual expected "It should fail with the right message" @@ -138,6 +139,7 @@ Shrunk 9 times to: 1 0 0 Result: Failed System.Exception: Expected true, got false. +Original seed: (1UL, 3UL) Focus on error: etestPropertyWithConfig (1UL, 3UL, 50) \"Deliberately failing test\"" Expect.equal actual expected "It should fail with the right message."