Improve support for interactive experiences

Expecto.Tests/Bug_RepeatedColourSetting.fs

@@ -0,0 +1,14 @@
+module Bug_RepeatedColourSetting
+
+open Expecto
+open Expecto.Logging
+
+
+[<Tests>]
+let tests =
+  ftest "Colour can be set repeatedly" {
+    let colours = [|Colour0; Colour256|]
+    colours |> Array.iter ANSIOutputWriter.setColourLevel
+
+    Expect.equal (ANSIOutputWriter.getColour ()) (Array.last colours) "Colour should be the last set value"
+  }

Expecto.Tests/Expecto.Tests.fsproj

@@ -13,6 +13,7 @@
     <Compile Include="FsCheckTests.fs" />
     <Compile Include="PerformanceTests.fs" />
     <Compile Include="Bug341.fs" />
+    <Compile Include="Bug_RepeatedColourSetting.fs" />
     <Compile Include="Main.fs" />
     <None Include="paket.references" />
     <ProjectReference Include="..\Expecto.Hopac\Expecto.Hopac.fsproj" />

Expecto.Tests/Tests.fs

@@ -1821,4 +1821,71 @@ let theory =
     testTheoryTask "task odd numbers" [1; 3; 5;] <| fun x -> task {
       Expect.isTrue (x % 2 = 1) "should be odd"
     }
+  ]
+
+[<Tests>]
+let interactiveRunTests =
+  testList "running for interactive flows" [
+    testList "logging config" [
+      testCase "set loggingConfig via CLIArguments" <| fun () ->
+        let tests = testList "Test me" [
+          testCase "I am a case" (fun () -> ())
+        ]
+
+        let globalLoggerOutput = new Text.StringBuilder()
+        let globalWriter = new StringWriter(globalLoggerOutput)
+        Global.initialise {
+          Global.defaultConfig with
+            getLogger = (fun name -> 
+              TextWriterTarget(name, LogLevel.Info, globalWriter)
+            )
+        }
+
+        (runTestsWithCLIArgs [] [|"--help"|] tests) |> ignore
+
+        let cliArgLoggerOutput = new Text.StringBuilder()
+        let cliArgWriter = new StringWriter(cliArgLoggerOutput)
+        let loggingConfigFactory _ = 
+          {
+            Global.defaultConfig with
+              getLogger = (fun name -> 
+                TextWriterTarget(name, LogLevel.Info, cliArgWriter)
+              )
+          }
+
+        (runTestsWithCLIArgs [LoggingConfigFactory (FromVerbosity loggingConfigFactory)] [|"--help"|] tests) |> ignore
+
+        Expect.equal (string cliArgLoggerOutput) (string globalLoggerOutput) "Caputred output should be the same with Global.initialize or CLIArguments.LoggingConfigFactory"
+    ]
+
+    testList "runTestsReturnLogs" [
+      testCase "can print help" <| fun () -> 
+        let tests = (testList "hi" [])
+        let output = runTestsReturnLogs [] [|"--help"|] tests
+        Expect.stringContains output "Options:" ""
+        Expect.stringContains output "--help                   Show this help message." "help message should contain the --help description"
+
+
+      testCase "can list tests" <| fun () -> 
+        let tests = (testList "hi" [
+          testCase "case1" (fun () -> ())
+          testCase "case2" (fun () -> ())
+        ])
+        let expectedTestNames = ["hi.case1"; "hi.case2"]
+        let testListText = String.Join(Environment.NewLine, expectedTestNames)
+
+        let output = runTestsReturnLogs [] [|"--list-tests"|] tests
+        Expect.stringContains output testListText ""
+
+      testCase "can run tests and show basic status counts" <| fun () ->
+        let tests = testList "Interactive Tests" [
+          testCase "I pass" (fun () -> ())
+          test "I fail" {
+            Expect.equal true false "Should fail"
+          }
+        ]
+
+        let output = runTestsReturnLogs [] [|"--colours"; "0"|] tests
+        Expect.stringContains output "1 passed, 0 ignored, 1 failed, 0 errored" ""
+    ]
   ]

Expecto/Expecto.Impl.fs

@@ -520,6 +520,9 @@ module Impl =
       colour: ColourLevel
       /// Split test names by `.` or `/`
       joinWith: JoinWith
+      /// A factory method taking the configured min log level and returning a logging config.
+      /// Can be used to swap out log targets like the LiterateConsoleTarget, TextWriterTarget, and OutputWindowTarget
+      loggingConfigFactory: (LogLevel -> LoggingConfig) option
     }
     static member defaultConfig =
       { runInParallel = true
@@ -546,6 +549,7 @@ module Impl =
         noSpinner = false
         colour = Colour8
         joinWith = JoinWith.Dot
+        loggingConfigFactory = None
       }
 
     member x.appendSummaryHandler handleSummary =

Expecto/Expecto.fs

@@ -236,7 +236,7 @@
     member inline __.TryWith(p, cf) = task.TryWith(p, cf)
     member __.Run f =
       let a = task {
           do! task.Run f
       }
       match focusState with
       | Normal -> testCaseTask name a
@@ -365,6 +365,10 @@
   type SummaryHandler =
     | SummaryHandler of (TestRunSummary -> unit)
 
+  [<ReferenceEquality>]
+  type LoggingConfigFactory = 
+    | FromVerbosity of (LogLevel -> LoggingConfig)
+
   /// The CLI arguments are the parameters that are possible to send to Expecto
   /// and change the runner's behaviour.
   type CLIArguments =
@@ -426,6 +430,9 @@
     | Append_Summary_Handler of SummaryHandler
     /// Specify test names join character.
     | JoinWith of split: string
+    /// A factory method taking the configured min log level and returning a logging config.
+    /// Can be used to swap out log targets like the LiterateConsoleTarget, TextWriterTarget, and OutputWindowTarget
+    | LoggingConfigFactory of LoggingConfigFactory
 
   let options = [
       "--sequenced", "Don't run the tests in parallel.", Args.none Sequenced
@@ -513,6 +520,7 @@
     | Printer p -> fun o -> { o with printer = p }
     | Verbosity l -> fun o -> { o with verbosity = l }
     | Append_Summary_Handler (SummaryHandler h) -> fun o -> o.appendSummaryHandler h
+    | LoggingConfigFactory (FromVerbosity f) -> fun o -> { o with loggingConfigFactory = Some f}
 
   [<CompilationRepresentation (CompilationRepresentationFlags.ModuleSuffix)>]
   module ExpectoConfig =
@@ -587,13 +595,18 @@
   let runTestsWithCLIArgsAndCancel (ct:CancellationToken) cliArgs args tests =
     let runTestsWithCancel (ct:CancellationToken) config (tests:Test) =
       ANSIOutputWriter.setColourLevel config.colour
-      Global.initialiseIfDefault
-        { Global.defaultConfig with
-            getLogger = fun name ->
-              LiterateConsoleTarget(
-                name, config.verbosity,
-                consoleSemaphore = Global.semaphore()) :> Logger
-        }
+
+      let loggingConfig =
+        match config.loggingConfigFactory with
+        | Some factory -> factory config.verbosity |> Global.initialise
+        | None -> 
+          { Global.defaultConfig with
+              getLogger = fun name ->
+                LiterateConsoleTarget(
+                  name, config.verbosity,
+                  consoleSemaphore = Global.semaphore()) :> Logger
+          } |> Global.initialiseIfDefault
+
       config.logName |> Option.iter setLogName
       if config.failOnFocusedTests && passesFocusTestCheck config tests |> not then
         1
@@ -648,3 +661,31 @@
   /// Returns 0 if all tests passed, otherwise 1
   let runTestsInAssemblyWithCLIArgs cliArgs args =
     runTestsInAssemblyWithCLIArgsAndCancel CancellationToken.None cliArgs args
+
+  /// Runs all given tests with the supplied typed command-line options.
+  /// Returns the console output as a string (with ANSI coloring by default)
+  /// Useful for interactive environments like F# interactive or notebooks
+  let runTestsReturnLogs cliArgs args tests =
+
+    let literateOutputWriter (outputBuilder: Text.StringBuilder) (text: (string*ConsoleColor) list) : unit =
+      let colorizeLine (text, color) = ColourText.colouriseText color text
+      let sbAppend (builder: Text.StringBuilder) (text: string) =
+        builder.Append(text)
+
+      text
+      |> List.iter (colorizeLine >> (sbAppend outputBuilder) >> ignore)
+
+    let outputBuilder = System.Text.StringBuilder("")
+
+    let loggingConfigFactory verbosity =
+      { Global.defaultConfig with
+          getLogger = fun name ->
+            Expecto.Logging.LiterateConsoleTarget(
+              name, 
+              minLevel = verbosity, 
+              outputWriter = (literateOutputWriter outputBuilder)) :> Expecto.Logging.Logger
+      }
+
+    let cliArgs = (LoggingConfigFactory (FromVerbosity loggingConfigFactory)) :: cliArgs
+    runTestsWithCLIArgs cliArgs args tests |> ignore
+    outputBuilder.ToString()

Expecto/Logging.fs

@@ -879,7 +879,7 @@ module internal ANSIOutputWriter =
     override __.WriteLine() = write "\n"
 
   let mutable internal colours = None
-  let internal setColourLevel c = if colours.IsNone then colours <- Some c
+  let internal setColourLevel c = colours <- Some c
   let internal getColour() = Option.defaultValue Colour8 colours
 
   let colourReset = "\u001b[0m"

README.md

@@ -53,6 +53,7 @@ What follows is the Table of Contents for this README, which also serves as the
   - [`runTestsWithCLIArgsAndCancel`](#runtestswithcliargsandcancel)
   - [`runTestsInAssemblyWithCLIArgs`](#runtestsinassemblywithcliargs)
   - [`runTestsInAssemblyWithCLIArgsAndCancel`](#runtestsinassemblywithcliargsandcancel)
+  - [`runTestsReturnLogs`](#runtestsreturnlogs)
   - [Filtering with `filter`](#filtering-with-filter)
   - [Shuffling with `shuffle`](#shuffling-with-shuffle)
   - [Stress testing](#stress-testing)
@@ -220,6 +221,8 @@ runTestsWithCLIArgs [] [||] simpleTest
 which returns 1 if any tests failed, otherwise 0. Useful for returning to the
 operating system as error code.
 
+For interactive environments, you can alternatively call [`runTestsReturnLogs`](#runtestsreturnlogs), which returns the console output as a string.
+
 It's worth noting that `<|` is just a way to change the associativity of the
 language parser. In other words; it's equivalent to:
 
@@ -250,6 +253,22 @@ Signature `CancellationToken -> CLIArguments seq -> string[] -> int`. Runs the t
 assembly and also overrides the passed `CLIArguments` with the command line
 parameters. All tests need to be marked with the `[<Tests>]` attribute.
 
+### `runTestsReturnLogs`
+
+Signature `CLIArguments seq -> string[] -> Test -> string`.
+Accepts the same arguments as `runTestsWithCLIArgs`, but returns the console output as a string.
+
+This is useful for interactive environments like [F# interactive](https://learn.microsoft.com/en-us/dotnet/fsharp/tools/fsharp-interactive/) and [Polyglot Notebooks](https://marketplace.visualstudio.com/items?itemName=ms-dotnettools.dotnet-interactive-vscode), where console output is not available but the returned string can be displayed instead.
+
+Note that ANSI colors are used by default, but can be turned off using `--colours 0`.
+```fsharp
+runTestsReturnLogs [] [|"--colours";"0"|] tests
+```
+
+Any valid CLI arguments work, including `--help` and `--list-tests`.
+
+
+
 ### Filtering with `filter`
 
 You can single out tests by filtering them by name (e.g. in the