From cfbf53d892b7bcc2ea6f4527d2dd7fc942f9fcb1 Mon Sep 17 00:00:00 2001 From: Jimmy Byrd Date: Wed, 10 Apr 2024 22:18:32 -0400 Subject: [PATCH 01/11] Add ActivitySource to expecto test runs --- Expecto/Expecto.Impl.fs | 117 +++++++++++++++++++++++++++++++++++++--- Expecto/Expecto.fs | 4 ++ 2 files changed, 114 insertions(+), 7 deletions(-) diff --git a/Expecto/Expecto.Impl.fs b/Expecto/Expecto.Impl.fs index db4b0145..29f95ff9 100644 --- a/Expecto/Expecto.Impl.fs +++ b/Expecto/Expecto.Impl.fs @@ -1,6 +1,7 @@ namespace Expecto open System +open System.Collections.Generic open System.Diagnostics open System.Reflection open System.Threading @@ -9,6 +10,20 @@ open Expecto.Logging.Message open Helpers open Mono.Cecil +//! The other option is to use a dedicated activity source for Expecto instead of adding it to the config + +// module ActivitySource = + +// let [] serviceName = "Expecto" // Should be public so consumers have a strong name when adding Sources +// let private version = lazy ( +// let assembly = typeof.Assembly +// let version = assembly.GetName().Version +// version.ToString() +// ) + +// let internal activitySource = lazy new ActivitySource(serviceName, version.Value) + + // TODO: make internal? module Impl = @@ -520,6 +535,11 @@ module Impl = colour: ColourLevel /// Split test names by `.` or `/` joinWith: JoinWith + // One option is to allow the consumer to provide an activity source + // only problem is the only way to update the config is by using the CLIArguments currently + // we would have to add a new CLIArgument but that doesn't really work as it's not a reallyCLI option + // or have another way of updating the config after it's been created + activitySource : ActivitySource option } static member defaultConfig = { runInParallel = true @@ -546,6 +566,7 @@ module Impl = noSpinner = false colour = Colour8 joinWith = JoinWith.Dot + activitySource = None } member x.appendSummaryHandler handleSummary = @@ -559,8 +580,63 @@ module Impl = } } + let inline internal setStatus (status : ActivityStatusCode) (span : Activity) = + if isNull span |> not then + span.SetStatus(status) |> ignore + + let inline internal setExn (e : exn) (span : Activity) = + if isNull span |> not then + let tags = + ActivityTagsCollection( + seq { + KeyValuePair("exception.type", box (e.GetType().Name)) + KeyValuePair("exception.stacktrace", box (e.ToString())) + if not <| String.IsNullOrEmpty(e.Message) then + KeyValuePair("exception.message", box e.Message) + } + ) + + ActivityEvent("exception", tags = tags) + |> span.AddEvent + |> ignore + + let inline internal setExnMarkFailed (e : exn) (span : Activity) = + if isNull span |> not then + setExn e span + span |> setStatus ActivityStatusCode.Error + + let setSourceLocation (sourceLoc : SourceLocation) (span : Activity) = + if isNull span |> not && sourceLoc <> SourceLocation.empty then + span.SetTag("code.lineno", sourceLoc.lineNumber) |> ignore + span.SetTag("code.filepath", sourceLoc.sourcePath) |> ignore + + let inline internal addOutcome (result : TestResult) (span : Activity) = + if isNull span |> not then + span.SetTag("test.result.status", result.tag) |> ignore + span.SetTag("test.result.message", result) |> ignore + + let inline internal start (span : Activity) = + if isNull span |> not then + span.Start() |> ignore + span + + let inline internal stop (span : Activity) = + if isNull span |> not then + span.Stop() |> ignore + + let inline internal createActivity (name : string) (source : ActivitySource option) = + match source with + | Some source when not(isNull source) -> source.CreateActivity(name, ActivityKind.Internal) + | _ -> null + let execTestAsync (ct:CancellationToken) config (test:FlatTest) : Async = async { + let span = + config.activitySource + |> createActivity (config.joinWith.format test.name) + span |> setSourceLocation (config.locate test.test) + + use span = start span let w = Stopwatch.StartNew() try match test.shouldSkipEvaluation with @@ -593,32 +669,59 @@ module Impl = ) do! test fsConfig w.Stop() - return TestSummary.single Passed (float w.ElapsedMilliseconds) + stop span + let result = Passed + addOutcome result span + setStatus ActivityStatusCode.Ok span + return TestSummary.single result (float w.ElapsedMilliseconds) with | :? AssertException as e -> w.Stop() + stop span let msg = "\n" + e.Message + "\n" + (e.StackTrace.Split('\n') |> Seq.skipWhile (fun l -> l.StartsWith(" at Expecto.Expect.")) |> Seq.truncate 5 |> String.concat "\n") - return TestSummary.single (Failed msg) (float w.ElapsedMilliseconds) + let result = Failed msg + addOutcome result span + setExnMarkFailed e span + return TestSummary.single result (float w.ElapsedMilliseconds) | :? FailedException as e -> w.Stop() - return TestSummary.single (Failed ("\n"+e.Message)) (float w.ElapsedMilliseconds) + stop span + let result = Failed ("\n"+e.Message) + addOutcome result span + setExnMarkFailed e span + return TestSummary.single result (float w.ElapsedMilliseconds) | :? IgnoreException as e -> w.Stop() - return TestSummary.single (Ignored e.Message) (float w.ElapsedMilliseconds) + stop span + let result = Ignored e.Message + addOutcome result span + setExn e span + return TestSummary.single result (float w.ElapsedMilliseconds) | :? AggregateException as e when e.InnerExceptions.Count = 1 -> w.Stop() + stop span if e.InnerException :? IgnoreException then - return TestSummary.single (Ignored e.InnerException.Message) (float w.ElapsedMilliseconds) + let result = Ignored e.InnerException.Message + addOutcome result span + setExn e span + return TestSummary.single result (float w.ElapsedMilliseconds) else - return TestSummary.single (Error e.InnerException) (float w.ElapsedMilliseconds) + let result = Error e.InnerException + addOutcome result span + setExnMarkFailed e span + return TestSummary.single result (float w.ElapsedMilliseconds) | e -> w.Stop() - return TestSummary.single (Error e) (float w.ElapsedMilliseconds) + stop span + let result = Error e + addOutcome result span + setExnMarkFailed e span + return TestSummary.single result (float w.ElapsedMilliseconds) } let private numberOfWorkers limit config = diff --git a/Expecto/Expecto.fs b/Expecto/Expecto.fs index cae655c4..2d9c1b70 100644 --- a/Expecto/Expecto.fs +++ b/Expecto/Expecto.fs @@ -11,6 +11,7 @@ module Tests = open Impl open Helpers open Expecto.Logging + open System.Diagnostics let mutable private afterRunTestsList = [] let private afterRunTestsListLock = obj() @@ -444,6 +445,8 @@ module Tests = | Append_Summary_Handler of SummaryHandler /// Specify test names join character. | JoinWith of split: string + // TODO This isn't really a CLIArgument but just to show a way of updating the config + | ActivitySource of ActivitySource let options = [ "--sequenced", "Don't run the tests in parallel.", Args.none Sequenced @@ -531,6 +534,7 @@ module Tests = | Printer p -> fun o -> { o with printer = p } | Verbosity l -> fun o -> { o with verbosity = l } | Append_Summary_Handler (SummaryHandler h) -> fun o -> o.appendSummaryHandler h + | ActivitySource s -> fun o -> { o with activitySource = Option.ofObj s } [] module ExpectoConfig = From 524d15dfb473bd51f964cd188a385ad767c97862 Mon Sep 17 00:00:00 2001 From: Jimmy Byrd Date: Wed, 10 Apr 2024 22:19:12 -0400 Subject: [PATCH 02/11] Add Otel Exporter to tests This can demonstrate tests showing in OTel --- Expecto.Tests/Main.fs | 33 +++++++++++++- Expecto.Tests/paket.references | 3 +- paket.dependencies | 1 + paket.lock | 83 ++++++++++++++++++++++++++++------ 4 files changed, 102 insertions(+), 18 deletions(-) diff --git a/Expecto.Tests/Main.fs b/Expecto.Tests/Main.fs index fae29267..4b1f3589 100644 --- a/Expecto.Tests/Main.fs +++ b/Expecto.Tests/Main.fs @@ -2,13 +2,42 @@ module Main open Expecto open Expecto.Logging +open OpenTelemetry.Resources +open OpenTelemetry +open OpenTelemetry.Trace +open System.Threading +open System.Diagnostics + +let serviceName = "Expecto.Tests" + +let logger = Log.create serviceName + + +let resourceBuilder = + ResourceBuilder + .CreateDefault() + .AddService(serviceName = serviceName) + -let logger = Log.create "Expecto.Tests" [] let main args = + let activitySource = new ActivitySource(serviceName) + use traceProvider = + Sdk + .CreateTracerProviderBuilder() + .AddSource(serviceName) + .SetResourceBuilder(resourceBuilder ) + .AddOtlpExporter() + .Build() + let tracer = traceProvider.GetTracer(serviceName) + // use span = tracer.StartActiveSpan("Expecto.main") + use span = tracer.StartRootSpan("Expecto.main") let test = Impl.testFromThisAssembly() |> Option.orDefault (TestList ([], Normal)) |> Test.shuffle "." - runTestsWithCLIArgs [NUnit_Summary "bin/Expecto.Tests.TestResults.xml"] args test + runTestsWithCLIArgs [NUnit_Summary "bin/Expecto.Tests.TestResults.xml"; CLIArguments.ActivitySource activitySource] args test + + + diff --git a/Expecto.Tests/paket.references b/Expecto.Tests/paket.references index 36cbbdb4..1d6bbdfc 100644 --- a/Expecto.Tests/paket.references +++ b/Expecto.Tests/paket.references @@ -1 +1,2 @@ -FsCheck \ No newline at end of file +FsCheck +OpenTelemetry.Exporter.OpenTelemetryProtocol \ No newline at end of file diff --git a/paket.dependencies b/paket.dependencies index 4b937e21..ba9d5aa4 100644 --- a/paket.dependencies +++ b/paket.dependencies @@ -9,6 +9,7 @@ nuget Hopac ~> 0.4 nuget DiffPlex ~> 1.5 nuget Mono.Cecil ~> 0.11 nuget BenchmarkDotNet ~> 0.13.5 +nuget OpenTelemetry.Exporter.OpenTelemetryProtocol group FsCheck3 source https://api.nuget.org/v3/index.json diff --git a/paket.lock b/paket.lock index 914c436f..7870e1bc 100644 --- a/paket.lock +++ b/paket.lock @@ -25,6 +25,16 @@ NUGET FSharp.Core (>= 4.2.3) FSharp.Core (7.0.200) Gee.External.Capstone (2.3) + Google.Protobuf (3.26.1) + System.Memory (>= 4.5.3) - restriction: || (&& (== net6.0) (>= net45)) (&& (== net6.0) (< net5.0)) (&& (== net6.0) (< netstandard2.0)) (== netstandard2.1) + System.Runtime.CompilerServices.Unsafe (>= 4.5.2) - restriction: || (&& (== net6.0) (< net5.0)) (== netstandard2.1) + Grpc.Core.Api (2.62) + Grpc.Net.Client (2.62) + Grpc.Net.Common (>= 2.62) + Microsoft.Extensions.Logging.Abstractions (>= 6.0) + System.Diagnostics.DiagnosticSource (>= 6.0.1) - restriction: || (&& (== net6.0) (>= net462)) (&& (== net6.0) (< netstandard2.1)) (== netstandard2.1) + Grpc.Net.Common (2.62) + Grpc.Core.Api (>= 2.62) Hopac (0.5.1) FSharp.Core (>= 4.5.2) Iced (1.18) @@ -50,22 +60,51 @@ NUGET Microsoft.Diagnostics.Tracing.TraceEvent (3.0.8) System.Runtime.CompilerServices.Unsafe (>= 5.0) Microsoft.DotNet.PlatformAbstractions (3.1.6) - Microsoft.Extensions.DependencyInjection (7.0) - Microsoft.Extensions.DependencyInjection.Abstractions (>= 7.0) - Microsoft.Extensions.DependencyInjection.Abstractions (7.0) - Microsoft.Extensions.Logging (7.0) - Microsoft.Extensions.DependencyInjection (>= 7.0) - Microsoft.Extensions.DependencyInjection.Abstractions (>= 7.0) - Microsoft.Extensions.Logging.Abstractions (>= 7.0) - Microsoft.Extensions.Options (>= 7.0) - System.Diagnostics.DiagnosticSource (>= 7.0) - restriction: || (&& (== net6.0) (>= net462)) (&& (== net6.0) (< netstandard2.1)) (== netstandard2.1) - Microsoft.Extensions.Logging.Abstractions (7.0) + Microsoft.Extensions.Configuration (8.0) + Microsoft.Extensions.Configuration.Abstractions (>= 8.0) + Microsoft.Extensions.Primitives (>= 8.0) + Microsoft.Extensions.Configuration.Abstractions (8.0) + Microsoft.Extensions.Primitives (>= 8.0) + Microsoft.Extensions.Configuration.Binder (8.0.1) + Microsoft.Extensions.Configuration.Abstractions (>= 8.0) + Microsoft.Extensions.DependencyInjection (8.0) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 8.0) + Microsoft.Extensions.DependencyInjection.Abstractions (8.0.1) + Microsoft.Extensions.Diagnostics.Abstractions (8.0) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 8.0) + Microsoft.Extensions.Options (>= 8.0) System.Buffers (>= 4.5.1) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) + System.Diagnostics.DiagnosticSource (>= 8.0) System.Memory (>= 4.5.5) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) - Microsoft.Extensions.Options (7.0.1) - Microsoft.Extensions.DependencyInjection.Abstractions (>= 7.0) - Microsoft.Extensions.Primitives (>= 7.0) - Microsoft.Extensions.Primitives (7.0) + Microsoft.Extensions.Logging (8.0) + Microsoft.Extensions.DependencyInjection (>= 8.0) + Microsoft.Extensions.Logging.Abstractions (>= 8.0) + Microsoft.Extensions.Options (>= 8.0) + System.Diagnostics.DiagnosticSource (>= 8.0) - restriction: || (&& (== net6.0) (>= net462)) (&& (== net6.0) (< netstandard2.1)) (== netstandard2.1) + Microsoft.Extensions.Logging.Abstractions (8.0.1) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 8.0.1) + System.Buffers (>= 4.5.1) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) + System.Memory (>= 4.5.5) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) + Microsoft.Extensions.Logging.Configuration (8.0) + Microsoft.Extensions.Configuration (>= 8.0) + Microsoft.Extensions.Configuration.Abstractions (>= 8.0) + Microsoft.Extensions.Configuration.Binder (>= 8.0) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 8.0) + Microsoft.Extensions.Logging (>= 8.0) + Microsoft.Extensions.Logging.Abstractions (>= 8.0) + Microsoft.Extensions.Options (>= 8.0) + Microsoft.Extensions.Options.ConfigurationExtensions (>= 8.0) + Microsoft.Extensions.Options (8.0.2) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 8.0) + Microsoft.Extensions.Primitives (>= 8.0) + System.ComponentModel.Annotations (>= 5.0) - restriction: || (&& (== net6.0) (< netstandard2.1)) (== netstandard2.1) + Microsoft.Extensions.Options.ConfigurationExtensions (8.0) + Microsoft.Extensions.Configuration.Abstractions (>= 8.0) + Microsoft.Extensions.Configuration.Binder (>= 8.0) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 8.0) + Microsoft.Extensions.Options (>= 8.0) + Microsoft.Extensions.Primitives (>= 8.0) + Microsoft.Extensions.Primitives (8.0) System.Memory (>= 4.5.5) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) System.Runtime.CompilerServices.Unsafe (>= 6.0) Microsoft.NETCore.Platforms (3.1) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp2.0)) (&& (== netstandard2.1) (>= netcoreapp3.1)) @@ -75,6 +114,19 @@ NUGET System.Security.AccessControl (>= 5.0) System.Security.Principal.Windows (>= 5.0) Mono.Cecil (0.11.4) + OpenTelemetry (1.8) + Microsoft.Extensions.Diagnostics.Abstractions (>= 8.0) + Microsoft.Extensions.Logging.Configuration (>= 8.0) + OpenTelemetry.Api.ProviderBuilderExtensions (>= 1.8) + OpenTelemetry.Api (1.8) + System.Diagnostics.DiagnosticSource (>= 8.0) + OpenTelemetry.Api.ProviderBuilderExtensions (1.8) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 8.0) + OpenTelemetry.Api (>= 1.8) + OpenTelemetry.Exporter.OpenTelemetryProtocol (1.8) + Google.Protobuf (>= 3.22.5 < 4.0) + Grpc.Net.Client (>= 2.52 < 3.0) + OpenTelemetry (>= 1.8) Perfolizer (0.2.1) System.Memory (>= 4.5.3) System.Buffers (4.5.1) - restriction: == netstandard2.1 @@ -82,7 +134,8 @@ NUGET System.Collections.Immutable (7.0) System.Memory (>= 4.5.5) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) System.Runtime.CompilerServices.Unsafe (>= 6.0) - System.Diagnostics.DiagnosticSource (7.0.2) - restriction: || (&& (== net6.0) (>= net462)) (&& (== net6.0) (< netstandard2.1)) (== netstandard2.1) + System.ComponentModel.Annotations (5.0) - restriction: || (&& (== net6.0) (< netstandard2.1)) (== netstandard2.1) + System.Diagnostics.DiagnosticSource (8.0) System.Memory (>= 4.5.5) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) System.Runtime.CompilerServices.Unsafe (>= 6.0) System.Management (7.0) From ec81a94d7e98ef6b4fc3ec370960dc05b27b0c64 Mon Sep 17 00:00:00 2001 From: Spencer Farley <2847259+farlee2121@users.noreply.github.com> Date: Sun, 14 Apr 2024 17:59:49 -0500 Subject: [PATCH 03/11] Demonstrate OpenTelemetry extension as a transform --- Expecto.Tests/Expecto.Tests.fsproj | 5 +- Expecto.Tests/Main.fs | 3 +- Expecto.Tests/OpenTelemetry.fs | 290 +++++++++++++++++++++++++++++ Expecto/Expecto.Impl.fs | 96 +--------- 4 files changed, 302 insertions(+), 92 deletions(-) create mode 100644 Expecto.Tests/OpenTelemetry.fs diff --git a/Expecto.Tests/Expecto.Tests.fsproj b/Expecto.Tests/Expecto.Tests.fsproj index f9595e24..84b1b122 100644 --- a/Expecto.Tests/Expecto.Tests.fsproj +++ b/Expecto.Tests/Expecto.Tests.fsproj @@ -1,4 +1,4 @@ - + Expecto.Tests @@ -13,6 +13,7 @@ + @@ -20,4 +21,4 @@ - + \ No newline at end of file diff --git a/Expecto.Tests/Main.fs b/Expecto.Tests/Main.fs index 4b1f3589..8e10c150 100644 --- a/Expecto.Tests/Main.fs +++ b/Expecto.Tests/Main.fs @@ -36,8 +36,9 @@ let main args = let test = Impl.testFromThisAssembly() |> Option.orDefault (TestList ([], Normal)) + |> OpenTelemetry.addOpenTelemetry_SpanPerTest Impl.ExpectoConfig.defaultConfig activitySource |> Test.shuffle "." - runTestsWithCLIArgs [NUnit_Summary "bin/Expecto.Tests.TestResults.xml"; CLIArguments.ActivitySource activitySource] args test + runTestsWithCLIArgs [NUnit_Summary "bin/Expecto.Tests.TestResults.xml"] args test diff --git a/Expecto.Tests/OpenTelemetry.fs b/Expecto.Tests/OpenTelemetry.fs new file mode 100644 index 00000000..5a959c37 --- /dev/null +++ b/Expecto.Tests/OpenTelemetry.fs @@ -0,0 +1,290 @@ +namespace Expecto + +module OpenTelemetry = + open System + open System.Diagnostics + open System.Collections.Generic + open System.Threading + open Impl + + module internal Activity = + let inline setStatus (status : ActivityStatusCode) (span : Activity) = + if isNull span |> not then + span.SetStatus(status) |> ignore + + let inline setExn (e : exn) (span : Activity) = + if isNull span |> not then + let tags = + ActivityTagsCollection( + seq { + KeyValuePair("exception.type", box (e.GetType().Name)) + KeyValuePair("exception.stacktrace", box (e.ToString())) + if not <| String.IsNullOrEmpty(e.Message) then + KeyValuePair("exception.message", box e.Message) + } + ) + + ActivityEvent("exception", tags = tags) + |> span.AddEvent + |> ignore + + let inline setExnMarkFailed (e : exn) (span : Activity) = + if isNull span |> not then + setExn e span + span |> setStatus ActivityStatusCode.Error + + let setSourceLocation (sourceLoc : SourceLocation) (span : Activity) = + if isNull span |> not && sourceLoc <> SourceLocation.empty then + span.SetTag("code.lineno", sourceLoc.lineNumber) |> ignore + span.SetTag("code.filepath", sourceLoc.sourcePath) |> ignore + + let inline addOutcome (result : TestResult) (span : Activity) = + if isNull span |> not then + span.SetTag("test.result.status", result.tag) |> ignore + span.SetTag("test.result.message", result) |> ignore + + let inline start (span : Activity) = + if isNull span |> not then + span.Start() |> ignore + span + + let inline stop (span : Activity) = + if isNull span |> not then + span.Stop() |> ignore + + let inline createActivity (name : string) (source : ActivitySource option) = + match source with + | Some source when not(isNull source) -> source.CreateActivity(name, ActivityKind.Internal) + | _ -> null + + open Activity + + open System.Runtime.ExceptionServices + + let inline internal reraiseAnywhere<'a> (e: exn) : 'a = + ExceptionDispatchInfo.Capture(e).Throw() + Unchecked.defaultof<'a> + + let wrapCodeWithSpan (span: Activity) (test: TestCode) = + match test with + | Sync test -> + TestCode.Sync (fun () -> + try + start span + test () + stop span + addOutcome Passed span + setStatus ActivityStatusCode.Ok span + with + | :? AssertException as e -> + stop span + // TODO: this message construction is fragile duplication + let msg = + "\n" + e.Message + "\n" + + (e.StackTrace.Split('\n') + |> Seq.skipWhile (fun l -> l.StartsWith(" at Expecto.Expect.")) + |> Seq.truncate 5 + |> String.concat "\n") + let result = Failed msg + addOutcome result span + setExnMarkFailed e span + reraiseAnywhere e + | :? FailedException as e -> + stop span + let result = Failed ("\n"+e.Message) + addOutcome result span + setExnMarkFailed e span + reraiseAnywhere e + | :? IgnoreException as e -> + stop span + let result = Ignored e.Message + addOutcome result span + setExn e span + reraiseAnywhere e + | :? AggregateException as e when e.InnerExceptions.Count = 1 -> + stop span + if e.InnerException :? IgnoreException then + let result = Ignored e.InnerException.Message + addOutcome result span + setExn e span + else + let result = Error e.InnerException + addOutcome result span + setExnMarkFailed e span + reraiseAnywhere e + | e -> + stop span + let result = Error e + addOutcome result span + setExnMarkFailed e span + reraiseAnywhere e + ) + + | Async test -> + TestCode.Async (async { + try + start span + do! test + stop span + addOutcome Passed span + setStatus ActivityStatusCode.Ok span + with + | :? AssertException as e -> + stop span + // TODO: this message construction is fragile duplication + let msg = + "\n" + e.Message + "\n" + + (e.StackTrace.Split('\n') + |> Seq.skipWhile (fun l -> l.StartsWith(" at Expecto.Expect.")) + |> Seq.truncate 5 + |> String.concat "\n") + let result = Failed msg + addOutcome result span + setExnMarkFailed e span + reraiseAnywhere e + | :? FailedException as e -> + stop span + let result = Failed ("\n"+e.Message) + addOutcome result span + setExnMarkFailed e span + reraiseAnywhere e + | :? IgnoreException as e -> + stop span + let result = Ignored e.Message + addOutcome result span + setExn e span + reraiseAnywhere e + | :? AggregateException as e when e.InnerExceptions.Count = 1 -> + stop span + if e.InnerException :? IgnoreException then + let result = Ignored e.InnerException.Message + addOutcome result span + setExn e span + else + let result = Error e.InnerException + addOutcome result span + setExnMarkFailed e span + reraiseAnywhere e + | e -> + stop span + let result = Error e + addOutcome result span + setExnMarkFailed e span + reraiseAnywhere e + }) + | AsyncFsCheck (testConfig, stressConfig, test) -> + TestCode.AsyncFsCheck (testConfig, stressConfig, fun fsCheckConfig -> async { + try + start span + do! test fsCheckConfig + stop span + addOutcome Passed span + setStatus ActivityStatusCode.Ok span + with + | :? AssertException as e -> + stop span + // TODO: this message construction is fragile duplication + let msg = + "\n" + e.Message + "\n" + + (e.StackTrace.Split('\n') + |> Seq.skipWhile (fun l -> l.StartsWith(" at Expecto.Expect.")) + |> Seq.truncate 5 + |> String.concat "\n") + let result = Failed msg + addOutcome result span + setExnMarkFailed e span + reraiseAnywhere e + | :? FailedException as e -> + stop span + let result = Failed ("\n"+e.Message) + addOutcome result span + setExnMarkFailed e span + reraiseAnywhere e + | :? IgnoreException as e -> + stop span + let result = Ignored e.Message + addOutcome result span + setExn e span + reraiseAnywhere e + | :? AggregateException as e when e.InnerExceptions.Count = 1 -> + stop span + if e.InnerException :? IgnoreException then + let result = Ignored e.InnerException.Message + addOutcome result span + setExn e span + else + let result = Error e.InnerException + addOutcome result span + setExnMarkFailed e span + reraiseAnywhere e + | e -> + stop span + let result = Error e + addOutcome result span + setExnMarkFailed e span + reraiseAnywhere e + }) + | SyncWithCancel test-> + TestCode.SyncWithCancel (fun ct -> + try + start span + test ct + stop span + addOutcome Passed span + setStatus ActivityStatusCode.Ok span + with + | :? AssertException as e -> + stop span + // TODO: this message construction is fragile duplication + let msg = + "\n" + e.Message + "\n" + + (e.StackTrace.Split('\n') + |> Seq.skipWhile (fun l -> l.StartsWith(" at Expecto.Expect.")) + |> Seq.truncate 5 + |> String.concat "\n") + let result = Failed msg + addOutcome result span + setExnMarkFailed e span + reraiseAnywhere e + | :? FailedException as e -> + stop span + let result = Failed ("\n"+e.Message) + addOutcome result span + setExnMarkFailed e span + reraiseAnywhere e + | :? IgnoreException as e -> + stop span + let result = Ignored e.Message + addOutcome result span + setExn e span + reraiseAnywhere e + | :? AggregateException as e when e.InnerExceptions.Count = 1 -> + stop span + if e.InnerException :? IgnoreException then + let result = Ignored e.InnerException.Message + addOutcome result span + setExn e span + else + let result = Error e.InnerException + addOutcome result span + setExnMarkFailed e span + reraiseAnywhere e + | e -> + stop span + let result = Error e + addOutcome result span + setExnMarkFailed e span + reraiseAnywhere e + ) + + + let addOpenTelemetry_SpanPerTest (config: ExpectoConfig) (activitySource: ActivitySource) (rootTest: Test) : Test = + + rootTest + |> Test.toTestCodeList + |> List.map (fun test -> + let span = createActivity (config.joinWith.format test.name) (Some activitySource) + span |> setSourceLocation (config.locate test.test) + {test with test = wrapCodeWithSpan span test.test} + ) + |> Test.fromFlatTests config.joinWith.asString diff --git a/Expecto/Expecto.Impl.fs b/Expecto/Expecto.Impl.fs index 29f95ff9..ebcb8d3e 100644 --- a/Expecto/Expecto.Impl.fs +++ b/Expecto/Expecto.Impl.fs @@ -580,63 +580,8 @@ module Impl = } } - let inline internal setStatus (status : ActivityStatusCode) (span : Activity) = - if isNull span |> not then - span.SetStatus(status) |> ignore - - let inline internal setExn (e : exn) (span : Activity) = - if isNull span |> not then - let tags = - ActivityTagsCollection( - seq { - KeyValuePair("exception.type", box (e.GetType().Name)) - KeyValuePair("exception.stacktrace", box (e.ToString())) - if not <| String.IsNullOrEmpty(e.Message) then - KeyValuePair("exception.message", box e.Message) - } - ) - - ActivityEvent("exception", tags = tags) - |> span.AddEvent - |> ignore - - let inline internal setExnMarkFailed (e : exn) (span : Activity) = - if isNull span |> not then - setExn e span - span |> setStatus ActivityStatusCode.Error - - let setSourceLocation (sourceLoc : SourceLocation) (span : Activity) = - if isNull span |> not && sourceLoc <> SourceLocation.empty then - span.SetTag("code.lineno", sourceLoc.lineNumber) |> ignore - span.SetTag("code.filepath", sourceLoc.sourcePath) |> ignore - - let inline internal addOutcome (result : TestResult) (span : Activity) = - if isNull span |> not then - span.SetTag("test.result.status", result.tag) |> ignore - span.SetTag("test.result.message", result) |> ignore - - let inline internal start (span : Activity) = - if isNull span |> not then - span.Start() |> ignore - span - - let inline internal stop (span : Activity) = - if isNull span |> not then - span.Stop() |> ignore - - let inline internal createActivity (name : string) (source : ActivitySource option) = - match source with - | Some source when not(isNull source) -> source.CreateActivity(name, ActivityKind.Internal) - | _ -> null - let execTestAsync (ct:CancellationToken) config (test:FlatTest) : Async = async { - let span = - config.activitySource - |> createActivity (config.joinWith.format test.name) - span |> setSourceLocation (config.locate test.test) - - use span = start span let w = Stopwatch.StartNew() try match test.shouldSkipEvaluation with @@ -669,59 +614,32 @@ module Impl = ) do! test fsConfig w.Stop() - stop span - let result = Passed - addOutcome result span - setStatus ActivityStatusCode.Ok span - return TestSummary.single result (float w.ElapsedMilliseconds) + return TestSummary.single Passed (float w.ElapsedMilliseconds) with | :? AssertException as e -> w.Stop() - stop span let msg = "\n" + e.Message + "\n" + (e.StackTrace.Split('\n') |> Seq.skipWhile (fun l -> l.StartsWith(" at Expecto.Expect.")) |> Seq.truncate 5 |> String.concat "\n") - let result = Failed msg - addOutcome result span - setExnMarkFailed e span - return TestSummary.single result (float w.ElapsedMilliseconds) + return TestSummary.single (Failed msg) (float w.ElapsedMilliseconds) | :? FailedException as e -> w.Stop() - stop span - let result = Failed ("\n"+e.Message) - addOutcome result span - setExnMarkFailed e span - return TestSummary.single result (float w.ElapsedMilliseconds) + return TestSummary.single (Failed ("\n"+e.Message)) (float w.ElapsedMilliseconds) | :? IgnoreException as e -> w.Stop() - stop span - let result = Ignored e.Message - addOutcome result span - setExn e span - return TestSummary.single result (float w.ElapsedMilliseconds) + return TestSummary.single (Ignored e.Message) (float w.ElapsedMilliseconds) | :? AggregateException as e when e.InnerExceptions.Count = 1 -> w.Stop() - stop span if e.InnerException :? IgnoreException then - let result = Ignored e.InnerException.Message - addOutcome result span - setExn e span - return TestSummary.single result (float w.ElapsedMilliseconds) + return TestSummary.single (Ignored e.InnerException.Message) (float w.ElapsedMilliseconds) else - let result = Error e.InnerException - addOutcome result span - setExnMarkFailed e span - return TestSummary.single result (float w.ElapsedMilliseconds) + return TestSummary.single (Error e.InnerException) (float w.ElapsedMilliseconds) | e -> w.Stop() - stop span - let result = Error e - addOutcome result span - setExnMarkFailed e span - return TestSummary.single result (float w.ElapsedMilliseconds) + return TestSummary.single (Error e) (float w.ElapsedMilliseconds) } let private numberOfWorkers limit config = From d0874318c13e89cf24002cabb2503956389844d9 Mon Sep 17 00:00:00 2001 From: Spencer Farley <2847259+farlee2121@users.noreply.github.com> Date: Mon, 15 Apr 2024 16:46:16 -0500 Subject: [PATCH 04/11] Consolidate redundant exception mapping in Open Telemetry wrapper --- Expecto.Tests/OpenTelemetry.fs | 198 +++++++-------------------------- 1 file changed, 38 insertions(+), 160 deletions(-) diff --git a/Expecto.Tests/OpenTelemetry.fs b/Expecto.Tests/OpenTelemetry.fs index 5a959c37..99468df8 100644 --- a/Expecto.Tests/OpenTelemetry.fs +++ b/Expecto.Tests/OpenTelemetry.fs @@ -65,6 +65,40 @@ module OpenTelemetry = ExceptionDispatchInfo.Capture(e).Throw() Unchecked.defaultof<'a> + module TestResult = + let ofException (e:Exception) : TestResult = + match e with + | :? AssertException as e -> + let msg = + "\n" + e.Message + "\n" + + (e.StackTrace.Split('\n') + |> Seq.skipWhile (fun l -> l.StartsWith(" at Expecto.Expect.")) + |> Seq.truncate 5 + |> String.concat "\n") + Failed msg + + | :? FailedException as e -> + Failed ("\n"+e.Message) + | :? IgnoreException as e -> + Ignored e.Message + | :? AggregateException as e when e.InnerExceptions.Count = 1 -> + if e.InnerException :? IgnoreException then + Ignored e.InnerException.Message + else + Error e.InnerException + | e -> + Error e + + + let addExceptionOutcomeToSpan (span: Activity) (e: Exception) = + let testResult = TestResult.ofException e + + addOutcome testResult span + match testResult with + | Ignored _ -> + setExn e span + | _ -> + setExnMarkFailed e span let wrapCodeWithSpan (span: Activity) (test: TestCode) = match test with | Sync test -> @@ -76,47 +110,8 @@ module OpenTelemetry = addOutcome Passed span setStatus ActivityStatusCode.Ok span with - | :? AssertException as e -> - stop span - // TODO: this message construction is fragile duplication - let msg = - "\n" + e.Message + "\n" + - (e.StackTrace.Split('\n') - |> Seq.skipWhile (fun l -> l.StartsWith(" at Expecto.Expect.")) - |> Seq.truncate 5 - |> String.concat "\n") - let result = Failed msg - addOutcome result span - setExnMarkFailed e span - reraiseAnywhere e - | :? FailedException as e -> - stop span - let result = Failed ("\n"+e.Message) - addOutcome result span - setExnMarkFailed e span - reraiseAnywhere e - | :? IgnoreException as e -> - stop span - let result = Ignored e.Message - addOutcome result span - setExn e span - reraiseAnywhere e - | :? AggregateException as e when e.InnerExceptions.Count = 1 -> - stop span - if e.InnerException :? IgnoreException then - let result = Ignored e.InnerException.Message - addOutcome result span - setExn e span - else - let result = Error e.InnerException - addOutcome result span - setExnMarkFailed e span - reraiseAnywhere e | e -> - stop span - let result = Error e - addOutcome result span - setExnMarkFailed e span + addExceptionOutcomeToSpan span e reraiseAnywhere e ) @@ -129,47 +124,8 @@ module OpenTelemetry = addOutcome Passed span setStatus ActivityStatusCode.Ok span with - | :? AssertException as e -> - stop span - // TODO: this message construction is fragile duplication - let msg = - "\n" + e.Message + "\n" + - (e.StackTrace.Split('\n') - |> Seq.skipWhile (fun l -> l.StartsWith(" at Expecto.Expect.")) - |> Seq.truncate 5 - |> String.concat "\n") - let result = Failed msg - addOutcome result span - setExnMarkFailed e span - reraiseAnywhere e - | :? FailedException as e -> - stop span - let result = Failed ("\n"+e.Message) - addOutcome result span - setExnMarkFailed e span - reraiseAnywhere e - | :? IgnoreException as e -> - stop span - let result = Ignored e.Message - addOutcome result span - setExn e span - reraiseAnywhere e - | :? AggregateException as e when e.InnerExceptions.Count = 1 -> - stop span - if e.InnerException :? IgnoreException then - let result = Ignored e.InnerException.Message - addOutcome result span - setExn e span - else - let result = Error e.InnerException - addOutcome result span - setExnMarkFailed e span - reraiseAnywhere e | e -> - stop span - let result = Error e - addOutcome result span - setExnMarkFailed e span + addExceptionOutcomeToSpan span e reraiseAnywhere e }) | AsyncFsCheck (testConfig, stressConfig, test) -> @@ -181,47 +137,8 @@ module OpenTelemetry = addOutcome Passed span setStatus ActivityStatusCode.Ok span with - | :? AssertException as e -> - stop span - // TODO: this message construction is fragile duplication - let msg = - "\n" + e.Message + "\n" + - (e.StackTrace.Split('\n') - |> Seq.skipWhile (fun l -> l.StartsWith(" at Expecto.Expect.")) - |> Seq.truncate 5 - |> String.concat "\n") - let result = Failed msg - addOutcome result span - setExnMarkFailed e span - reraiseAnywhere e - | :? FailedException as e -> - stop span - let result = Failed ("\n"+e.Message) - addOutcome result span - setExnMarkFailed e span - reraiseAnywhere e - | :? IgnoreException as e -> - stop span - let result = Ignored e.Message - addOutcome result span - setExn e span - reraiseAnywhere e - | :? AggregateException as e when e.InnerExceptions.Count = 1 -> - stop span - if e.InnerException :? IgnoreException then - let result = Ignored e.InnerException.Message - addOutcome result span - setExn e span - else - let result = Error e.InnerException - addOutcome result span - setExnMarkFailed e span - reraiseAnywhere e | e -> - stop span - let result = Error e - addOutcome result span - setExnMarkFailed e span + addExceptionOutcomeToSpan span e reraiseAnywhere e }) | SyncWithCancel test-> @@ -233,47 +150,8 @@ module OpenTelemetry = addOutcome Passed span setStatus ActivityStatusCode.Ok span with - | :? AssertException as e -> - stop span - // TODO: this message construction is fragile duplication - let msg = - "\n" + e.Message + "\n" + - (e.StackTrace.Split('\n') - |> Seq.skipWhile (fun l -> l.StartsWith(" at Expecto.Expect.")) - |> Seq.truncate 5 - |> String.concat "\n") - let result = Failed msg - addOutcome result span - setExnMarkFailed e span - reraiseAnywhere e - | :? FailedException as e -> - stop span - let result = Failed ("\n"+e.Message) - addOutcome result span - setExnMarkFailed e span - reraiseAnywhere e - | :? IgnoreException as e -> - stop span - let result = Ignored e.Message - addOutcome result span - setExn e span - reraiseAnywhere e - | :? AggregateException as e when e.InnerExceptions.Count = 1 -> - stop span - if e.InnerException :? IgnoreException then - let result = Ignored e.InnerException.Message - addOutcome result span - setExn e span - else - let result = Error e.InnerException - addOutcome result span - setExnMarkFailed e span - reraiseAnywhere e | e -> - stop span - let result = Error e - addOutcome result span - setExnMarkFailed e span + addExceptionOutcomeToSpan span e reraiseAnywhere e ) From b7bde26147a53838696e28489a154e8a47c30e04 Mon Sep 17 00:00:00 2001 From: Jimmy Byrd Date: Thu, 18 Apr 2024 22:51:31 -0400 Subject: [PATCH 05/11] Refactor wrapCodeWithSpan --- Expecto.Tests/OpenTelemetry.fs | 78 +++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/Expecto.Tests/OpenTelemetry.fs b/Expecto.Tests/OpenTelemetry.fs index 99468df8..86ed10ef 100644 --- a/Expecto.Tests/OpenTelemetry.fs +++ b/Expecto.Tests/OpenTelemetry.fs @@ -7,13 +7,16 @@ module OpenTelemetry = open System.Threading open Impl + module internal Activity = + let inline isNotNull x = isNull x |> not + let inline setStatus (status : ActivityStatusCode) (span : Activity) = - if isNull span |> not then + if isNotNull span then span.SetStatus(status) |> ignore let inline setExn (e : exn) (span : Activity) = - if isNull span |> not then + if isNotNull span|> not then let tags = ActivityTagsCollection( seq { @@ -29,37 +32,41 @@ module OpenTelemetry = |> ignore let inline setExnMarkFailed (e : exn) (span : Activity) = - if isNull span |> not then + if isNotNull span then setExn e span span |> setStatus ActivityStatusCode.Error let setSourceLocation (sourceLoc : SourceLocation) (span : Activity) = - if isNull span |> not && sourceLoc <> SourceLocation.empty then + if isNotNull span && sourceLoc <> SourceLocation.empty then span.SetTag("code.lineno", sourceLoc.lineNumber) |> ignore span.SetTag("code.filepath", sourceLoc.sourcePath) |> ignore let inline addOutcome (result : TestResult) (span : Activity) = - if isNull span |> not then + if isNotNull span then span.SetTag("test.result.status", result.tag) |> ignore span.SetTag("test.result.message", result) |> ignore let inline start (span : Activity) = - if isNull span |> not then + if isNotNull span then span.Start() |> ignore span let inline stop (span : Activity) = - if isNull span |> not then + if isNotNull span then span.Stop() |> ignore - let inline createActivity (name : string) (source : ActivitySource option) = + let inline setEndTimeNow (span : Activity) = + if isNotNull span then + span.SetEndTime(DateTime.UtcNow) |> ignore + + let inline createActivity (name : string) (source : ActivitySource) = match source with - | Some source when not(isNull source) -> source.CreateActivity(name, ActivityKind.Internal) + | source when not(isNull source) -> source.CreateActivity(name, ActivityKind.Internal) | _ -> null open Activity - open System.Runtime.ExceptionServices + open System.IO let inline internal reraiseAnywhere<'a> (e: exn) : 'a = ExceptionDispatchInfo.Capture(e).Throw() @@ -99,70 +106,71 @@ module OpenTelemetry = setExn e span | _ -> setExnMarkFailed e span + let wrapCodeWithSpan (span: Activity) (test: TestCode) = + let inline handleSuccess span = + setEndTimeNow span + addOutcome Passed span + setStatus ActivityStatusCode.Ok span + let inline handleFailure span e = + setEndTimeNow span + addExceptionOutcomeToSpan span e + reraiseAnywhere e + match test with | Sync test -> TestCode.Sync (fun () -> + use span = start span + File.AppendAllText(Path.Combine(__SOURCE_DIRECTORY__, "wrapCodeWithSpan.log"), $"{span.DisplayName}\n") try - start span test () - stop span - addOutcome Passed span - setStatus ActivityStatusCode.Ok span + handleSuccess span with | e -> - addExceptionOutcomeToSpan span e - reraiseAnywhere e + handleFailure span e ) | Async test -> TestCode.Async (async { + use span = start span try - start span do! test - stop span - addOutcome Passed span - setStatus ActivityStatusCode.Ok span + handleSuccess span with | e -> - addExceptionOutcomeToSpan span e - reraiseAnywhere e + handleFailure span e }) | AsyncFsCheck (testConfig, stressConfig, test) -> TestCode.AsyncFsCheck (testConfig, stressConfig, fun fsCheckConfig -> async { + use span = start span try - start span do! test fsCheckConfig - stop span - addOutcome Passed span - setStatus ActivityStatusCode.Ok span + handleSuccess span with | e -> - addExceptionOutcomeToSpan span e - reraiseAnywhere e + handleFailure span e }) | SyncWithCancel test-> TestCode.SyncWithCancel (fun ct -> + use span = start span try - start span test ct - stop span - addOutcome Passed span - setStatus ActivityStatusCode.Ok span + handleSuccess span with | e -> - addExceptionOutcomeToSpan span e - reraiseAnywhere e + handleFailure span e ) + let addOpenTelemetry_SpanPerTest (config: ExpectoConfig) (activitySource: ActivitySource) (rootTest: Test) : Test = rootTest |> Test.toTestCodeList |> List.map (fun test -> - let span = createActivity (config.joinWith.format test.name) (Some activitySource) + let span = activitySource |> createActivity (config.joinWith.format test.name) span |> setSourceLocation (config.locate test.test) {test with test = wrapCodeWithSpan span test.test} ) |> Test.fromFlatTests config.joinWith.asString + From 25194a111c041a0bb9898185236b29b10980a659 Mon Sep 17 00:00:00 2001 From: Jimmy Byrd Date: Thu, 18 Apr 2024 22:51:58 -0400 Subject: [PATCH 06/11] remove ActivitySource cli flag --- Expecto/Expecto.fs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Expecto/Expecto.fs b/Expecto/Expecto.fs index 2d9c1b70..4dd3c7dd 100644 --- a/Expecto/Expecto.fs +++ b/Expecto/Expecto.fs @@ -445,8 +445,7 @@ module Tests = | Append_Summary_Handler of SummaryHandler /// Specify test names join character. | JoinWith of split: string - // TODO This isn't really a CLIArgument but just to show a way of updating the config - | ActivitySource of ActivitySource + let options = [ "--sequenced", "Don't run the tests in parallel.", Args.none Sequenced @@ -534,7 +533,6 @@ module Tests = | Printer p -> fun o -> { o with printer = p } | Verbosity l -> fun o -> { o with verbosity = l } | Append_Summary_Handler (SummaryHandler h) -> fun o -> o.appendSummaryHandler h - | ActivitySource s -> fun o -> { o with activitySource = Option.ofObj s } [] module ExpectoConfig = From 4d32e4b4f21740eed02143bac74c09218be620e1 Mon Sep 17 00:00:00 2001 From: Jimmy Byrd Date: Thu, 18 Apr 2024 22:53:27 -0400 Subject: [PATCH 07/11] Set provider to be disposed at process exit --- Expecto.Tests/OpenTelemetry.fs | 6 +++++- Expecto.Tests/Tests.fs | 37 ++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/Expecto.Tests/OpenTelemetry.fs b/Expecto.Tests/OpenTelemetry.fs index 86ed10ef..9cb7de8d 100644 --- a/Expecto.Tests/OpenTelemetry.fs +++ b/Expecto.Tests/OpenTelemetry.fs @@ -164,7 +164,6 @@ module OpenTelemetry = let addOpenTelemetry_SpanPerTest (config: ExpectoConfig) (activitySource: ActivitySource) (rootTest: Test) : Test = - rootTest |> Test.toTestCodeList |> List.map (fun test -> @@ -174,3 +173,8 @@ module OpenTelemetry = ) |> Test.fromFlatTests config.joinWith.asString + + let addOpenTelemetry_SpanPerTest_WithProvider (config: ExpectoConfig) (activitySource: ActivitySource) (providerFactory : unit -> #IDisposable) (rootTest: Test) : Test = + let provider = providerFactory() + AppDomain.CurrentDomain.ProcessExit.Add(fun _ -> provider.Dispose()) + addOpenTelemetry_SpanPerTest config activitySource rootTest \ No newline at end of file diff --git a/Expecto.Tests/Tests.fs b/Expecto.Tests/Tests.fs index 7fc608ac..0b06d936 100644 --- a/Expecto.Tests/Tests.fs +++ b/Expecto.Tests/Tests.fs @@ -1380,6 +1380,8 @@ let asyncTests = ] open System.Threading.Tasks +open OpenTelemetry +open System.Diagnostics [] let taskTests = @@ -1855,3 +1857,38 @@ let theory = } ] ] + +open OpenTelemetry.Resources +open OpenTelemetry.Trace + +let serviceName = "Expecto.Tests" + +let resourceBuilder () = + ResourceBuilder + .CreateDefault() + .AddService(serviceName = serviceName) + +let traceProvider () = + Sdk + .CreateTracerProviderBuilder() + .AddSource(serviceName) + .SetResourceBuilder(resourceBuilder ()) + .AddOtlpExporter() + .Build() + + +[] +let fixtures = + let rng = Random() + let tests = [ + for i in 1..(Environment.ProcessorCount * 2) do + testCaseAsync (sprintf "test %d" i) <| async { + printfn "Running test %d" i + do! Async.Sleep(rng.Next(1, 5000)) + printfn "Finished Running test %d" i + } + ] + + let source = new ActivitySource(serviceName) + testList "MyTests" tests + |> addOpenTelemetry_SpanPerTest_WithProvider ExpectoConfig.defaultConfig (source) (fun () -> traceProvider ()) From 41848d9a3c6636f95591b9172699d4faf84e4329 Mon Sep 17 00:00:00 2001 From: Jimmy Byrd Date: Thu, 18 Apr 2024 22:53:40 -0400 Subject: [PATCH 08/11] Show working with yolodev --- paket.dependencies | 2 ++ paket.lock | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/paket.dependencies b/paket.dependencies index ba9d5aa4..ce6e1219 100644 --- a/paket.dependencies +++ b/paket.dependencies @@ -10,6 +10,8 @@ nuget DiffPlex ~> 1.5 nuget Mono.Cecil ~> 0.11 nuget BenchmarkDotNet ~> 0.13.5 nuget OpenTelemetry.Exporter.OpenTelemetryProtocol +nuget YoloDev.Expecto.TestSdk +nuget Microsoft.NET.Test.Sdk group FsCheck3 source https://api.nuget.org/v3/index.json diff --git a/paket.lock b/paket.lock index 7870e1bc..0713eaa3 100644 --- a/paket.lock +++ b/paket.lock @@ -21,6 +21,9 @@ NUGET BenchmarkDotNet.Annotations (0.13.5) CommandLineParser (2.7.82) DiffPlex (1.7.1) + Expecto (10.2.1) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) + FSharp.Core (>= 7.0.200) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) + Mono.Cecil (>= 0.11.4 < 1.0) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) FsCheck (2.16.5) FSharp.Core (>= 4.2.3) FSharp.Core (7.0.200) @@ -50,6 +53,7 @@ NUGET System.Threading.Tasks.Extensions (>= 4.5.3) Microsoft.CodeAnalysis.CSharp (3.5) Microsoft.CodeAnalysis.Common (3.5) + Microsoft.CodeCoverage (17.9) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net462)) (&& (== netstandard2.1) (>= netcoreapp3.1)) Microsoft.Diagnostics.NETCore.Client (0.2.410101) Microsoft.Bcl.AsyncInterfaces (>= 1.1) Microsoft.Extensions.Logging (>= 2.1.1) @@ -107,13 +111,22 @@ NUGET Microsoft.Extensions.Primitives (8.0) System.Memory (>= 4.5.5) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) System.Runtime.CompilerServices.Unsafe (>= 6.0) + Microsoft.NET.Test.Sdk (17.9) + Microsoft.CodeCoverage (>= 17.9) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net462)) (&& (== netstandard2.1) (>= netcoreapp3.1)) + Microsoft.TestPlatform.TestHost (>= 17.9) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp3.1)) Microsoft.NETCore.Platforms (3.1) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp2.0)) (&& (== netstandard2.1) (>= netcoreapp3.1)) + Microsoft.TestPlatform.ObjectModel (17.9) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp3.1)) + System.Reflection.Metadata (>= 1.6) + Microsoft.TestPlatform.TestHost (17.9) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp3.1)) + Microsoft.TestPlatform.ObjectModel (>= 17.9) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp3.1)) + Newtonsoft.Json (>= 13.0.1) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp3.1)) Microsoft.Win32.Registry (5.0) - restriction: == netstandard2.1 System.Buffers (>= 4.5.1) - restriction: || (&& (== net6.0) (>= monoandroid) (< netstandard1.3)) (&& (== net6.0) (>= monotouch)) (&& (== net6.0) (< netcoreapp2.0)) (&& (== net6.0) (>= xamarinios)) (&& (== net6.0) (>= xamarinmac)) (&& (== net6.0) (>= xamarintvos)) (&& (== net6.0) (>= xamarinwatchos)) (== netstandard2.1) System.Memory (>= 4.5.4) - restriction: || (&& (== net6.0) (< netcoreapp2.0)) (&& (== net6.0) (< netcoreapp2.1)) (&& (== net6.0) (>= uap10.1)) (== netstandard2.1) System.Security.AccessControl (>= 5.0) System.Security.Principal.Windows (>= 5.0) Mono.Cecil (0.11.4) + Newtonsoft.Json (13.0.3) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp3.1)) OpenTelemetry (1.8) Microsoft.Extensions.Diagnostics.Abstractions (>= 8.0) Microsoft.Extensions.Logging.Configuration (>= 8.0) @@ -132,7 +145,6 @@ NUGET System.Buffers (4.5.1) - restriction: == netstandard2.1 System.CodeDom (7.0) System.Collections.Immutable (7.0) - System.Memory (>= 4.5.5) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) System.Runtime.CompilerServices.Unsafe (>= 6.0) System.ComponentModel.Annotations (5.0) - restriction: || (&& (== net6.0) (< netstandard2.1)) (== netstandard2.1) System.Diagnostics.DiagnosticSource (8.0) @@ -158,6 +170,10 @@ NUGET System.Runtime.CompilerServices.Unsafe (>= 4.7) - restriction: || (&& (== net6.0) (>= net461)) (&& (== net6.0) (< netcoreapp2.0)) (&& (== net6.0) (< netcoreapp3.1)) (== netstandard2.1) System.Threading.Tasks.Extensions (4.5.4) System.Runtime.CompilerServices.Unsafe (>= 4.5.3) - restriction: || (&& (== net6.0) (>= net461)) (&& (== net6.0) (< netcoreapp2.1)) (&& (== net6.0) (< netstandard1.0)) (&& (== net6.0) (< netstandard2.0)) (&& (== net6.0) (>= wp8)) (== netstandard2.1) + YoloDev.Expecto.TestSdk (0.14.3) + Expecto (>= 10.0 < 11.0) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) + FSharp.Core (>= 7.0.200) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) + System.Collections.Immutable (>= 6.0) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) GROUP Build STORAGE: NONE From 2a952bffa157767fb1dbfbd4dfa24ab2bbcba0a9 Mon Sep 17 00:00:00 2001 From: Jimmy Byrd Date: Thu, 18 Apr 2024 22:54:29 -0400 Subject: [PATCH 09/11] cleanup --- Expecto.Tests/Main.fs | 23 +++-------------------- Expecto.Tests/OpenTelemetry.fs | 1 - 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/Expecto.Tests/Main.fs b/Expecto.Tests/Main.fs index 8e10c150..fb043e7b 100644 --- a/Expecto.Tests/Main.fs +++ b/Expecto.Tests/Main.fs @@ -7,38 +7,21 @@ open OpenTelemetry open OpenTelemetry.Trace open System.Threading open System.Diagnostics +open System let serviceName = "Expecto.Tests" let logger = Log.create serviceName -let resourceBuilder = - ResourceBuilder - .CreateDefault() - .AddService(serviceName = serviceName) - - - [] let main args = - let activitySource = new ActivitySource(serviceName) - use traceProvider = - Sdk - .CreateTracerProviderBuilder() - .AddSource(serviceName) - .SetResourceBuilder(resourceBuilder ) - .AddOtlpExporter() - .Build() - let tracer = traceProvider.GetTracer(serviceName) - // use span = tracer.StartActiveSpan("Expecto.main") - use span = tracer.StartRootSpan("Expecto.main") + let test = Impl.testFromThisAssembly() |> Option.orDefault (TestList ([], Normal)) - |> OpenTelemetry.addOpenTelemetry_SpanPerTest Impl.ExpectoConfig.defaultConfig activitySource |> Test.shuffle "." - runTestsWithCLIArgs [NUnit_Summary "bin/Expecto.Tests.TestResults.xml"] args test + runTestsWithCLIArgs [NUnit_Summary "bin/Expecto.Tests.TestResults.xml";] args test diff --git a/Expecto.Tests/OpenTelemetry.fs b/Expecto.Tests/OpenTelemetry.fs index 9cb7de8d..343070b6 100644 --- a/Expecto.Tests/OpenTelemetry.fs +++ b/Expecto.Tests/OpenTelemetry.fs @@ -121,7 +121,6 @@ module OpenTelemetry = | Sync test -> TestCode.Sync (fun () -> use span = start span - File.AppendAllText(Path.Combine(__SOURCE_DIRECTORY__, "wrapCodeWithSpan.log"), $"{span.DisplayName}\n") try test () handleSuccess span From d3db9f71a04bfe11300efeb3e0f62d4a5e2e93d7 Mon Sep 17 00:00:00 2001 From: Jimmy Byrd Date: Thu, 18 Apr 2024 22:56:12 -0400 Subject: [PATCH 10/11] cleanup --- Expecto/Expecto.Impl.fs | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/Expecto/Expecto.Impl.fs b/Expecto/Expecto.Impl.fs index ebcb8d3e..83d33665 100644 --- a/Expecto/Expecto.Impl.fs +++ b/Expecto/Expecto.Impl.fs @@ -10,20 +10,6 @@ open Expecto.Logging.Message open Helpers open Mono.Cecil -//! The other option is to use a dedicated activity source for Expecto instead of adding it to the config - -// module ActivitySource = - -// let [] serviceName = "Expecto" // Should be public so consumers have a strong name when adding Sources -// let private version = lazy ( -// let assembly = typeof.Assembly -// let version = assembly.GetName().Version -// version.ToString() -// ) - -// let internal activitySource = lazy new ActivitySource(serviceName, version.Value) - - // TODO: make internal? module Impl = @@ -535,11 +521,6 @@ module Impl = colour: ColourLevel /// Split test names by `.` or `/` joinWith: JoinWith - // One option is to allow the consumer to provide an activity source - // only problem is the only way to update the config is by using the CLIArguments currently - // we would have to add a new CLIArgument but that doesn't really work as it's not a reallyCLI option - // or have another way of updating the config after it's been created - activitySource : ActivitySource option } static member defaultConfig = { runInParallel = true @@ -566,7 +547,6 @@ module Impl = noSpinner = false colour = Colour8 joinWith = JoinWith.Dot - activitySource = None } member x.appendSummaryHandler handleSummary = From 40068a0db08e91811073bfdaa63b927e37aa35b3 Mon Sep 17 00:00:00 2001 From: Jimmy Byrd Date: Thu, 18 Apr 2024 23:16:05 -0400 Subject: [PATCH 11/11] Add provider to ProcessExit --- Expecto.Tests/Expecto.Tests.fsproj | 3 +- Expecto.Tests/OpenTelemetry.fs | 7 ----- Expecto.Tests/Tests.fs | 47 ++++++++++++++++++------------ Expecto.Tests/paket.references | 4 ++- 4 files changed, 33 insertions(+), 28 deletions(-) diff --git a/Expecto.Tests/Expecto.Tests.fsproj b/Expecto.Tests/Expecto.Tests.fsproj index 84b1b122..33e53e9c 100644 --- a/Expecto.Tests/Expecto.Tests.fsproj +++ b/Expecto.Tests/Expecto.Tests.fsproj @@ -4,16 +4,17 @@ Expecto.Tests Exe net6.0 + false + - diff --git a/Expecto.Tests/OpenTelemetry.fs b/Expecto.Tests/OpenTelemetry.fs index 343070b6..57f9603c 100644 --- a/Expecto.Tests/OpenTelemetry.fs +++ b/Expecto.Tests/OpenTelemetry.fs @@ -160,8 +160,6 @@ module OpenTelemetry = handleFailure span e ) - - let addOpenTelemetry_SpanPerTest (config: ExpectoConfig) (activitySource: ActivitySource) (rootTest: Test) : Test = rootTest |> Test.toTestCodeList @@ -172,8 +170,3 @@ module OpenTelemetry = ) |> Test.fromFlatTests config.joinWith.asString - - let addOpenTelemetry_SpanPerTest_WithProvider (config: ExpectoConfig) (activitySource: ActivitySource) (providerFactory : unit -> #IDisposable) (rootTest: Test) : Test = - let provider = providerFactory() - AppDomain.CurrentDomain.ProcessExit.Add(fun _ -> provider.Dispose()) - addOpenTelemetry_SpanPerTest config activitySource rootTest \ No newline at end of file diff --git a/Expecto.Tests/Tests.fs b/Expecto.Tests/Tests.fs index 0b06d936..2f3ce2e2 100644 --- a/Expecto.Tests/Tests.fs +++ b/Expecto.Tests/Tests.fs @@ -10,6 +10,31 @@ open Expecto open Expecto.Impl open Expecto.Logging open System.Globalization +open OpenTelemetry.Resources +open OpenTelemetry.Trace +open System.Diagnostics +open OpenTelemetry + +let serviceName = "Expecto.Tests" + +let source = new ActivitySource(serviceName) + +let resourceBuilder () = + ResourceBuilder + .CreateDefault() + .AddService(serviceName = serviceName) + +let traceProvider () = + Sdk + .CreateTracerProviderBuilder() + .AddSource(serviceName) + .SetResourceBuilder(resourceBuilder ()) + .AddOtlpExporter() + .Build() +do + let provider = traceProvider() + AppDomain.CurrentDomain.ProcessExit.Add(fun _ -> provider.Dispose()) + module Dummy = @@ -1830,6 +1855,7 @@ let cancel = ) ] + [] let theory = testList "theory testing" [ @@ -1857,24 +1883,8 @@ let theory = } ] ] + |> addOpenTelemetry_SpanPerTest ExpectoConfig.defaultConfig source -open OpenTelemetry.Resources -open OpenTelemetry.Trace - -let serviceName = "Expecto.Tests" - -let resourceBuilder () = - ResourceBuilder - .CreateDefault() - .AddService(serviceName = serviceName) - -let traceProvider () = - Sdk - .CreateTracerProviderBuilder() - .AddSource(serviceName) - .SetResourceBuilder(resourceBuilder ()) - .AddOtlpExporter() - .Build() [] @@ -1889,6 +1899,5 @@ let fixtures = } ] - let source = new ActivitySource(serviceName) testList "MyTests" tests - |> addOpenTelemetry_SpanPerTest_WithProvider ExpectoConfig.defaultConfig (source) (fun () -> traceProvider ()) + |> addOpenTelemetry_SpanPerTest ExpectoConfig.defaultConfig source diff --git a/Expecto.Tests/paket.references b/Expecto.Tests/paket.references index 1d6bbdfc..4c4484e0 100644 --- a/Expecto.Tests/paket.references +++ b/Expecto.Tests/paket.references @@ -1,2 +1,4 @@ FsCheck -OpenTelemetry.Exporter.OpenTelemetryProtocol \ No newline at end of file +OpenTelemetry.Exporter.OpenTelemetryProtocol +YoloDev.Expecto.TestSdk +Microsoft.NET.Test.Sdk \ No newline at end of file