diff --git a/README.md b/README.md index b0333937..405b5d5a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![NuGet version](https://img.shields.io/nuget/vpre/openai.svg)](https://www.nuget.org/packages/OpenAI/absoluteLatest) -The OpenAI .NET library provides convenient access to the OpenAI REST API from .NET applications. +The OpenAI .NET library provides convenient access to the OpenAI REST API from .NET applications. It is generated from our [OpenAPI specification](https://github.com/openai/openai-openapi) in collaboration with Microsoft. @@ -26,6 +26,7 @@ It is generated from our [OpenAPI specification](https://github.com/openai/opena - [Advanced scenarios](#advanced-scenarios) - [Using protocol methods](#using-protocol-methods) - [Automatically retrying errors](#automatically-retrying-errors) +- [Observability](#observability) ## Getting started @@ -714,7 +715,7 @@ For example, to use the protocol method variant of the `ChatClient`'s `CompleteC ChatClient client = new("gpt-4o", Environment.GetEnvironmentVariable("OPENAI_API_KEY")); BinaryData input = BinaryData.FromBytes(""" -{ +{ "model": "gpt-4o", "messages": [ { @@ -749,3 +750,7 @@ By default, the client classes will automatically retry the following errors up - 502 Bad Gateway - 503 Service Unavailable - 504 Gateway Timeout + +## Observability + +OpenAI .NET library supports experimental distributed tracing and metrics with OpenTelemetry. Check out [Observability with OpenTelemetry](./docs/observability.md) for more details. diff --git a/docs/observability.md b/docs/observability.md new file mode 100644 index 00000000..8dd3290a --- /dev/null +++ b/docs/observability.md @@ -0,0 +1,57 @@ +## Observability with OpenTelemetry + +> Note: +> OpenAI .NET SDK instrumentation is in development and is not complete. See [Available sources and meters](#available-sources-and-meters) section for the list of covered operations. + +OpenAI .NET library is instrumented with distributed tracing and metrics using .NET [tracing](https://learn.microsoft.com/dotnet/core/diagnostics/distributed-tracing) +and [metrics](https://learn.microsoft.com/dotnet/core/diagnostics/metrics-instrumentation) API and supports [OpenTelemetry](https://learn.microsoft.com/dotnet/core/diagnostics/observability-with-otel). + +OpenAI .NET instrumentation follows [OpenTelemetry Semantic Conventions for Generative AI systems](https://github.com/open-telemetry/semantic-conventions/tree/main/docs/gen-ai). + +### How to enable + +The instrumentation is **experimental** - volume and semantics of the telemetry items may change. + +To enable the instrumentation: + +1. Set instrumentation feature-flag using one of the following options: + + - set the `OPENAI_EXPERIMENTAL_ENABLE_OPEN_TELEMETRY` environment variable to `"true"` + - set the `OpenAI.Experimental.EnableOpenTelemetry` context switch to true in your application code when application + is starting and before initializing any OpenAI clients. For example: + + ```csharp + AppContext.SetSwitch("OpenAI.Experimental.EnableOpenTelemetry", true); + ``` + +2. Enable OpenAI telemetry: + + ```csharp + builder.Services.AddOpenTelemetry() + .WithTracing(b => + { + b.AddSource("OpenAI.*") + ... + .AddOtlpExporter(); + }) + .WithMetrics(b => + { + b.AddMeter("OpenAI.*") + ... + .AddOtlpExporter(); + }); + ``` + + Distributed tracing is enabled with `AddSource("OpenAI.*")` which tells OpenTelemetry to listen to all [ActivitySources](https://learn.microsoft.com/dotnet/api/system.diagnostics.activitysource) with names starting with `OpenAI.*`. + + Similarly, metrics are configured with `AddMeter("OpenAI.*")` which enables all OpenAI-related [Meters](https://learn.microsoft.com/dotnet/api/system.diagnostics.metrics.meter). + +Consider enabling [HTTP client instrumentation](https://www.nuget.org/packages/OpenTelemetry.Instrumentation.Http) to see all HTTP client +calls made by your application including those done by the OpenAI SDK. +Check out [OpenTelemetry documentation](https://opentelemetry.io/docs/languages/net/getting-started/) for more details. + +### Available sources and meters + +The following sources and meters are available: + +- `OpenAI.ChatClient` - records traces and metrics for `ChatClient` operations (except streaming and protocol methods which are not instrumented yet) diff --git a/src/Custom/Chat/ChatClient.cs b/src/Custom/Chat/ChatClient.cs index 54c31bed..2f366b9a 100644 --- a/src/Custom/Chat/ChatClient.cs +++ b/src/Custom/Chat/ChatClient.cs @@ -1,3 +1,4 @@ +using OpenAI.Telemetry; using System; using System.ClientModel; using System.ClientModel.Primitives; @@ -14,6 +15,7 @@ namespace OpenAI.Chat; public partial class ChatClient { private readonly string _model; + private readonly OpenTelemetrySource _telemetry; /// /// Initializes a new instance of that will use an API key when authenticating. @@ -62,6 +64,7 @@ protected internal ChatClient(ClientPipeline pipeline, string model, Uri endpoin _model = model; _pipeline = pipeline; _endpoint = endpoint; + _telemetry = new OpenTelemetrySource(model, endpoint); } /// @@ -77,11 +80,22 @@ public virtual async Task> CompleteChatAsync(IEnume options ??= new(); CreateChatCompletionOptions(messages, ref options); - - using BinaryContent content = options.ToBinaryContent(); - - ClientResult result = await CompleteChatAsync(content, cancellationToken.ToRequestOptions()).ConfigureAwait(false); - return ClientResult.FromValue(ChatCompletion.FromResponse(result.GetRawResponse()), result.GetRawResponse()); + using OpenTelemetryScope scope = _telemetry.StartChatScope(options); + + try + { + using BinaryContent content = options.ToBinaryContent(); + + ClientResult result = await CompleteChatAsync(content, cancellationToken.ToRequestOptions()).ConfigureAwait(false); + ChatCompletion chatCompletion = ChatCompletion.FromResponse(result.GetRawResponse()); + scope?.RecordChatCompletion(chatCompletion); + return ClientResult.FromValue(chatCompletion, result.GetRawResponse()); + } + catch (Exception ex) + { + scope?.RecordException(ex); + throw; + } } /// @@ -105,11 +119,22 @@ public virtual ClientResult CompleteChat(IEnumerable @@ -200,7 +225,7 @@ private void CreateChatCompletionOptions(IEnumerable messages, ref { options.Messages = messages.ToList(); options.Model = _model; - options.Stream = stream + options.Stream = stream ? true : null; options.StreamOptions = stream ? options.StreamOptions : null; diff --git a/src/OpenAI.csproj b/src/OpenAI.csproj index eed204d5..7d19ce11 100644 --- a/src/OpenAI.csproj +++ b/src/OpenAI.csproj @@ -12,7 +12,7 @@ true - + true OpenAI.png @@ -21,7 +21,7 @@ true snupkg - + true @@ -29,7 +29,7 @@ $(NoWarn),1570,1573,1574,1591 - + $(NoWarn),0618 @@ -63,7 +63,6 @@ true - @@ -73,5 +72,6 @@ + diff --git a/src/Utility/AppContextSwitchHelper.cs b/src/Utility/AppContextSwitchHelper.cs new file mode 100644 index 00000000..34d98529 --- /dev/null +++ b/src/Utility/AppContextSwitchHelper.cs @@ -0,0 +1,33 @@ +using System; + +namespace OpenAI; + +internal static class AppContextSwitchHelper +{ + /// + /// Determines if either an AppContext switch or its corresponding Environment Variable is set + /// + /// Name of the AppContext switch. + /// Name of the Environment variable. + /// If the AppContext switch has been set, returns the value of the switch. + /// If the AppContext switch has not been set, returns the value of the environment variable. + /// False if neither is set. + /// + public static bool GetConfigValue(string appContexSwitchName, string environmentVariableName) + { + // First check for the AppContext switch, giving it priority over the environment variable. + if (AppContext.TryGetSwitch(appContexSwitchName, out bool value)) + { + return value; + } + // AppContext switch wasn't used. Check the environment variable. + string envVar = Environment.GetEnvironmentVariable(environmentVariableName); + if (envVar != null && (envVar.Equals("true", StringComparison.OrdinalIgnoreCase) || envVar.Equals("1"))) + { + return true; + } + + // Default to false. + return false; + } +} diff --git a/src/Utility/Telemetry/OpenTelemetryConstants.cs b/src/Utility/Telemetry/OpenTelemetryConstants.cs new file mode 100644 index 00000000..d5a0906a --- /dev/null +++ b/src/Utility/Telemetry/OpenTelemetryConstants.cs @@ -0,0 +1,33 @@ +namespace OpenAI.Telemetry; + +internal class OpenTelemetryConstants +{ + // follow OpenTelemetry GenAI semantic conventions: + // https://github.com/open-telemetry/semantic-conventions/tree/v1.27.0/docs/gen-ai + + public const string ErrorTypeKey = "error.type"; + public const string ServerAddressKey = "server.address"; + public const string ServerPortKey = "server.port"; + + public const string GenAiClientOperationDurationMetricName = "gen_ai.client.operation.duration"; + public const string GenAiClientTokenUsageMetricName = "gen_ai.client.token.usage"; + + public const string GenAiOperationNameKey = "gen_ai.operation.name"; + + public const string GenAiRequestMaxTokensKey = "gen_ai.request.max_tokens"; + public const string GenAiRequestModelKey = "gen_ai.request.model"; + public const string GenAiRequestTemperatureKey = "gen_ai.request.temperature"; + public const string GenAiRequestTopPKey = "gen_ai.request.top_p"; + + public const string GenAiResponseIdKey = "gen_ai.response.id"; + public const string GenAiResponseFinishReasonKey = "gen_ai.response.finish_reasons"; + public const string GenAiResponseModelKey = "gen_ai.response.model"; + + public const string GenAiSystemKey = "gen_ai.system"; + public const string GenAiSystemValue = "openai"; + + public const string GenAiTokenTypeKey = "gen_ai.token.type"; + + public const string GenAiUsageInputTokensKey = "gen_ai.usage.input_tokens"; + public const string GenAiUsageOutputTokensKey = "gen_ai.usage.output_tokens"; +} diff --git a/src/Utility/Telemetry/OpenTelemetryScope.cs b/src/Utility/Telemetry/OpenTelemetryScope.cs new file mode 100644 index 00000000..8cdda7e2 --- /dev/null +++ b/src/Utility/Telemetry/OpenTelemetryScope.cs @@ -0,0 +1,222 @@ +using OpenAI.Chat; +using System; +using System.ClientModel; +using System.Diagnostics; +using System.Diagnostics.Metrics; + +using static OpenAI.Telemetry.OpenTelemetryConstants; + +namespace OpenAI.Telemetry; + +internal class OpenTelemetryScope : IDisposable +{ + private static readonly ActivitySource s_chatSource = new ActivitySource("OpenAI.ChatClient"); + private static readonly Meter s_chatMeter = new Meter("OpenAI.ChatClient"); + + // TODO: add explicit histogram buckets once System.Diagnostics.DiagnosticSource 9.0 is used + private static readonly Histogram s_duration = s_chatMeter.CreateHistogram(GenAiClientOperationDurationMetricName, "s", "Measures GenAI operation duration."); + private static readonly Histogram s_tokens = s_chatMeter.CreateHistogram(GenAiClientTokenUsageMetricName, "{token}", "Measures the number of input and output token used."); + + private readonly string _operationName; + private readonly string _serverAddress; + private readonly int _serverPort; + private readonly string _requestModel; + + private Stopwatch _duration; + private Activity _activity; + private TagList _commonTags; + + private OpenTelemetryScope( + string model, string operationName, + string serverAddress, int serverPort) + { + _requestModel = model; + _operationName = operationName; + _serverAddress = serverAddress; + _serverPort = serverPort; + } + + private static bool IsChatEnabled => s_chatSource.HasListeners() || s_tokens.Enabled || s_duration.Enabled; + + public static OpenTelemetryScope StartChat(string model, string operationName, + string serverAddress, int serverPort, ChatCompletionOptions options) + { + if (IsChatEnabled) + { + var scope = new OpenTelemetryScope(model, operationName, serverAddress, serverPort); + scope.StartChat(options); + return scope; + } + + return null; + } + + private void StartChat(ChatCompletionOptions options) + { + _duration = Stopwatch.StartNew(); + _commonTags = new TagList + { + { GenAiSystemKey, GenAiSystemValue }, + { GenAiRequestModelKey, _requestModel }, + { ServerAddressKey, _serverAddress }, + { ServerPortKey, _serverPort }, + { GenAiOperationNameKey, _operationName }, + }; + + _activity = s_chatSource.StartActivity(string.Concat(_operationName, " ", _requestModel), ActivityKind.Client); + if (_activity?.IsAllDataRequested == true) + { + RecordCommonAttributes(); + SetActivityTagIfNotNull(GenAiRequestMaxTokensKey, options?.MaxTokens); + SetActivityTagIfNotNull(GenAiRequestTemperatureKey, options?.Temperature); + SetActivityTagIfNotNull(GenAiRequestTopPKey, options?.TopP); + } + + return; + } + + public void RecordChatCompletion(ChatCompletion completion) + { + RecordMetrics(completion.Model, null, completion.Usage?.InputTokens, completion.Usage?.OutputTokens); + + if (_activity?.IsAllDataRequested == true) + { + RecordResponseAttributes(completion.Id, completion.Model, completion.FinishReason, completion.Usage); + } + } + + public void RecordException(Exception ex) + { + var errorType = GetErrorType(ex); + RecordMetrics(null, errorType, null, null); + if (_activity?.IsAllDataRequested == true) + { + _activity?.SetTag(OpenTelemetryConstants.ErrorTypeKey, errorType); + _activity?.SetStatus(ActivityStatusCode.Error, ex?.Message ?? errorType); + } + } + + public void Dispose() + { + _activity?.Stop(); + } + + private void RecordCommonAttributes() + { + _activity.SetTag(GenAiSystemKey, GenAiSystemValue); + _activity.SetTag(GenAiRequestModelKey, _requestModel); + _activity.SetTag(ServerAddressKey, _serverAddress); + _activity.SetTag(ServerPortKey, _serverPort); + _activity.SetTag(GenAiOperationNameKey, _operationName); + } + + private void RecordMetrics(string responseModel, string errorType, int? inputTokensUsage, int? outputTokensUsage) + { + // tags is a struct, let's copy and modify them + var tags = _commonTags; + + if (responseModel != null) + { + tags.Add(GenAiResponseModelKey, responseModel); + } + + if (inputTokensUsage != null) + { + var inputUsageTags = tags; + inputUsageTags.Add(GenAiTokenTypeKey, "input"); + s_tokens.Record(inputTokensUsage.Value, inputUsageTags); + } + + if (outputTokensUsage != null) + { + var outputUsageTags = tags; + outputUsageTags.Add(GenAiTokenTypeKey, "output"); + s_tokens.Record(outputTokensUsage.Value, outputUsageTags); + } + + if (errorType != null) + { + tags.Add(ErrorTypeKey, errorType); + } + + s_duration.Record(_duration.Elapsed.TotalSeconds, tags); + } + + private void RecordResponseAttributes(string responseId, string model, ChatFinishReason? finishReason, ChatTokenUsage usage) + { + SetActivityTagIfNotNull(GenAiResponseIdKey, responseId); + SetActivityTagIfNotNull(GenAiResponseModelKey, model); + SetActivityTagIfNotNull(GenAiUsageInputTokensKey, usage?.InputTokens); + SetActivityTagIfNotNull(GenAiUsageOutputTokensKey, usage?.OutputTokens); + SetFinishReasonAttribute(finishReason); + } + + private void SetFinishReasonAttribute(ChatFinishReason? finishReason) + { + if (finishReason == null) + { + return; + } + + var reasonStr = finishReason switch + { + ChatFinishReason.ContentFilter => "content_filter", + ChatFinishReason.FunctionCall => "function_call", + ChatFinishReason.Length => "length", + ChatFinishReason.Stop => "stop", + ChatFinishReason.ToolCalls => "tool_calls", + _ => finishReason.ToString(), + }; + + // There could be multiple finish reasons, so semantic conventions use array type for the corrresponding attribute. + // It's likely to change, but for now let's report it as array. + _activity.SetTag(GenAiResponseFinishReasonKey, new[] { reasonStr }); + } + + private string GetChatMessageRole(ChatMessageRole? role) => + role switch + { + ChatMessageRole.Assistant => "assistant", + ChatMessageRole.Function => "function", + ChatMessageRole.System => "system", + ChatMessageRole.Tool => "tool", + ChatMessageRole.User => "user", + _ => role?.ToString(), + }; + + private string GetErrorType(Exception exception) + { + if (exception is ClientResultException requestFailedException) + { + // TODO (lmolkova) when we start targeting .NET 8 we should put + // requestFailedException.InnerException.HttpRequestError into error.type + return requestFailedException.Status.ToString(); + } + + return exception?.GetType()?.FullName; + } + + private void SetActivityTagIfNotNull(string name, object value) + { + if (value != null) + { + _activity.SetTag(name, value); + } + } + + private void SetActivityTagIfNotNull(string name, int? value) + { + if (value.HasValue) + { + _activity.SetTag(name, value.Value); + } + } + + private void SetActivityTagIfNotNull(string name, float? value) + { + if (value.HasValue) + { + _activity.SetTag(name, value.Value); + } + } +} diff --git a/src/Utility/Telemetry/OpenTelemetrySource.cs b/src/Utility/Telemetry/OpenTelemetrySource.cs new file mode 100644 index 00000000..a0ac1fe4 --- /dev/null +++ b/src/Utility/Telemetry/OpenTelemetrySource.cs @@ -0,0 +1,30 @@ +using OpenAI.Chat; +using System; + +namespace OpenAI.Telemetry; + +internal class OpenTelemetrySource +{ + private const string ChatOperationName = "chat"; + private readonly bool IsOTelEnabled = AppContextSwitchHelper + .GetConfigValue("OpenAI.Experimental.EnableOpenTelemetry", "OPENAI_EXPERIMENTAL_ENABLE_OPEN_TELEMETRY"); + + private readonly string _serverAddress; + private readonly int _serverPort; + private readonly string _model; + + public OpenTelemetrySource(string model, Uri endpoint) + { + _serverAddress = endpoint.Host; + _serverPort = endpoint.Port; + _model = model; + } + + public OpenTelemetryScope StartChatScope(ChatCompletionOptions completionsOptions) + { + return IsOTelEnabled + ? OpenTelemetryScope.StartChat(_model, ChatOperationName, _serverAddress, _serverPort, completionsOptions) + : null; + } + +} diff --git a/tests/Chat/ChatSmokeTests.cs b/tests/Chat/ChatSmokeTests.cs index a3849d3a..80d7e6b1 100644 --- a/tests/Chat/ChatSmokeTests.cs +++ b/tests/Chat/ChatSmokeTests.cs @@ -1,14 +1,19 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel; using NUnit.Framework; using OpenAI.Chat; +using OpenAI.Tests.Telemetry; using OpenAI.Tests.Utility; using System; using System.ClientModel; using System.ClientModel.Primitives; +using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net; using System.Text.Json; using System.Threading.Tasks; +using static OpenAI.Tests.Telemetry.TestMeterListener; +using static OpenAI.Tests.TestHelpers; namespace OpenAI.Tests.Chat; diff --git a/tests/Chat/ChatTests.cs b/tests/Chat/ChatTests.cs index 5527c2d3..05fe667f 100644 --- a/tests/Chat/ChatTests.cs +++ b/tests/Chat/ChatTests.cs @@ -1,6 +1,7 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel; using NUnit.Framework; using OpenAI.Chat; +using OpenAI.Tests.Telemetry; using OpenAI.Tests.Utility; using System; using System.ClientModel; @@ -12,6 +13,7 @@ using System.Net; using System.Text.Json; using System.Threading.Tasks; +using static OpenAI.Tests.Telemetry.TestMeterListener; using static OpenAI.Tests.TestHelpers; namespace OpenAI.Tests.Chat; @@ -334,4 +336,39 @@ public async Task JsonResult() Assert.That(greenProperty.GetString().ToLowerInvariant(), Contains.Substring("00ff00")); Assert.That(blueProperty.GetString().ToLowerInvariant(), Contains.Substring("0000ff")); } + + + [Test] + [NonParallelizable] + public async Task HelloWorldChatWithTracingAndMetrics() + { + using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); + using TestActivityListener activityListener = new TestActivityListener("OpenAI.ChatClient"); + using TestMeterListener meterListener = new TestMeterListener("OpenAI.ChatClient"); + + ChatClient client = GetTestClient(TestScenario.Chat); + IEnumerable messages = [new UserChatMessage("Hello, world!")]; + ClientResult result = IsAsync + ? await client.CompleteChatAsync(messages) + : client.CompleteChat(messages); + + Assert.AreEqual(1, activityListener.Activities.Count); + TestActivityListener.ValidateChatActivity(activityListener.Activities.Single(), result.Value); + + List durations = meterListener.GetMeasurements("gen_ai.client.operation.duration"); + Assert.AreEqual(1, durations.Count); + ValidateChatMetricTags(durations.Single(), result.Value); + + List usages = meterListener.GetMeasurements("gen_ai.client.token.usage"); + Assert.AreEqual(2, usages.Count); + + Assert.True(usages[0].tags.TryGetValue("gen_ai.token.type", out var type)); + Assert.IsInstanceOf(type); + + TestMeasurement input = (type is "input") ? usages[0] : usages[1]; + TestMeasurement output = (type is "input") ? usages[1] : usages[0]; + + Assert.AreEqual(result.Value.Usage.InputTokens, input.value); + Assert.AreEqual(result.Value.Usage.OutputTokens, output.value); + } } diff --git a/tests/OpenAI.Tests.csproj b/tests/OpenAI.Tests.csproj index b33b036e..c05ab9f8 100644 --- a/tests/OpenAI.Tests.csproj +++ b/tests/OpenAI.Tests.csproj @@ -1,4 +1,4 @@ - + net8.0 @@ -15,4 +15,9 @@ + + + + + \ No newline at end of file diff --git a/tests/Telemetry/ChatTelemetryTests.cs b/tests/Telemetry/ChatTelemetryTests.cs new file mode 100644 index 00000000..d3b043a7 --- /dev/null +++ b/tests/Telemetry/ChatTelemetryTests.cs @@ -0,0 +1,315 @@ +using NUnit.Framework; +using OpenAI.Chat; +using OpenAI.Telemetry; +using System; +using System.ClientModel.Primitives; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Net.Sockets; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using static OpenAI.Tests.Telemetry.TestMeterListener; +using static OpenAI.Tests.Telemetry.TestActivityListener; + +namespace OpenAI.Tests.Telemetry; + +[TestFixture] +[NonParallelizable] +[Category("Smoke")] +public class ChatTelemetryTests +{ + private const string RequestModel = "requestModel"; + private const string Host = "host"; + private const int Port = 42; + private static readonly string Endpoint = $"https://{Host}:{Port}/path"; + private const string CompletionId = "chatcmpl-9fG9OILMJnKZARXDwxoCnLcvDsDDX"; + private const string CompletionContent = "hello world"; + private const string ResponseModel = "responseModel"; + private const string FinishReason = "stop"; + private const int PromptTokens = 2; + private const int CompletionTokens = 42; + + [Test] + public void AllTelemetryOff() + { + var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); + Assert.IsNull(telemetry.StartChatScope(new ChatCompletionOptions())); + Assert.IsNull(Activity.Current); + } + + [Test] + public void SwitchOffAllTelemetryOn() + { + using var activityListener = new TestActivityListener("OpenAI.ChatClient"); + using var meterListener = new TestMeterListener("OpenAI.ChatClient"); + var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); + Assert.IsNull(telemetry.StartChatScope(new ChatCompletionOptions())); + Assert.IsNull(Activity.Current); + } + + [Test] + public void MetricsOnTracingOff() + { + using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); + + var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); + + using var meterListener = new TestMeterListener("OpenAI.ChatClient"); + + var elapsedMax = Stopwatch.StartNew(); + using var scope = telemetry.StartChatScope(new ChatCompletionOptions()); + var elapsedMin = Stopwatch.StartNew(); + + Assert.Null(Activity.Current); + Assert.NotNull(scope); + + // so we have some duration to measure + Thread.Sleep(20); + + elapsedMin.Stop(); + + var response = CreateChatCompletion(); + scope.RecordChatCompletion(response); + scope.Dispose(); + + ValidateDuration(meterListener, response, elapsedMin.Elapsed, elapsedMax.Elapsed); + ValidateUsage(meterListener, response, PromptTokens, CompletionTokens); + } + + [Test] + public void MetricsOnTracingOffException() + { + using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); + + var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); + using var meterListener = new TestMeterListener("OpenAI.ChatClient"); + + using (var scope = telemetry.StartChatScope(new ChatCompletionOptions())) + { + scope.RecordException(new TaskCanceledException()); + } + + ValidateDuration(meterListener, null, TimeSpan.MinValue, TimeSpan.MaxValue); + Assert.IsNull(meterListener.GetMeasurements("gen_ai.client.token.usage")); + } + + [Test] + public void TracingOnMetricsOff() + { + using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); + + var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); + using var listener = new TestActivityListener("OpenAI.ChatClient"); + + var chatCompletion = CreateChatCompletion(); + + Activity activity = null; + using (var scope = telemetry.StartChatScope(new ChatCompletionOptions())) + { + activity = Activity.Current; + Assert.IsNull(activity.GetTagItem("gen_ai.request.temperature")); + Assert.IsNull(activity.GetTagItem("gen_ai.request.top_p")); + Assert.IsNull(activity.GetTagItem("gen_ai.request.max_tokens")); + + Assert.NotNull(scope); + + scope.RecordChatCompletion(chatCompletion); + } + + Assert.Null(Activity.Current); + Assert.AreEqual(1, listener.Activities.Count); + + ValidateChatActivity(listener.Activities.Single(), chatCompletion, RequestModel, Host, Port); + } + + [Test] + public void ChatTracingAllAttributes() + { + using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); + var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); + using var listener = new TestActivityListener("OpenAI.ChatClient"); + var options = new ChatCompletionOptions() + { + Temperature = 0.42f, + MaxTokens = 200, + TopP = 0.9f + }; + SetMessages(options, new UserChatMessage("hello")); + + var chatCompletion = CreateChatCompletion(); + + using (var scope = telemetry.StartChatScope(options)) + { + Assert.AreEqual(options.Temperature.Value, (float)Activity.Current.GetTagItem("gen_ai.request.temperature"), 0.01); + Assert.AreEqual(options.TopP.Value, (float)Activity.Current.GetTagItem("gen_ai.request.top_p"), 0.01); + Assert.AreEqual(options.MaxTokens.Value, Activity.Current.GetTagItem("gen_ai.request.max_tokens")); + scope.RecordChatCompletion(chatCompletion); + } + Assert.Null(Activity.Current); + + ValidateChatActivity(listener.Activities.Single(), chatCompletion, RequestModel, Host, Port); + } + + [Test] + public void ChatTracingException() + { + using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); + + var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); + using var listener = new TestActivityListener("OpenAI.ChatClient"); + + var error = new SocketException(42, "test error"); + using (var scope = telemetry.StartChatScope(new ChatCompletionOptions())) + { + scope.RecordException(error); + } + + Assert.Null(Activity.Current); + + ValidateChatActivity(listener.Activities.Single(), error, RequestModel, Host, Port); + } + + [Test] + public async Task ChatTracingAndMetricsMultiple() + { + using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); + var source = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); + + using var activityListener = new TestActivityListener("OpenAI.ChatClient"); + using var meterListener = new TestMeterListener("OpenAI.ChatClient"); + + var options = new ChatCompletionOptions(); + + var tasks = new Task[5]; + int numberOfSuccessfulResponses = 3; + int totalPromptTokens = 0, totalCompletionTokens = 0; + for (int i = 0; i < tasks.Length; i ++) + { + int t = i; + // don't let Activity.Current escape the scope + tasks[i] = Task.Run(async () => + { + using var scope = source.StartChatScope(options); + await Task.Delay(10); + if (t < numberOfSuccessfulResponses) + { + var promptTokens = Random.Shared.Next(100); + var completionTokens = Random.Shared.Next(100); + + var completion = CreateChatCompletion(promptTokens, completionTokens); + totalPromptTokens += promptTokens; + totalCompletionTokens += completionTokens; + scope.RecordChatCompletion(completion); + } + else + { + scope.RecordException(new TaskCanceledException()); + } + }); + } + + await Task.WhenAll(tasks); + + Assert.AreEqual(tasks.Length, activityListener.Activities.Count); + + var durations = meterListener.GetMeasurements("gen_ai.client.operation.duration"); + Assert.AreEqual(tasks.Length, durations.Count); + Assert.AreEqual(numberOfSuccessfulResponses, durations.Count(d => !d.tags.ContainsKey("error.type"))); + + var usages = meterListener.GetMeasurements("gen_ai.client.token.usage"); + // we don't report usage if there was no response + Assert.AreEqual(numberOfSuccessfulResponses * 2, usages.Count); + Assert.IsEmpty(usages.Where(u => u.tags.ContainsKey("error.type"))); + + Assert.AreEqual(totalPromptTokens, usages + .Where(u => u.tags.Contains(new KeyValuePair("gen_ai.token.type", "input"))) + .Sum(u => (long)u.value)); + Assert.AreEqual(totalCompletionTokens, usages + .Where(u => u.tags.Contains(new KeyValuePair("gen_ai.token.type", "output"))) + .Sum(u => (long)u.value)); + } + + private void SetMessages(ChatCompletionOptions options, params ChatMessage[] messages) + { + var messagesProperty = typeof(ChatCompletionOptions).GetProperty("Messages", BindingFlags.Instance | BindingFlags.NonPublic); + messagesProperty.SetValue(options, messages.ToList()); + } + + private void ValidateDuration(TestMeterListener listener, ChatCompletion response, TimeSpan durationMin, TimeSpan durationMax) + { + var duration = listener.GetInstrument("gen_ai.client.operation.duration"); + Assert.IsNotNull(duration); + Assert.IsInstanceOf>(duration); + + var measurements = listener.GetMeasurements("gen_ai.client.operation.duration"); + Assert.IsNotNull(measurements); + Assert.AreEqual(1, measurements.Count); + + var measurement = measurements[0]; + Assert.IsInstanceOf(measurement.value); + Assert.GreaterOrEqual((double)measurement.value, durationMin.TotalSeconds); + Assert.LessOrEqual((double)measurement.value, durationMax.TotalSeconds); + + ValidateChatMetricTags(measurement, response, RequestModel, Host, Port); + } + + private void ValidateUsage(TestMeterListener listener, ChatCompletion response, int inputTokens, int outputTokens) + { + var usage = listener.GetInstrument("gen_ai.client.token.usage"); + Assert.IsNotNull(usage); + Assert.IsInstanceOf>(usage); + + var measurements = listener.GetMeasurements("gen_ai.client.token.usage"); + Assert.IsNotNull(measurements); + Assert.AreEqual(2, measurements.Count); + + foreach (var measurement in measurements) + { + Assert.IsInstanceOf(measurement.value); + ValidateChatMetricTags(measurement, response, RequestModel, Host, Port); + } + + Assert.True(measurements[0].tags.TryGetValue("gen_ai.token.type", out var type)); + Assert.IsInstanceOf(type); + + TestMeasurement input = (type is "input") ? measurements[0] : measurements[1]; + TestMeasurement output = (type is "input") ? measurements[1] : measurements[0]; + + Assert.AreEqual(inputTokens, input.value); + Assert.AreEqual(outputTokens, output.value); + } + + private static ChatCompletion CreateChatCompletion(int promptTokens = PromptTokens, int completionTokens = CompletionTokens) + { + var completion = BinaryData.FromString( + $$""" + { + "id": "{{CompletionId}}", + "created": 1719621282, + "choices": [ + { + "message": { + "role": "assistant", + "content": "{{CompletionContent}}" + }, + "logprobs": null, + "index": 0, + "finish_reason": "{{FinishReason}}" + } + ], + "model": "{{ResponseModel}}", + "system_fingerprint": "fp_7ec89fabc6", + "usage": { + "completion_tokens": {{completionTokens}}, + "prompt_tokens": {{promptTokens}}, + "total_tokens": 42 + } + } + """); + + return ModelReaderWriter.Read(completion); + } +} diff --git a/tests/Telemetry/TestActivityListener.cs b/tests/Telemetry/TestActivityListener.cs new file mode 100644 index 00000000..a236691b --- /dev/null +++ b/tests/Telemetry/TestActivityListener.cs @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using NUnit.Framework; +using OpenAI.Chat; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; + +namespace OpenAI.Tests.Telemetry; + +internal class TestActivityListener : IDisposable +{ + private readonly ActivityListener _listener; + private readonly ConcurrentQueue stoppedActivities = new ConcurrentQueue(); + + public TestActivityListener(string sourceName) + { + _listener = new ActivityListener() + { + ActivityStopped = stoppedActivities.Enqueue, + ShouldListenTo = s => s.Name == sourceName, + Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded, + }; + + ActivitySource.AddActivityListener(_listener); + } + + public List Activities => stoppedActivities.ToList(); + + public void Dispose() + { + _listener.Dispose(); + } + + public static void ValidateChatActivity(Activity activity, ChatCompletion response, string requestModel = "gpt-3.5-turbo", string host = "api.openai.com", int port = 443) + { + Assert.NotNull(activity); + Assert.AreEqual($"chat {requestModel}", activity.DisplayName); + Assert.AreEqual("chat", activity.GetTagItem("gen_ai.operation.name")); + Assert.AreEqual("openai", activity.GetTagItem("gen_ai.system")); + Assert.AreEqual(requestModel, activity.GetTagItem("gen_ai.request.model")); + + Assert.AreEqual(host, activity.GetTagItem("server.address")); + Assert.AreEqual(port, activity.GetTagItem("server.port")); + + if (response != null) + { + Assert.AreEqual(response.Model, activity.GetTagItem("gen_ai.response.model")); + Assert.AreEqual(response.Id, activity.GetTagItem("gen_ai.response.id")); + Assert.AreEqual(new[] { response.FinishReason.ToString().ToLower() }, activity.GetTagItem("gen_ai.response.finish_reasons")); + Assert.AreEqual(response.Usage.OutputTokens, activity.GetTagItem("gen_ai.usage.output_tokens")); + Assert.AreEqual(response.Usage.InputTokens, activity.GetTagItem("gen_ai.usage.input_tokens")); + Assert.AreEqual(ActivityStatusCode.Unset, activity.Status); + Assert.Null(activity.StatusDescription); + Assert.Null(activity.GetTagItem("error.type")); + } + else + { + Assert.AreEqual(ActivityStatusCode.Error, activity.Status); + Assert.NotNull(activity.GetTagItem("error.type")); + } + } + + public static void ValidateChatActivity(Activity activity, Exception ex, string requestModel = "gpt-3.5-turbo", string host = "api.openai.com", int port = 443) + { + ValidateChatActivity(activity, (ChatCompletion)null, requestModel, host, port); + Assert.AreEqual(ex.GetType().FullName, activity.GetTagItem("error.type")); + } +} diff --git a/tests/Telemetry/TestAppContextSwitchHelper.cs b/tests/Telemetry/TestAppContextSwitchHelper.cs new file mode 100644 index 00000000..5faf5eca --- /dev/null +++ b/tests/Telemetry/TestAppContextSwitchHelper.cs @@ -0,0 +1,25 @@ +using System; + +namespace OpenAI.Tests.Telemetry; + +internal class TestAppContextSwitchHelper : IDisposable +{ + private const string OpenTelemetrySwitchName = "OpenAI.Experimental.EnableOpenTelemetry"; + + private string _switchName; + private TestAppContextSwitchHelper(string switchName) + { + _switchName = switchName; + AppContext.SetSwitch(_switchName, true); + } + + public static IDisposable EnableOpenTelemetry() + { + return new TestAppContextSwitchHelper(OpenTelemetrySwitchName); + } + + public void Dispose() + { + AppContext.SetSwitch(_switchName, false); + } +} diff --git a/tests/Telemetry/TestMeterListener.cs b/tests/Telemetry/TestMeterListener.cs new file mode 100644 index 00000000..187f9401 --- /dev/null +++ b/tests/Telemetry/TestMeterListener.cs @@ -0,0 +1,84 @@ +using NUnit.Framework; +using OpenAI.Chat; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics.Metrics; + +namespace OpenAI.Tests.Telemetry; + +internal class TestMeterListener : IDisposable +{ + public record TestMeasurement(object value, Dictionary tags); + + private readonly ConcurrentDictionary> _measurements = new (); + private readonly ConcurrentDictionary _instruments = new (); + private readonly MeterListener _listener; + public TestMeterListener(string meterName) + { + _listener = new MeterListener(); + _listener.InstrumentPublished = (i, l) => + { + if (i.Meter.Name == meterName) + { + l.EnableMeasurementEvents(i); + } + }; + _listener.SetMeasurementEventCallback(OnMeasurementRecorded); + _listener.SetMeasurementEventCallback(OnMeasurementRecorded); + _listener.Start(); + } + + public List GetMeasurements(string instrumentName) + { + _measurements.TryGetValue(instrumentName, out var list); + return list; + } + + public Instrument GetInstrument(string instrumentName) + { + _instruments.TryGetValue(instrumentName, out var instrument); + return instrument; + } + + private void OnMeasurementRecorded(Instrument instrument, T measurement, ReadOnlySpan> tags, object state) + { + _instruments.TryAdd(instrument.Name, instrument); + + var testMeasurement = new TestMeasurement(measurement, new Dictionary(tags.ToArray())); + _measurements.AddOrUpdate(instrument.Name, + k => new() { testMeasurement }, + (k, l) => + { + l.Add(testMeasurement); + return l; + }); + } + + public void Dispose() + { + _listener.Dispose(); + } + + public static void ValidateChatMetricTags(TestMeasurement measurement, ChatCompletion response, string requestModel = "gpt-3.5-turbo", string host = "api.openai.com", int port = 443) + { + Assert.AreEqual("openai", measurement.tags["gen_ai.system"]); + Assert.AreEqual("chat", measurement.tags["gen_ai.operation.name"]); + Assert.AreEqual(host, measurement.tags["server.address"]); + Assert.AreEqual(requestModel, measurement.tags["gen_ai.request.model"]); + Assert.AreEqual(port, measurement.tags["server.port"]); + + if (response != null) + { + Assert.AreEqual(response.Model, measurement.tags["gen_ai.response.model"]); + Assert.False(measurement.tags.ContainsKey("error.type")); + } + } + + public static void ValidateChatMetricTags(TestMeasurement measurement, Exception ex, string requestModel = "gpt-3.5-turbo", string host = "api.openai.com", int port = 443) + { + ValidateChatMetricTags(measurement, (ChatCompletion)null, requestModel, host, port); + Assert.True(measurement.tags.ContainsKey("error.type")); + Assert.AreEqual(ex.GetType().FullName, measurement.tags["error.type"]); + } +}