summary: AWS Bedrock instrumentation (#2314)

feat: Add auto-instrumentation for AWS Bedrock

feat: New configuration options are available specific to [AI monitoring](https://docs.newrelic.com/docs/apm/agents/net-agent/configuration/net-agent-configuration/#ai_monitoring).

feat: A new AI monitoring related public API method has been added - [SetLlmTokenCountingCallback](https://docs.newrelic.com/docs/apm/agents/net-agent/net-agent-api/set-llm-token-counting-callback/)

notice: New Relic AI monitoring is the industry’s first APM solution that provides end-to-end visibility for AI Large Language Model (LLM) applications. It enables end-to-end visibility into the key components of an AI LLM application. With AI monitoring, users can monitor, alert, and debug AI-powered applications for reliability, latency, performance, security and cost. AI monitoring also enables AI/LLM specific insights (metrics, events, logs and traces) which can easily integrate to build advanced guardrails for enterprise security, privacy and compliance.

notice: AI monitoring offers custom-built insights and tracing for the complete lifecycle of an LLM’s prompts and responses, from raw user input to repaired/polished responses. AI monitoring provides built-in integrations with popular LLMs and components of the AI development stack. This release provides instrumentation for AWS Bedrock.

notice: When AI monitoring is enabled, the agent will now capture AI LLM related data. This data will be visible under a new APM tab called AI Responses. See our [AI Monitoring documentation](https://docs.newrelic.com/docs/ai-monitoring/intro-to-ai-monitoring/) for more details.

.github/workflows/all_solutions.yml

@@ -247,7 +247,8 @@ jobs:
           DistributedTracing, 
           Errors, 
           HttpClientInstrumentation, 
-          InfiniteTracing, 
+          InfiniteTracing,
+          LLM,
           Logging.ContextData, 
           Logging.HsmAndCsp, 
           Logging.LocalDecoration, 

.github/workflows/run_integration_tests.yml

@@ -53,7 +53,7 @@ jobs:
           if [ "${{ inputs.integration-test-namespaces }}" == "ALL" ] ; then
             # Use the full list of namespaces
             namespaces="[ 'AgentFeatures', 'AgentLogs', 'AgentMetrics', 'Api', 'AppDomainCaching', 'AspNetCore', 'BasicInstrumentation', 'CatInbound', 'CatOutbound', 'CodeLevelMetrics', 'Configuration', \
-          'CSP', 'CustomAttributes', 'CustomInstrumentation', 'DataTransmission', 'DistributedTracing', 'Errors', 'HttpClientInstrumentation', 'InfiniteTracing', 'Logging.ContextData', \
+          'CSP', 'CustomAttributes', 'CustomInstrumentation', 'DataTransmission', 'DistributedTracing', 'Errors', 'HttpClientInstrumentation', 'InfiniteTracing', 'LLM', 'Logging.ContextData', \
           'Logging.HsmAndCsp', 'Logging.LocalDecoration', 'Logging.LogLevelDetection', 'Logging.MaxSamplesStored', 'Logging.MetricsAndForwarding', 'Logging.ZeroMaxSamplesStored', \
           'Owin', 'MassTransit', 'ReJit.NetCore', 'ReJit.NetFramework', 'RequestHandling', 'RequestHeadersCapture.AspNet', 'RequestHeadersCapture.AspNetCore', 'RequestHeadersCapture.EnvironmentVariables', \
           'RequestHeadersCapture.Owin', 'RequestHeadersCapture.WCF', 'RestSharp', 'WCF.Client.IIS.ASPDisabled', 'WCF.Client.IIS.ASPEnabled', 'WCF.Client.Self', \

FullAgent.sln

@@ -178,6 +178,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Home", "src\Agent\NewRelic\
 		{87EF7419-8390-49F2-8C51-4DF9B886E834} = {87EF7419-8390-49F2-8C51-4DF9B886E834}
 		{8D2B52DD-D45C-481D-92F0-28990F168820} = {8D2B52DD-D45C-481D-92F0-28990F168820}
 		{9784EAEA-C32F-4DC4-BD84-4075BBA541AB} = {9784EAEA-C32F-4DC4-BD84-4075BBA541AB}
+		{9C20BC4E-7A9F-4518-B3E7-C1FFD6C0EC8A} = {9C20BC4E-7A9F-4518-B3E7-C1FFD6C0EC8A}
 		{9E9E1367-E09E-45F9-9A01-7D4489FBCB23} = {9E9E1367-E09E-45F9-9A01-7D4489FBCB23}
 		{A4B357E3-9CEF-492C-A0E2-2397D15F1CD6} = {A4B357E3-9CEF-492C-A0E2-2397D15F1CD6}
 		{AA683341-1FF5-4D45-A831-1BAF3C100A5C} = {AA683341-1FF5-4D45-A831-1BAF3C100A5C}
@@ -218,6 +219,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Kafka", "src\Agent\NewRelic
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AspNetCore6Plus", "src\Agent\NewRelic\Agent\Extensions\Providers\Wrapper\AspNetCore6Plus\AspNetCore6Plus.csproj", "{D4F48A7F-F3D3-4303-921D-BF7FE34B7118}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bedrock", "src\Agent\NewRelic\Agent\Extensions\Providers\Wrapper\Bedrock\Bedrock.csproj", "{9C20BC4E-7A9F-4518-B3E7-C1FFD6C0EC8A}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -456,6 +459,10 @@ Global
 		{D4F48A7F-F3D3-4303-921D-BF7FE34B7118}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{D4F48A7F-F3D3-4303-921D-BF7FE34B7118}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{D4F48A7F-F3D3-4303-921D-BF7FE34B7118}.Release|Any CPU.Build.0 = Release|Any CPU
+		{9C20BC4E-7A9F-4518-B3E7-C1FFD6C0EC8A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{9C20BC4E-7A9F-4518-B3E7-C1FFD6C0EC8A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{9C20BC4E-7A9F-4518-B3E7-C1FFD6C0EC8A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{9C20BC4E-7A9F-4518-B3E7-C1FFD6C0EC8A}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -526,6 +533,7 @@ Global
 		{F921A316-A8D2-4992-B894-69A3B5CA34E3} = {5E86E10A-C38F-48CB-ADE9-67B22BB2F50A}
 		{270A9CC8-8031-49F4-A380-1389E7517DB7} = {5E86E10A-C38F-48CB-ADE9-67B22BB2F50A}
 		{D4F48A7F-F3D3-4303-921D-BF7FE34B7118} = {5E86E10A-C38F-48CB-ADE9-67B22BB2F50A}
+		{9C20BC4E-7A9F-4518-B3E7-C1FFD6C0EC8A} = {5E86E10A-C38F-48CB-ADE9-67B22BB2F50A}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		EnterpriseLibraryConfigurationToolBinariesPath = packages\Unity.2.1.505.2\lib\NET35

build/ArtifactBuilder/CoreAgentComponents.cs

@@ -59,6 +59,7 @@ protected override void CreateAgentComponents()
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.MassTransitLegacy.dll",
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Kafka.dll",
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.AspNetCore6Plus.dll",
+                $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Bedrock.dll",
             };
 
             var wrapperXmls = new[]
@@ -82,6 +83,7 @@ protected override void CreateAgentComponents()
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.MassTransitLegacy.Instrumentation.xml",
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Kafka.Instrumentation.xml",
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.AspNetCore6Plus.Instrumentation.xml",
+                $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Bedrock.Instrumentation.xml",
             };
 
             ExtensionXsd = $@"{SourceHomeBuilderPath}\extensions\extension.xsd";

build/ArtifactBuilder/FrameworkAgentComponents.cs

@@ -67,6 +67,7 @@ protected override void CreateAgentComponents()
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.MassTransit.dll",
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.MassTransitLegacy.dll",
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Kafka.dll",
+                $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Bedrock.dll",
             };
 
             var wrapperXmls = new[]
@@ -105,6 +106,7 @@ protected override void CreateAgentComponents()
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.MassTransit.Instrumentation.xml",
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.MassTransitLegacy.Instrumentation.xml",
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Kafka.Instrumentation.xml",
+                $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Bedrock.Instrumentation.xml",
             };
 
             ExtensionXsd = $@"{SourceHomeBuilderPath}\extensions\extension.xsd";

src/Agent/MsiInstaller/Installer/Product.wxs

@@ -474,6 +474,9 @@ SPDX-License-Identifier: Apache-2.0
       <Component Id="KafkaWrapperComponent" Guid="{BE97F5F5-A392-48A0-8888-B2973EA09490}">
         <File Id="KafkaWrapperFile" Name="NewRelic.Providers.Wrapper.Kafka.dll" KeyPath="yes" Source="$(var.HomeFolderPath)\extensions\NewRelic.Providers.Wrapper.Kafka.dll"/>
       </Component>
+      <Component Id="BedrockWrapperComponent" Guid="{C913E912-97D1-4042-8E11-A35E04C6A6E5}">
+        <File Id="BedrockWrapperFile" Name="NewRelic.Providers.Wrapper.Bedrock.dll" KeyPath="yes" Source="$(var.HomeFolderPath)\extensions\NewRelic.Providers.Wrapper.Bedrock.dll"/>
+      </Component>
 
       <!-- Reference libraries -->
       <Component Id="NewRelicCoreReferenceComponent" Guid="{C196ED1E-0FBA-4D36-9C34-E969B0C9E8C9}">
@@ -545,6 +548,9 @@ SPDX-License-Identifier: Apache-2.0
       <Component Id="CoreAspNetCore6PlusWrapperComponent" Guid="{3B7C5E60-BA9D-4B1C-8B16-B16025B075C0}">
         <File Id="CoreAspNetCore6PlusWrapperFile" Name="NewRelic.Providers.Wrapper.AspNetCore6Plus.dll" KeyPath="yes" Source="$(var.HomeFolderPath)_coreclr\extensions\NewRelic.Providers.Wrapper.AspNetCore6Plus.dll"/>
       </Component>
+      <Component Id="CoreBedrockWrapperComponent" Guid="{7DA522F5-38D4-4E7E-AA2B-857D01812152}">
+        <File Id="CoreBedrockWrapperFile" Name="NewRelic.Providers.Wrapper.Bedrock.dll" KeyPath="yes" Source="$(var.HomeFolderPath)_coreclr\extensions\NewRelic.Providers.Wrapper.Bedrock.dll"/>
+      </Component>
 
       <!-- Reference libraries -->
       <Component Id="CoreNewRelicCoreReferenceComponent" Guid="{DD2BE979-7D4B-47EA-9FBE-F6B381D70E0B}">
@@ -659,6 +665,9 @@ SPDX-License-Identifier: Apache-2.0
       <Component Id="KafkaInstrumentationComponent" Guid="{9FC86A0E-CACD-4DA8-84F8-20997C904913}">
         <File Id="KafkaInstrumentationFile" Name="NewRelic.Providers.Wrapper.Kafka.Instrumentation.xml" KeyPath="yes" Source="$(var.HomeFolderPath)\extensions\NewRelic.Providers.Wrapper.Kafka.Instrumentation.xml"/>
       </Component>
+      <Component Id="BedrockInstrumentationComponent" Guid="{D6F0A2D6-D8D5-4C61-8056-1C9AA4D07132}">
+        <File Id="BedrockInstrumentationFile" Name="NewRelic.Providers.Wrapper.Bedrock.Instrumentation.xml" KeyPath="yes" Source="$(var.HomeFolderPath)\extensions\NewRelic.Providers.Wrapper.Bedrock.Instrumentation.xml"/>
+      </Component>
     </ComponentGroup>
 
     <ComponentGroup Id="CoreNewRelic.Agent.Extensions.Instrumentation" Directory="CoreExtensionsFolder">
@@ -719,6 +728,9 @@ SPDX-License-Identifier: Apache-2.0
       <Component Id="CoreAspNetCore6PlusInstrumentationComponent" Guid="{1CC1E672-5AA5-4F6F-A736-748EA556653A}">
         <File Id="CoreAspNetCore6PlusInstrumentationFile" Name="NewRelic.Providers.Wrapper.AspNetCore6Plus.Instrumentation.xml" KeyPath="yes" Source="$(var.HomeFolderPath)_coreclr\extensions\NewRelic.Providers.Wrapper.AspNetCore6Plus.Instrumentation.xml"/>
       </Component>
+      <Component Id="CoreBedrockInstrumentationComponent" Guid="{CF3A17F1-3C5C-4B72-8E3B-10E53EB94913}">
+        <File Id="CoreBedrockInstrumentationFile" Name="NewRelic.Providers.Wrapper.Bedrock.Instrumentation.xml" KeyPath="yes" Source="$(var.HomeFolderPath)_coreclr\extensions\NewRelic.Providers.Wrapper.Bedrock.Instrumentation.xml"/>
+      </Component>
     </ComponentGroup>
 
     <!-- Extensions XSD-->

src/Agent/NewRelic.Api.Agent/NewRelic.cs

@@ -849,6 +849,45 @@ public static void SetErrorGroupCallback(Func<IReadOnlyDictionary<string, object
             }
         }
 
+        /// <summary> Sets the method that will be invoked to define the token count of completion.
+        ///
+        /// The callback takes the model name and input value, and returns an integer of the token count.
+        /// A value returned from the callback that is less than or equal to 0 will be ignored.
+        /// </summary>
+        /// <param name="callback">The callback to invoke to generate the token count based on the model and input..</param>
+        [MethodImpl(MethodImplOptions.NoOptimization | MethodImplOptions.NoInlining)]
+        public static void SetLlmTokenCountingCallback(Func<string, string, int> callback)
+        {
+            try
+            {
+                System.Diagnostics.Trace.WriteLine("NewRelic.SetLlmTokenCountingCallback()");
+            }
+            catch
+            {
+                // Swallow any exception thrown from here
+            }
+        }
+
+        /// <summary>
+        /// Creates an event with the customer feedback on the LLM interaction.
+        /// </summary>
+        /// <param name="traceId">Required. ID of the trace where the chat completion(s) related to the feedback occurred</param>
+        /// <param name="rating">Required. Rating provided by an end user. Must be string or int</param>
+        /// <param name="category">Optional. Category of the feedback as provided by the end user</param>
+        /// <param name="message">Optional. Freeform text feedback from an end user</param>
+        /// <param name="metadata">Optional. Set of key-value pairs to store any other desired data to submit with the feedback event</param>
+        public static void RecordLlmFeedbackEvent(string traceId, object rating, string category = "", string message = "", IDictionary<string, object>? metadata = null)
+        {
+            try
+            {
+                System.Diagnostics.Trace.WriteLine("NewRelic.RecordLlmFeedbackEvent()");
+            }
+            catch
+            {
+                // Swallow any exception thrown from here
+            }
+        }
+
         #endregion
     }
 

src/Agent/NewRelic/Agent/Core/Agent.cs

@@ -13,6 +13,7 @@
 using NewRelic.Agent.Core.Metrics;
 using NewRelic.Agent.Core.Segments;
 using NewRelic.Agent.Core.Transactions;
+using NewRelic.Agent.Core.Transformers;
 using NewRelic.Agent.Core.Transformers.TransactionTransformer;
 using NewRelic.Agent.Core.Utilities;
 using NewRelic.Agent.Core.WireModels;
@@ -65,14 +66,17 @@ public class Agent : IAgent // any changes to api, update the interface in exten
         private Extensions.Logging.ILogger _logger;
         private readonly ISimpleSchedulingService _simpleSchedulingService;
 
+        private readonly ICustomEventTransformer _customEventTransformer;
+
         public Agent(ITransactionService transactionService, ITransactionTransformer transactionTransformer,
             IThreadPoolStatic threadPoolStatic, ITransactionMetricNameMaker transactionMetricNameMaker, IPathHashMaker pathHashMaker,
             ICatHeaderHandler catHeaderHandler, IDistributedTracePayloadHandler distributedTracePayloadHandler,
             ISyntheticsHeaderHandler syntheticsHeaderHandler, ITransactionFinalizer transactionFinalizer,
             IBrowserMonitoringPrereqChecker browserMonitoringPrereqChecker, IBrowserMonitoringScriptMaker browserMonitoringScriptMaker,
             IConfigurationService configurationService, IAgentHealthReporter agentHealthReporter, IAgentTimerService agentTimerService,
             IMetricNameService metricNameService, Api.ITraceMetadataFactory traceMetadataFactory, ICATSupportabilityMetricCounters catMetricCounters,
-            ILogEventAggregator logEventAggregator, ILogContextDataFilter logContextDataFilter, ISimpleSchedulingService simpleSchedulingService)
+            ILogEventAggregator logEventAggregator, ILogContextDataFilter logContextDataFilter, ISimpleSchedulingService simpleSchedulingService,
+            ICustomEventTransformer customEventTransformer)
         {
             _transactionService = transactionService;
             _transactionTransformer = transactionTransformer;
@@ -95,6 +99,8 @@ public Agent(ITransactionService transactionService, ITransactionTransformer tra
             _logContextDataFilter = logContextDataFilter;
             _simpleSchedulingService = simpleSchedulingService;
 
+            _customEventTransformer = customEventTransformer;
+
             Instance = this;
         }
 
@@ -411,6 +417,61 @@ public Dictionary<string, string> GetLinkingMetadata()
 
         #region ExperimentalApi
 
+        public void RecordLlmEvent(string eventType, IDictionary<string, object> attributes)
+        {
+            if (!_configurationService.Configuration.AiMonitoringEnabled)
+            {
+                return;
+            }
+
+            // Record metric is streaming has been disabled
+            if (!_configurationService.Configuration.AiMonitoringStreamingEnabled)
+            {
+                RecordSupportabilityMetric("Supportability/DotNet/ML/Streaming/Disabled");
+            }
+
+            var transaction = _transactionService.GetCurrentInternalTransaction();
+            transaction.SetLlmTransaction(true);
+
+            // Any custom attributes that are prefixed with "llm." must be added to the event
+            var transactionAttributes = transaction.TransactionMetadata.UserAndRequestAttributes.GetAllAttributeValuesDic();
+            foreach (var attribute in transactionAttributes)
+            {
+                if (attribute.Key.StartsWith("llm.", StringComparison.InvariantCultureIgnoreCase))
+                {
+                    attributes.Add(attribute.Key, attribute.Value);
+                }
+            }
+
+            // Always use callback for token count if one is provided
+            if ((eventType == "LlmChatCompletionMessage" || eventType == "LlmEmbedding")
+                && _configurationService.Configuration.LlmTokenCountingCallback != null)
+            {
+                // message and embedding events have different attribute names for the content of the message
+                var content = eventType == "LlmChatCompletionMessage" ? (string)attributes["content"] : (string)attributes["input"];
+
+                // messages only have response models, embeddings have both
+                var model = attributes.TryGetValue("request.model", out var requestModel) ? (string)requestModel : (string)attributes["response.model"];
+
+                // Use a nullable so that the CustomEvent code will automatically remove the attribute if the callback returns null
+                var tokenCount = _configurationService.Configuration.LlmTokenCountingCallback?.Invoke(model, content);
+                if (tokenCount.HasValue && tokenCount.Value > 0)
+                {
+                    attributes["token_count"] = tokenCount;
+                }
+            }
+
+            // If record content is disabled, we need to remove the content and input attributes
+            // We will still want the token counts so removal occurs after the attempt to get the token count
+            if (!_configurationService.Configuration.AiMonitoringRecordContentEnabled)
+            {
+                attributes.Remove("content"); // ChatMessages
+                attributes.Remove("input"); // Embeddings
+            }
+
+            _customEventTransformer.Transform(eventType, attributes, transaction.Priority);
+        }
+
         public ISimpleSchedulingService SimpleSchedulingService
         {
             get { return _simpleSchedulingService; }

src/Agent/NewRelic/Agent/Core/Api/AgentApi.cs

@@ -642,6 +642,40 @@ void work()
             }
             TryInvoke(work, apiName, ApiMethod.SetErrorGroupCallback);
         }
+
+        /// <summary> Sets the method that will be invoked to define the token count of completion.
+        ///
+        /// The callback takes the model name and input value, and returns an integer of the token count.
+        /// A value returned from the callback that is less than or equal to 0 will be ignored.
+        /// </summary>
+        /// <param name="callback">The callback to invoke to generate the token count based on the model and input..</param>
+        public static void SetLlmTokenCountingCallback(Func<string, string, int> callback)
+        {
+            const string apiName = nameof(SetLlmTokenCountingCallback);
+            void work()
+            {
+                InternalApi.SetLlmTokenCountingCallback(callback);
+            }
+            TryInvoke(work, apiName, ApiMethod.SetLlmTokenCountingCallback);
+        }
+
+        /// <summary>
+        /// Creates an event with the customer feedback on the LLM interaction.
+        /// </summary>
+        /// <param name="traceId">Required. ID of the trace where the chat completion(s) related to the feedback occurred</param>
+        /// <param name="rating">Required. Rating provided by an end user. Must be string or int</param>
+        /// <param name="category">Optional. Category of the feedback as provided by the end user</param>
+        /// <param name="message">Optional. Freeform text feedback from an end user</param>
+        /// <param name="metadata">Optional. Set of key-value pairs to store any other desired data to submit with the feedback event</param>
+        public static void RecordLlmFeedbackEvent(string traceId, object rating, string category = "", string message = "", IDictionary<string, object>? metadata = null)
+        {
+            const string apiName = nameof(RecordLlmFeedbackEvent);
+            void work()
+            {
+                InternalApi.RecordLlmFeedbackEvent(traceId, rating, category, message, metadata);
+            }
+            TryInvoke(work, apiName, ApiMethod.RecordLlmFeedbackEvent);
+        }
     }
 }