From ef2ad4dc48253d66510bf17c4cadb668597bb809 Mon Sep 17 00:00:00 2001 From: "tingchuan.li" Date: Thu, 19 Jun 2025 20:33:29 +0800 Subject: [PATCH] feat(observability): Refactor content observation mechanism to support both logging and tracing This commit refactors the content observation mechanism by adding support for trace recording without altering the original logging functionality. The motivation for this change is to ensure complete context availability in scenarios such as integration with Langfuse, which is essential for proper functionality. Main changes: 1. New configuration options: 1. `trace-prompt`: Enables recording prompts to trace 2. `trace-completion` : Enables recording completions to trace 3. `trace-prompt-size` : Limits the length of prompt context 4. `content-formatter` : Handles content formatting, currently supporting both 'text' and 'langfuse' modes 2. Added two handlers, ChatModelCompletionObservationTraceHandler and ChatModelPromptContentObservationTraceHandler, to support recording context to trace. 3. Introduced the MessageFormatter interface and its subclasses to support formatting for prompt and completion content. 4. Rolled back parts of the code and dependencies from commit ca843e85. Signed-off-by: tingchuan.li --- .../ChatObservationAutoConfiguration.java | 60 +++- .../ChatObservationProperties.java | 53 +++ ...ChatObservationAutoConfigurationTests.java | 334 ++++++++++++++++-- spring-ai-commons/pom.xml | 2 +- .../conventions/AiObservationAttributes.java | 13 +- .../conventions/AiObservationEventNames.java | 59 ++++ .../ai/observation/tracing/TracingHelper.java | 66 ++++ .../tracing/TracingHelperTests.java | 143 ++++++++ spring-ai-model/pom.xml | 2 +- ...odelCompletionObservationTraceHandler.java | 88 +++++ ...lPromptContentObservationTraceHandler.java | 98 +++++ .../AiObservationContentFormatterName.java | 29 ++ .../trace/LangfuseMessageFormatter.java | 72 ++++ .../observation/trace/MessageFormatter.java | 31 ++ .../trace/TextMessageFormatter.java | 34 ++ ...CompletionObservationTraceHandlerTest.java | 90 +++++ ...mptContentObservationTraceHandlerTest.java | 89 +++++ .../trace/LangfuseMessageFormatterTest.java | 83 +++++ .../trace/TextMessageFormatterTest.java | 40 +++ 19 files changed, 1354 insertions(+), 32 deletions(-) create mode 100644 spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationEventNames.java create mode 100644 spring-ai-commons/src/main/java/org/springframework/ai/observation/tracing/TracingHelper.java create mode 100644 spring-ai-commons/src/test/java/org/springframework/ai/observation/tracing/TracingHelperTests.java create mode 100644 spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelCompletionObservationTraceHandler.java create mode 100644 spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelPromptContentObservationTraceHandler.java create mode 100644 spring-ai-model/src/main/java/org/springframework/ai/chat/observation/trace/AiObservationContentFormatterName.java create mode 100644 spring-ai-model/src/main/java/org/springframework/ai/chat/observation/trace/LangfuseMessageFormatter.java create mode 100644 spring-ai-model/src/main/java/org/springframework/ai/chat/observation/trace/MessageFormatter.java create mode 100644 spring-ai-model/src/main/java/org/springframework/ai/chat/observation/trace/TextMessageFormatter.java create mode 100644 spring-ai-model/src/test/java/org/springframework/ai/chat/observation/ChatModelCompletionObservationTraceHandlerTest.java create mode 100644 spring-ai-model/src/test/java/org/springframework/ai/chat/observation/ChatModelPromptContentObservationTraceHandlerTest.java create mode 100644 spring-ai-model/src/test/java/org/springframework/ai/chat/observation/trace/LangfuseMessageFormatterTest.java create mode 100644 spring-ai-model/src/test/java/org/springframework/ai/chat/observation/trace/TextMessageFormatterTest.java diff --git a/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/main/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationAutoConfiguration.java b/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/main/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationAutoConfiguration.java index 02193e15d49..6141862fa20 100644 --- a/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/main/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationAutoConfiguration.java +++ b/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/main/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationAutoConfiguration.java @@ -26,10 +26,7 @@ import org.springframework.ai.chat.client.advisor.observation.AdvisorObservationContext; import org.springframework.ai.chat.client.observation.ChatClientObservationContext; import org.springframework.ai.chat.model.ChatModel; -import org.springframework.ai.chat.observation.ChatModelCompletionObservationHandler; -import org.springframework.ai.chat.observation.ChatModelMeterObservationHandler; -import org.springframework.ai.chat.observation.ChatModelObservationContext; -import org.springframework.ai.chat.observation.ChatModelPromptContentObservationHandler; +import org.springframework.ai.chat.observation.*; import org.springframework.ai.embedding.observation.EmbeddingModelObservationContext; import org.springframework.ai.image.observation.ImageModelObservationContext; import org.springframework.ai.model.observation.ErrorLoggingObservationHandler; @@ -70,6 +67,16 @@ private static void logCompletionWarning() { "You have enabled logging out the completion content with the risk of exposing sensitive or private information. Please, be careful!"); } + private static void tracePromptContentWarning() { + logger.warn( + "You have enabled tracing out the prompt content with the risk of exposing sensitive or private information. Please, be careful!"); + } + + private static void traceCompletionWarning() { + logger.warn( + "You have enabled tracing out the completion content with the risk of exposing sensitive or private information. Please, be careful!"); + } + @Bean @ConditionalOnMissingBean @ConditionalOnBean(MeterRegistry.class) @@ -104,6 +111,30 @@ TracingAwareLoggingObservationHandler chatModelComp return new TracingAwareLoggingObservationHandler<>(new ChatModelCompletionObservationHandler(), tracer); } + @Bean + @ConditionalOnMissingBean(value = ChatModelPromptContentObservationTraceHandler.class, + name = "chatModelPromptContentObservationTraceHandler") + @ConditionalOnProperty(prefix = ChatObservationProperties.CONFIG_PREFIX, name = "trace-prompt", + havingValue = "true") + TracingAwareLoggingObservationHandler chatModelPromptContentObservationTraceHandler( + ChatObservationProperties properties, Tracer tracer) { + tracePromptContentWarning(); + return new TracingAwareLoggingObservationHandler<>(new ChatModelPromptContentObservationTraceHandler( + properties.getContentFormatter(), properties.getTracePromptSize()), tracer); + } + + @Bean + @ConditionalOnMissingBean(value = ChatModelCompletionObservationTraceHandler.class, + name = "chatModelCompletionObservationTraceHandler") + @ConditionalOnProperty(prefix = ChatObservationProperties.CONFIG_PREFIX, name = "trace-completion", + havingValue = "true") + TracingAwareLoggingObservationHandler chatModelCompletionObservationTraceHandler( + ChatObservationProperties properties, Tracer tracer) { + traceCompletionWarning(); + return new TracingAwareLoggingObservationHandler<>( + new ChatModelCompletionObservationTraceHandler(properties.getContentFormatter()), tracer); + } + @Bean @ConditionalOnMissingBean @ConditionalOnProperty(prefix = ChatObservationProperties.CONFIG_PREFIX, name = "include-error-logging", @@ -139,6 +170,27 @@ ChatModelCompletionObservationHandler chatModelCompletionObservationHandler() { return new ChatModelCompletionObservationHandler(); } + @Bean + @ConditionalOnMissingBean() + @ConditionalOnProperty(prefix = ChatObservationProperties.CONFIG_PREFIX, name = "trace-prompt", + havingValue = "true") + ChatModelPromptContentObservationTraceHandler chatModelPromptContentObservationTraceHandler( + ChatObservationProperties properties) { + tracePromptContentWarning(); + return new ChatModelPromptContentObservationTraceHandler(properties.getContentFormatter(), + properties.getTracePromptSize()); + } + + @Bean + @ConditionalOnMissingBean() + @ConditionalOnProperty(prefix = ChatObservationProperties.CONFIG_PREFIX, name = "trace-completion", + havingValue = "true") + ChatModelCompletionObservationTraceHandler chatModelCompletionObservationTraceHandler( + ChatObservationProperties properties) { + traceCompletionWarning(); + return new ChatModelCompletionObservationTraceHandler(properties.getContentFormatter()); + } + } } diff --git a/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/main/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationProperties.java b/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/main/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationProperties.java index 5096eb3ce87..087489a71a0 100644 --- a/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/main/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationProperties.java +++ b/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/main/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationProperties.java @@ -16,6 +16,7 @@ package org.springframework.ai.model.chat.observation.autoconfigure; +import org.springframework.ai.chat.observation.trace.AiObservationContentFormatterName; import org.springframework.boot.context.properties.ConfigurationProperties; /** @@ -40,6 +41,26 @@ public class ChatObservationProperties { */ private boolean logPrompt = false; + /** + * Whether to trace the completion content in the observations. + */ + private boolean traceCompletion = false; + + /** + * Whether to trace the prompt content in the observations. + */ + private boolean tracePrompt = false; + + /** + * prompt size in trace, smaller than 1 is unlimit + */ + private int tracePromptSize = 10; + + /** + * prompt and completion formatter + */ + private AiObservationContentFormatterName contentFormatter = AiObservationContentFormatterName.TEXT; + /** * Whether to include error logging in the observations. */ @@ -61,6 +82,38 @@ public void setLogPrompt(boolean logPrompt) { this.logPrompt = logPrompt; } + public boolean isTraceCompletion() { + return traceCompletion; + } + + public void setTraceCompletion(boolean traceCompletion) { + this.traceCompletion = traceCompletion; + } + + public boolean isTracePrompt() { + return tracePrompt; + } + + public void setTracePrompt(boolean tracePrompt) { + this.tracePrompt = tracePrompt; + } + + public int getTracePromptSize() { + return tracePromptSize; + } + + public void setTracePromptSize(int tracePromptSize) { + this.tracePromptSize = tracePromptSize; + } + + public AiObservationContentFormatterName getContentFormatter() { + return contentFormatter; + } + + public void setContentFormatter(AiObservationContentFormatterName contentFormatter) { + this.contentFormatter = contentFormatter; + } + public boolean isIncludeErrorLogging() { return this.includeErrorLogging; } diff --git a/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/test/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationAutoConfigurationTests.java b/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/test/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationAutoConfigurationTests.java index 2209029a1d4..7a685c5915e 100644 --- a/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/test/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationAutoConfigurationTests.java +++ b/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/test/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationAutoConfigurationTests.java @@ -24,10 +24,8 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.ai.chat.client.observation.ChatClientObservationContext; -import org.springframework.ai.chat.observation.ChatModelCompletionObservationHandler; -import org.springframework.ai.chat.observation.ChatModelMeterObservationHandler; -import org.springframework.ai.chat.observation.ChatModelObservationContext; -import org.springframework.ai.chat.observation.ChatModelPromptContentObservationHandler; +import org.springframework.ai.chat.observation.*; +import org.springframework.ai.chat.observation.trace.AiObservationContentFormatterName; import org.springframework.ai.model.observation.ErrorLoggingObservationHandler; import org.springframework.ai.observation.TracingAwareLoggingObservationHandler; import org.springframework.boot.autoconfigure.AutoConfigurations; @@ -70,7 +68,9 @@ void handlersNoTracer() { .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) .doesNotHaveBean(ChatModelCompletionObservationHandler.class) .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) - .doesNotHaveBean(ErrorLoggingObservationHandler.class)); + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); } @Test @@ -79,7 +79,9 @@ void handlersWithTracer() { .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) .doesNotHaveBean(ChatModelCompletionObservationHandler.class) .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) - .doesNotHaveBean(ErrorLoggingObservationHandler.class)); + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); } @Test @@ -89,7 +91,9 @@ void promptContentHandlerEnabledNoTracer(CapturedOutput output) { .run(context -> assertThat(context).hasSingleBean(ChatModelPromptContentObservationHandler.class) .doesNotHaveBean(ChatModelCompletionObservationHandler.class) .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) - .doesNotHaveBean(ErrorLoggingObservationHandler.class)); + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); assertThat(output).contains( "You have enabled logging out the prompt content with the risk of exposing sensitive or private information. Please, be careful!"); } @@ -101,7 +105,9 @@ void promptContentHandlerEnabledWithTracer(CapturedOutput output) { .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) .doesNotHaveBean(ChatModelCompletionObservationHandler.class) .hasSingleBean(TracingAwareLoggingObservationHandler.class) - .doesNotHaveBean(ErrorLoggingObservationHandler.class)); + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); assertThat(output).contains( "You have enabled logging out the prompt content with the risk of exposing sensitive or private information. Please, be careful!"); } @@ -113,7 +119,9 @@ void promptContentHandlerDisabledNoTracer() { .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) .doesNotHaveBean(ChatModelCompletionObservationHandler.class) .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) - .doesNotHaveBean(ErrorLoggingObservationHandler.class)); + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); } @Test @@ -123,7 +131,9 @@ void promptContentHandlerDisabledWithTracer() { .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) .doesNotHaveBean(ChatModelCompletionObservationHandler.class) .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) - .doesNotHaveBean(ErrorLoggingObservationHandler.class)); + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); } @Test @@ -133,7 +143,9 @@ void completionHandlerEnabledNoTracer(CapturedOutput output) { .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) .hasSingleBean(ChatModelCompletionObservationHandler.class) .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) - .doesNotHaveBean(ErrorLoggingObservationHandler.class)); + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); assertThat(output).contains( "You have enabled logging out the completion content with the risk of exposing sensitive or private information. Please, be careful!"); } @@ -145,7 +157,9 @@ void completionHandlerEnabledWithTracer(CapturedOutput output) { .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) .doesNotHaveBean(ChatModelCompletionObservationHandler.class) .hasSingleBean(TracingAwareLoggingObservationHandler.class) - .doesNotHaveBean(ErrorLoggingObservationHandler.class)); + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); assertThat(output).contains( "You have enabled logging out the completion content with the risk of exposing sensitive or private information. Please, be careful!"); } @@ -157,7 +171,9 @@ void completionHandlerDisabledNoTracer() { .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) .doesNotHaveBean(ChatModelCompletionObservationHandler.class) .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) - .doesNotHaveBean(ErrorLoggingObservationHandler.class)); + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); } @Test @@ -167,7 +183,113 @@ void completionHandlerDisabledWithTracer() { .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) .doesNotHaveBean(ChatModelCompletionObservationHandler.class) .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) - .doesNotHaveBean(ErrorLoggingObservationHandler.class)); + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); + } + + @Test + void promptTraceContentHandlerEnabledNoTracer(CapturedOutput output) { + this.contextRunner.withClassLoader(new FilteredClassLoader(Tracer.class)) + .withPropertyValues("spring.ai.chat.observations.trace-prompt=true") + .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationHandler.class) + .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .hasSingleBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); + assertThat(output).contains( + "You have enabled tracing out the prompt content with the risk of exposing sensitive or private information. Please, be careful!"); + } + + @Test + void promptTraceContentHandlerEnabledWithTracer(CapturedOutput output) { + this.contextRunner.withUserConfiguration(TracerConfiguration.class) + .withPropertyValues("spring.ai.chat.observations.trace-prompt=true") + .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationHandler.class) + .hasSingleBean(TracingAwareLoggingObservationHandler.class) + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); + assertThat(output).contains( + "You have enabled tracing out the prompt content with the risk of exposing sensitive or private information. Please, be careful!"); + } + + @Test + void promptTraceContentHandlerDisabledNoTracer() { + this.contextRunner.withClassLoader(new FilteredClassLoader(Tracer.class)) + .withPropertyValues("spring.ai.chat.observations.trace-prompt=false") + .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationHandler.class) + .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); + } + + @Test + void promptTraceContentHandlerDisabledWithTracer() { + this.contextRunner.withUserConfiguration(TracerConfiguration.class) + .withPropertyValues("spring.ai.chat.observations.trace-prompt=false") + .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationHandler.class) + .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); + } + + @Test + void completionTraceHandlerEnabledNoTracer(CapturedOutput output) { + this.contextRunner.withClassLoader(new FilteredClassLoader(Tracer.class)) + .withPropertyValues("spring.ai.chat.observations.trace-completion=true") + .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationHandler.class) + .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .hasSingleBean(ChatModelCompletionObservationTraceHandler.class)); + assertThat(output).contains( + "You have enabled tracing out the completion content with the risk of exposing sensitive or private information. Please, be careful!"); + } + + @Test + void completionTraceHandlerEnabledWithTracer(CapturedOutput output) { + this.contextRunner.withUserConfiguration(TracerConfiguration.class) + .withPropertyValues("spring.ai.chat.observations.trace-completion=true") + .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationHandler.class) + .hasSingleBean(TracingAwareLoggingObservationHandler.class) + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); + assertThat(output).contains( + "You have enabled tracing out the completion content with the risk of exposing sensitive or private information. Please, be careful!"); + } + + @Test + void completionTraceHandlerDisabledNoTracer() { + this.contextRunner.withClassLoader(new FilteredClassLoader(Tracer.class)) + .withPropertyValues("spring.ai.chat.observations.trace-completion=false") + .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationHandler.class) + .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); + } + + @Test + void completionTraceHandlerDisabledWithTracer() { + this.contextRunner.withUserConfiguration(TracerConfiguration.class) + .withPropertyValues("spring.ai.chat.observations.trace-completion=false") + .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationHandler.class) + .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); } @Test @@ -177,7 +299,9 @@ void errorLoggingHandlerEnabledNoTracer() { .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) .doesNotHaveBean(ChatModelCompletionObservationHandler.class) .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) - .doesNotHaveBean(ErrorLoggingObservationHandler.class)); + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); } @Test @@ -187,7 +311,9 @@ void errorLoggingHandlerEnabledWithTracer() { .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) .doesNotHaveBean(ChatModelCompletionObservationHandler.class) .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) - .hasSingleBean(ErrorLoggingObservationHandler.class)); + .hasSingleBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); } @Test @@ -197,7 +323,9 @@ void errorLoggingHandlerDisabledNoTracer() { .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) .doesNotHaveBean(ChatModelCompletionObservationHandler.class) .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) - .doesNotHaveBean(ErrorLoggingObservationHandler.class)); + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); } @Test @@ -207,7 +335,9 @@ void errorLoggingHandlerDisabledWithTracer() { .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) .doesNotHaveBean(ChatModelCompletionObservationHandler.class) .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) - .doesNotHaveBean(ErrorLoggingObservationHandler.class)); + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); } @Test @@ -219,7 +349,9 @@ void customChatModelPromptContentObservationHandlerNoTracer() { .hasBean("customChatModelPromptContentObservationHandler") .doesNotHaveBean(ChatModelCompletionObservationHandler.class) .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) - .doesNotHaveBean(ErrorLoggingObservationHandler.class)); + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); } @Test @@ -231,7 +363,9 @@ void customChatModelPromptContentObservationHandlerWithTracer() { .hasBean("customChatModelPromptContentObservationHandler") .doesNotHaveBean(ChatModelCompletionObservationHandler.class) .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) - .doesNotHaveBean(ErrorLoggingObservationHandler.class)); + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); } @Test @@ -245,7 +379,9 @@ void customTracingAwareLoggingObservationHandlerForChatModelPromptContent() { .doesNotHaveBean(ChatModelCompletionObservationHandler.class) .hasSingleBean(TracingAwareLoggingObservationHandler.class) .hasBean("chatModelPromptContentObservationHandler") - .doesNotHaveBean(ErrorLoggingObservationHandler.class); + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class); assertThat(context.getBean(TracingAwareLoggingObservationHandler.class)).isSameAs( CustomTracingAwareLoggingObservationHandlerForChatModelPromptContentConfiguration.handlerInstance); }); @@ -260,7 +396,9 @@ void customChatModelCompletionObservationHandlerNoTracer() { .hasSingleBean(ChatModelCompletionObservationHandler.class) .hasBean("customChatModelCompletionObservationHandler") .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) - .doesNotHaveBean(ErrorLoggingObservationHandler.class)); + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); } @Test @@ -272,7 +410,9 @@ void customChatModelCompletionObservationHandlerWithTracer() { .hasSingleBean(ChatModelCompletionObservationHandler.class) .hasBean("customChatModelCompletionObservationHandler") .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) - .doesNotHaveBean(ErrorLoggingObservationHandler.class)); + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); } @Test @@ -285,12 +425,108 @@ void customTracingAwareLoggingObservationHandlerForChatModelCompletion() { .doesNotHaveBean(ChatModelCompletionObservationHandler.class) .hasSingleBean(TracingAwareLoggingObservationHandler.class) .hasBean("chatModelCompletionObservationHandler") - .doesNotHaveBean(ErrorLoggingObservationHandler.class); + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class); assertThat(context.getBean(TracingAwareLoggingObservationHandler.class)).isSameAs( CustomTracingAwareLoggingObservationHandlerForChatModelCompletionConfiguration.handlerInstance); }); } + @Test + void customChatModelPromptTraceContentObservationHandlerNoTracer() { + this.contextRunner.withClassLoader(new FilteredClassLoader(Tracer.class)) + .withUserConfiguration(ChatModelPromptContentObservationTraceHandlerConfiguration.class) + .withPropertyValues("spring.ai.chat.observations.trace-prompt=true") + .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) + .hasBean("customChatModelPromptContentObservationTraceHandler") + .doesNotHaveBean(ChatModelCompletionObservationHandler.class) + .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .hasSingleBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); + } + + @Test + void customChatModelPromptTraceContentObservationHandlerWithTracer() { + this.contextRunner.withUserConfiguration(TracerConfiguration.class) + .withUserConfiguration(ChatModelPromptContentObservationTraceHandlerConfiguration.class) + .withPropertyValues("spring.ai.chat.observations.trace-prompt=true") + .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) + .hasBean("customChatModelPromptContentObservationTraceHandler") + .doesNotHaveBean(ChatModelCompletionObservationHandler.class) + .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .hasSingleBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); + } + + @Test + void customTracingAwareLoggingObservationHandlerForChatModelPromptContentTrace() { + this.contextRunner.withUserConfiguration(TracerConfiguration.class) + .withUserConfiguration( + CustomTracingAwareLoggingObservationHandlerForChatModelPromptContentTraceConfiguration.class) + .withPropertyValues("spring.ai.chat.observations.trace-prompt=true") + .run(context -> { + assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationHandler.class) + .hasSingleBean(TracingAwareLoggingObservationHandler.class) + .hasBean("chatModelPromptContentObservationTraceHandler") + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class); + assertThat(context.getBean(TracingAwareLoggingObservationHandler.class)).isSameAs( + CustomTracingAwareLoggingObservationHandlerForChatModelPromptContentTraceConfiguration.handlerInstance); + }); + } + + @Test + void customChatModelCompletionTraceObservationHandlerNoTracer() { + this.contextRunner.withClassLoader(new FilteredClassLoader(Tracer.class)) + .withUserConfiguration(CustomChatModelCompletionObservationTraceHandlerConfiguration.class) + .withPropertyValues("spring.ai.chat.observations.trace-completion=true") + .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationHandler.class) + .hasBean("customChatModelCompletionObservationTraceHandler") + .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .hasSingleBean(ChatModelCompletionObservationTraceHandler.class)); + } + + @Test + void customChatModelCompletionTraceObservationHandlerWithTracer() { + this.contextRunner.withUserConfiguration(TracerConfiguration.class) + .withUserConfiguration(CustomChatModelCompletionObservationTraceHandlerConfiguration.class) + .withPropertyValues("spring.ai.chat.observations.trace-completion=true") + .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationHandler.class) + .hasBean("customChatModelCompletionObservationTraceHandler") + .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .hasSingleBean(ChatModelCompletionObservationTraceHandler.class)); + } + + @Test + void customTracingAwareLoggingObservationHandlerForChatModelCompletionTrace() { + this.contextRunner.withUserConfiguration(TracerConfiguration.class) + .withUserConfiguration( + CustomTracingAwareLoggingObservationHandlerForChatModelCompletionTraceConfiguration.class) + .withPropertyValues("spring.ai.chat.observations.trace-completion=true") + .run(context -> { + assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationHandler.class) + .hasSingleBean(TracingAwareLoggingObservationHandler.class) + .hasBean("chatModelCompletionObservationTraceHandler") + .doesNotHaveBean(ErrorLoggingObservationHandler.class) + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class); + assertThat(context.getBean(TracingAwareLoggingObservationHandler.class)).isSameAs( + CustomTracingAwareLoggingObservationHandlerForChatModelCompletionTraceConfiguration.handlerInstance); + }); + } + @Test void customErrorLoggingObservationHandler() { this.contextRunner.withUserConfiguration(TracerConfiguration.class) @@ -300,7 +536,9 @@ void customErrorLoggingObservationHandler() { .doesNotHaveBean(ChatModelCompletionObservationHandler.class) .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) .hasSingleBean(ErrorLoggingObservationHandler.class) - .hasBean("customErrorLoggingObservationHandler")); + .hasBean("customErrorLoggingObservationHandler") + .doesNotHaveBean(ChatModelPromptContentObservationTraceHandler.class) + .doesNotHaveBean(ChatModelCompletionObservationTraceHandler.class)); } @Configuration(proxyBeanMethods = false) @@ -359,6 +597,52 @@ TracingAwareLoggingObservationHandler chatModelComp } + @Configuration(proxyBeanMethods = false) + static class ChatModelPromptContentObservationTraceHandlerConfiguration { + + @Bean + ChatModelPromptContentObservationTraceHandler customChatModelPromptContentObservationTraceHandler() { + return new ChatModelPromptContentObservationTraceHandler(AiObservationContentFormatterName.TEXT, -1); + } + + } + + @Configuration(proxyBeanMethods = false) + static class CustomTracingAwareLoggingObservationHandlerForChatModelPromptContentTraceConfiguration { + + static TracingAwareLoggingObservationHandler handlerInstance = new TracingAwareLoggingObservationHandler<>( + new ChatModelPromptContentObservationTraceHandler(AiObservationContentFormatterName.TEXT, -1), null); + + @Bean + TracingAwareLoggingObservationHandler chatModelPromptContentObservationTraceHandler() { + return handlerInstance; + } + + } + + @Configuration(proxyBeanMethods = false) + static class CustomChatModelCompletionObservationTraceHandlerConfiguration { + + @Bean + ChatModelCompletionObservationTraceHandler customChatModelCompletionObservationTraceHandler() { + return new ChatModelCompletionObservationTraceHandler(AiObservationContentFormatterName.TEXT); + } + + } + + @Configuration(proxyBeanMethods = false) + static class CustomTracingAwareLoggingObservationHandlerForChatModelCompletionTraceConfiguration { + + static TracingAwareLoggingObservationHandler handlerInstance = new TracingAwareLoggingObservationHandler<>( + new ChatModelCompletionObservationTraceHandler(AiObservationContentFormatterName.TEXT), null); + + @Bean + TracingAwareLoggingObservationHandler chatModelCompletionObservationTraceHandler() { + return handlerInstance; + } + + } + @Configuration(proxyBeanMethods = false) static class CustomErrorLoggingObservationHandlerConfiguration { diff --git a/spring-ai-commons/pom.xml b/spring-ai-commons/pom.xml index 513877df8ed..523b452771e 100644 --- a/spring-ai-commons/pom.xml +++ b/spring-ai-commons/pom.xml @@ -61,7 +61,7 @@ io.micrometer - micrometer-tracing + micrometer-tracing-bridge-otel true diff --git a/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationAttributes.java b/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationAttributes.java index 8fbdb8c9175..be9099f4abe 100644 --- a/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationAttributes.java +++ b/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationAttributes.java @@ -126,7 +126,18 @@ public enum AiObservationAttributes { /** * The total number of tokens used in the model exchange. */ - USAGE_TOTAL_TOKENS("gen_ai.usage.total_tokens"); + USAGE_TOTAL_TOKENS("gen_ai.usage.total_tokens"), + + // GenAI Content + + /** + * The full prompt sent to the model. + */ + PROMPT("gen_ai.prompt"), + /** + * The full response received from the model. + */ + COMPLETION("gen_ai.completion"); private final String value; diff --git a/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationEventNames.java b/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationEventNames.java new file mode 100644 index 00000000000..859c7e67b58 --- /dev/null +++ b/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationEventNames.java @@ -0,0 +1,59 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.observation.conventions; + +/** + * Collection of event names used in AI observations. Based on the OpenTelemetry Semantic + * Conventions for AI Systems. + * + * @author Thomas Vitale + * @since 1.0.0 + * @see OTel + * Semantic Conventions. + */ +public enum AiObservationEventNames { + + // @formatter:off + + /** + * Prompt for content generation. + */ + CONTENT_PROMPT("gen_ai.content.prompt"), + + /** + * Completion of content generation. + */ + CONTENT_COMPLETION("gen_ai.content.completion"); + + private final String value; + + AiObservationEventNames(String value) { + this.value = value; + } + + /** + * Return the value of the event name. + * @return the value of the event name + */ + public String value() { + return this.value; + } + + // @formatter:on + +} diff --git a/spring-ai-commons/src/main/java/org/springframework/ai/observation/tracing/TracingHelper.java b/spring-ai-commons/src/main/java/org/springframework/ai/observation/tracing/TracingHelper.java new file mode 100644 index 00000000000..86270ff9c66 --- /dev/null +++ b/spring-ai-commons/src/main/java/org/springframework/ai/observation/tracing/TracingHelper.java @@ -0,0 +1,66 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.observation.tracing; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +import io.micrometer.tracing.handler.TracingObservationHandler; +import io.opentelemetry.api.trace.Span; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.springframework.lang.Nullable; + +/** + * Utilities to prepare and process traces for observability. + * + * @author Thomas Vitale + */ +public final class TracingHelper { + + private static final Logger logger = LoggerFactory.getLogger(TracingHelper.class); + + private TracingHelper() { + } + + @Nullable + public static Span extractOtelSpan(@Nullable TracingObservationHandler.TracingContext tracingContext) { + if (tracingContext == null) { + return null; + } + + io.micrometer.tracing.Span micrometerSpan = tracingContext.getSpan(); + try { + Method toOtelMethod = tracingContext.getSpan() + .getClass() + .getDeclaredMethod("toOtel", io.micrometer.tracing.Span.class); + toOtelMethod.setAccessible(true); + Object otelSpanObject = toOtelMethod.invoke(null, micrometerSpan); + if (otelSpanObject instanceof Span otelSpan) { + return otelSpan; + } + } + catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException ex) { + logger.warn("It wasn't possible to extract the OpenTelemetry Span object from Micrometer", ex); + return null; + } + + return null; + } + +} diff --git a/spring-ai-commons/src/test/java/org/springframework/ai/observation/tracing/TracingHelperTests.java b/spring-ai-commons/src/test/java/org/springframework/ai/observation/tracing/TracingHelperTests.java new file mode 100644 index 00000000000..78ed778b2a8 --- /dev/null +++ b/spring-ai-commons/src/test/java/org/springframework/ai/observation/tracing/TracingHelperTests.java @@ -0,0 +1,143 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.observation.tracing; + +import java.util.concurrent.TimeUnit; + +import io.micrometer.tracing.Span; +import io.micrometer.tracing.TraceContext; +import io.micrometer.tracing.handler.TracingObservationHandler; +import io.micrometer.tracing.otel.bridge.OtelCurrentTraceContext; +import io.micrometer.tracing.otel.bridge.OtelTracer; +import io.opentelemetry.api.OpenTelemetry; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Unit tests for {@link TracingHelper}. + * + * @author Thomas Vitale + */ +class TracingHelperTests { + + @Test + void extractOtelSpanWhenTracingContextIsNull() { + var actualOtelSpan = TracingHelper.extractOtelSpan(null); + assertThat(actualOtelSpan).isNull(); + } + + @Test + void extractOtelSpanWhenMethodDoesNotExist() { + var tracingContext = new TracingObservationHandler.TracingContext(); + tracingContext.setSpan(Span.NOOP); + var actualOtelSpan = TracingHelper.extractOtelSpan(tracingContext); + assertThat(actualOtelSpan).isNull(); + } + + @Test + void extractOtelSpanWhenSpanIsNotOpenTelemetry() { + var tracingContext = new TracingObservationHandler.TracingContext(); + tracingContext.setSpan(new DemoOtherSpan()); + var actualOtelSpan = TracingHelper.extractOtelSpan(tracingContext); + assertThat(actualOtelSpan).isNull(); + } + + @Test + void extractOtelSpanWhenSpanIsOpenTelemetry() { + var tracingContext = new TracingObservationHandler.TracingContext(); + var otelTracer = new OtelTracer(OpenTelemetry.noop().getTracer("test"), new OtelCurrentTraceContext(), null); + tracingContext.setSpan(otelTracer.nextSpan()); + var actualOtelSpan = TracingHelper.extractOtelSpan(tracingContext); + assertThat(actualOtelSpan).isNotNull(); + assertThat(actualOtelSpan).isInstanceOf(io.opentelemetry.api.trace.Span.class); + } + + static class DemoOtherSpan implements Span { + + private static Span toOtel(Span span) { + return Span.NOOP; + } + + @Override + public boolean isNoop() { + return false; + } + + @Override + public TraceContext context() { + return null; + } + + @Override + public Span start() { + return null; + } + + @Override + public Span name(String s) { + return null; + } + + @Override + public Span event(String s) { + return null; + } + + @Override + public Span event(String s, long l, TimeUnit timeUnit) { + return null; + } + + @Override + public Span tag(String s, String s1) { + return null; + } + + @Override + public Span error(Throwable throwable) { + return null; + } + + @Override + public void end() { + + } + + @Override + public void end(long l, TimeUnit timeUnit) { + + } + + @Override + public void abandon() { + + } + + @Override + public Span remoteServiceName(String s) { + return null; + } + + @Override + public Span remoteIpAndPort(String s, int i) { + return null; + } + + } + +} diff --git a/spring-ai-model/pom.xml b/spring-ai-model/pom.xml index b90f76db1e9..9831a862b5a 100644 --- a/spring-ai-model/pom.xml +++ b/spring-ai-model/pom.xml @@ -60,7 +60,7 @@ io.micrometer - micrometer-tracing + micrometer-tracing-bridge-otel true diff --git a/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelCompletionObservationTraceHandler.java b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelCompletionObservationTraceHandler.java new file mode 100644 index 00000000000..9d865e368ba --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelCompletionObservationTraceHandler.java @@ -0,0 +1,88 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.chat.observation; + +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationHandler; +import io.micrometer.tracing.handler.TracingObservationHandler; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import org.springframework.ai.chat.observation.trace.AiObservationContentFormatterName; +import org.springframework.ai.chat.observation.trace.LangfuseMessageFormatter; +import org.springframework.ai.chat.observation.trace.MessageFormatter; +import org.springframework.ai.chat.observation.trace.TextMessageFormatter; +import org.springframework.ai.observation.conventions.AiObservationAttributes; +import org.springframework.ai.observation.conventions.AiObservationEventNames; +import org.springframework.ai.observation.tracing.TracingHelper; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; + +import java.util.List; + +/** + * Handler for emitting the chat completion content to trace. + * + * @author tingchuan.li + * @since 1.0.0 + */ +public class ChatModelCompletionObservationTraceHandler implements ObservationHandler { + + private final MessageFormatter messageFormatter; + + public ChatModelCompletionObservationTraceHandler(AiObservationContentFormatterName formatterName) { + if (formatterName == AiObservationContentFormatterName.LANGFUSE) { + messageFormatter = new LangfuseMessageFormatter(); + } + else { + messageFormatter = new TextMessageFormatter(); + } + } + + @Override + public void onStop(ChatModelObservationContext context) { + if (context.getResponse() == null || context.getResponse().getResults() == null + || CollectionUtils.isEmpty(context.getResponse().getResults())) { + return; + } + + if (!StringUtils.hasText(context.getResponse().getResult().getOutput().getText())) { + return; + } + List completion = context.getResponse() + .getResults() + .stream() + .filter(generation -> generation.getOutput() != null + && StringUtils.hasText(generation.getOutput().getText())) + .map(generation -> this.messageFormatter.format(generation.getOutput())) + .toList(); + + TracingObservationHandler.TracingContext tracingContext = context + .getRequired(TracingObservationHandler.TracingContext.class); + Span currentSpan = TracingHelper.extractOtelSpan(tracingContext); + if (currentSpan != null) { + currentSpan.addEvent(AiObservationEventNames.CONTENT_COMPLETION.value(), + Attributes.of(AttributeKey.stringArrayKey(AiObservationAttributes.COMPLETION.value()), completion)); + } + } + + @Override + public boolean supportsContext(Observation.Context context) { + return context instanceof ChatModelObservationContext; + } + +} diff --git a/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelPromptContentObservationTraceHandler.java b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelPromptContentObservationTraceHandler.java new file mode 100644 index 00000000000..eae209b9888 --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelPromptContentObservationTraceHandler.java @@ -0,0 +1,98 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.chat.observation; + +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationHandler; +import io.micrometer.tracing.handler.TracingObservationHandler; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import org.springframework.ai.chat.observation.trace.AiObservationContentFormatterName; +import org.springframework.ai.chat.observation.trace.LangfuseMessageFormatter; +import org.springframework.ai.chat.observation.trace.MessageFormatter; +import org.springframework.ai.chat.observation.trace.TextMessageFormatter; +import org.springframework.ai.content.Content; +import org.springframework.ai.observation.ObservabilityHelper; +import org.springframework.ai.observation.conventions.AiObservationAttributes; +import org.springframework.ai.observation.conventions.AiObservationEventNames; +import org.springframework.ai.observation.tracing.TracingHelper; +import org.springframework.util.CollectionUtils; + +import java.util.List; + +/** + * Handler for emitting the chat prompt content to trace. + * + * @author tingchuan.li + * @since 1.0.0 + */ +public class ChatModelPromptContentObservationTraceHandler implements ObservationHandler { + + private final MessageFormatter messageFormatter; + + private final int tracePromptSize; + + public ChatModelPromptContentObservationTraceHandler(AiObservationContentFormatterName formatterName, + int tracePromptSize) { + this.tracePromptSize = tracePromptSize; + if (formatterName == AiObservationContentFormatterName.LANGFUSE) { + messageFormatter = new LangfuseMessageFormatter(); + } + else { + messageFormatter = new TextMessageFormatter(); + } + } + + @Override + public void onStop(ChatModelObservationContext context) { + if (CollectionUtils.isEmpty(context.getRequest().getInstructions())) { + return; + } + int skip = calculateSkips(context.getRequest().getInstructions().size(), this.tracePromptSize); + List prompt = context.getRequest() + .getInstructions() + .stream() + .map(this.messageFormatter::format) + .skip(skip) + .toList(); + TracingObservationHandler.TracingContext tracingContext = context + .getRequired(TracingObservationHandler.TracingContext.class); + Span currentSpan = TracingHelper.extractOtelSpan(tracingContext); + if (currentSpan != null) { + currentSpan.addEvent(AiObservationEventNames.CONTENT_PROMPT.value(), + Attributes.of(AttributeKey.stringArrayKey(AiObservationAttributes.PROMPT.value()), prompt)); + } + } + + private int calculateSkips(int dataSize, int remainsSize) { + if (remainsSize <= 0) { + return 0; + } + if (dataSize <= 0) { + return 0; + } + int skip = dataSize - remainsSize; + return Math.max(skip, 0); + } + + @Override + public boolean supportsContext(Observation.Context context) { + return context instanceof ChatModelObservationContext; + } + +} diff --git a/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/trace/AiObservationContentFormatterName.java b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/trace/AiObservationContentFormatterName.java new file mode 100644 index 00000000000..cdc49f259ee --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/trace/AiObservationContentFormatterName.java @@ -0,0 +1,29 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.chat.observation.trace; + +/** + * formatter config + * + * @author tingchuan.li + * @since 1.0.0 + */ +public enum AiObservationContentFormatterName { + + TEXT, LANGFUSE + +} diff --git a/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/trace/LangfuseMessageFormatter.java b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/trace/LangfuseMessageFormatter.java new file mode 100644 index 00000000000..bd9d1804d6a --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/trace/LangfuseMessageFormatter.java @@ -0,0 +1,72 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.chat.observation.trace; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.ToolResponseMessage; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; + +import java.util.HashMap; +import java.util.Map; + +/** + * format the message in order to have a pretty langfuse display + * + * @author tingchuan.li + * @since 1.0.0 + */ +public class LangfuseMessageFormatter implements MessageFormatter { + + private static final ObjectMapper OBJECT_MAPPER = JsonMapper.builder().build(); + + private static final String ROLE = "role"; + + private static final String CONTENT = "content"; + + @Override + public String format(Message message) { + try { + if (message instanceof AssistantMessage && !StringUtils.hasText(message.getText()) + && !CollectionUtils.isEmpty(((AssistantMessage) message).getToolCalls())) { + // tool call request + Map map = new HashMap<>(); + map.put(ROLE, message.getMessageType().getValue()); + map.put(CONTENT, ((AssistantMessage) message).getToolCalls()); + return OBJECT_MAPPER.writeValueAsString(map); + } + if (message instanceof ToolResponseMessage) { + // tool call response + Map map = new HashMap<>(); + map.put(ROLE, message.getMessageType().getValue()); + map.put(CONTENT, ((ToolResponseMessage) message).getResponses()); + return OBJECT_MAPPER.writeValueAsString(map); + } + Map map = new HashMap<>(); + map.put(ROLE, message.getMessageType().getValue()); + map.put(CONTENT, message.getText()); + return OBJECT_MAPPER.writeValueAsString(map); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + +} diff --git a/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/trace/MessageFormatter.java b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/trace/MessageFormatter.java new file mode 100644 index 00000000000..8d6c3c6bf70 --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/trace/MessageFormatter.java @@ -0,0 +1,31 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.chat.observation.trace; + +import org.springframework.ai.chat.messages.Message; + +/** + * format message to string + * + * @author tingchuan.li + * @since 1.0.0 + */ +public interface MessageFormatter { + + String format(Message message); + +} diff --git a/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/trace/TextMessageFormatter.java b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/trace/TextMessageFormatter.java new file mode 100644 index 00000000000..7402ee69f78 --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/trace/TextMessageFormatter.java @@ -0,0 +1,34 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.chat.observation.trace; + +import org.springframework.ai.chat.messages.Message; + +/** + * just get content + * + * @author tingchuan.li + * @since 1.0.0 + */ +public class TextMessageFormatter implements MessageFormatter { + + @Override + public String format(Message message) { + return message.getText(); + } + +} diff --git a/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/ChatModelCompletionObservationTraceHandlerTest.java b/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/ChatModelCompletionObservationTraceHandlerTest.java new file mode 100644 index 00000000000..5b448a551ec --- /dev/null +++ b/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/ChatModelCompletionObservationTraceHandlerTest.java @@ -0,0 +1,90 @@ +package org.springframework.ai.chat.observation; + +import io.micrometer.tracing.handler.TracingObservationHandler; +import io.micrometer.tracing.otel.bridge.OtelCurrentTraceContext; +import io.micrometer.tracing.otel.bridge.OtelTracer; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.observation.trace.AiObservationContentFormatterName; +import org.springframework.ai.chat.prompt.ChatOptions; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.observation.conventions.AiObservationAttributes; +import org.springframework.ai.observation.conventions.AiObservationEventNames; +import org.springframework.ai.observation.tracing.TracingHelper; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Unit tests for {@link ChatModelCompletionObservationTraceHandler}. + * + * @author tingchuan.li + */ +class ChatModelCompletionObservationTraceHandlerTest { + + static TracingObservationHandler.TracingContext createTracingContext() { + var sdkTracer = SdkTracerProvider.builder().build().get("test"); + var otelTracer = new OtelTracer(sdkTracer, new OtelCurrentTraceContext(), null); + var span = otelTracer.nextSpan(); + var tracingContext = new TracingObservationHandler.TracingContext(); + tracingContext.setSpan(span); + return tracingContext; + } + + static ChatModelObservationContext createChatModelObservationContext( + TracingObservationHandler.TracingContext tracingContext) { + var observationContext = ChatModelObservationContext.builder() + .prompt(new Prompt("supercalifragilisticexpialidocious", + ChatOptions.builder().model("spoonful-of-sugar").build())) + .provider("mary-poppins") + .build(); + observationContext.setResponse(new ChatResponse(List.of(new Generation(new AssistantMessage("say please")), + new Generation(new AssistantMessage("seriously, say please"))))); + observationContext.put(TracingObservationHandler.TracingContext.class, tracingContext); + return observationContext; + } + + @Test + void whenCompletionWithTextThenSpanEvent() { + var tracingContext = createTracingContext(); + var observationContext = createChatModelObservationContext(tracingContext); + new ChatModelCompletionObservationTraceHandler(AiObservationContentFormatterName.TEXT) + .onStop(observationContext); + var otelSpan = TracingHelper.extractOtelSpan(tracingContext); + assertThat(otelSpan).isNotNull(); + var spanData = ((ReadableSpan) otelSpan).toSpanData(); + assertThat(spanData.getEvents().size()).isEqualTo(1); + assertThat(spanData.getEvents().get(0).getName()).isEqualTo(AiObservationEventNames.CONTENT_COMPLETION.value()); + assertThat(spanData.getEvents() + .get(0) + .getAttributes() + .get(AttributeKey.stringArrayKey(AiObservationAttributes.COMPLETION.value()))) + .containsOnly("say please", "seriously, say please"); + } + + @Test + void whenCompletionWithLangfuseThenSpanEvent() { + var tracingContext = createTracingContext(); + var observationContext = createChatModelObservationContext(tracingContext); + new ChatModelCompletionObservationTraceHandler(AiObservationContentFormatterName.LANGFUSE) + .onStop(observationContext); + var otelSpan = TracingHelper.extractOtelSpan(tracingContext); + assertThat(otelSpan).isNotNull(); + var spanData = ((ReadableSpan) otelSpan).toSpanData(); + assertThat(spanData.getEvents().size()).isEqualTo(1); + assertThat(spanData.getEvents().get(0).getName()).isEqualTo(AiObservationEventNames.CONTENT_COMPLETION.value()); + assertThat(spanData.getEvents() + .get(0) + .getAttributes() + .get(AttributeKey.stringArrayKey(AiObservationAttributes.COMPLETION.value()))) + .containsOnly("{\"role\":\"assistant\",\"content\":\"say please\"}", + "{\"role\":\"assistant\",\"content\":\"seriously, say please\"}"); + } + +} diff --git a/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/ChatModelPromptContentObservationTraceHandlerTest.java b/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/ChatModelPromptContentObservationTraceHandlerTest.java new file mode 100644 index 00000000000..0fdce2b00d0 --- /dev/null +++ b/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/ChatModelPromptContentObservationTraceHandlerTest.java @@ -0,0 +1,89 @@ +package org.springframework.ai.chat.observation; + +import io.micrometer.tracing.handler.TracingObservationHandler; +import io.micrometer.tracing.otel.bridge.OtelCurrentTraceContext; +import io.micrometer.tracing.otel.bridge.OtelTracer; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.observation.trace.AiObservationContentFormatterName; +import org.springframework.ai.chat.prompt.ChatOptions; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.observation.conventions.AiObservationAttributes; +import org.springframework.ai.observation.conventions.AiObservationEventNames; +import org.springframework.ai.observation.tracing.TracingHelper; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Unit tests for {@link ChatModelPromptContentObservationTraceHandler}. + * + * @author tingchuan.li + */ +class ChatModelPromptContentObservationTraceHandlerTest { + + static TracingObservationHandler.TracingContext createTracingContext() { + var sdkTracer = SdkTracerProvider.builder().build().get("test"); + var otelTracer = new OtelTracer(sdkTracer, new OtelCurrentTraceContext(), null); + var span = otelTracer.nextSpan(); + var tracingContext = new TracingObservationHandler.TracingContext(); + tracingContext.setSpan(span); + return tracingContext; + } + + static ChatModelObservationContext createChatModelObservationContext( + TracingObservationHandler.TracingContext tracingContext) { + var observationContext = ChatModelObservationContext.builder() + .prompt(new Prompt("supercalifragilisticexpialidocious", + ChatOptions.builder().model("spoonful-of-sugar").build())) + .provider("mary-poppins") + .build(); + observationContext.setResponse(new ChatResponse(List.of(new Generation(new AssistantMessage("say please")), + new Generation(new AssistantMessage("seriously, say please"))))); + observationContext.put(TracingObservationHandler.TracingContext.class, tracingContext); + return observationContext; + } + + @Test + void whenPromptWithTextThenSpanEvent() { + var tracingContext = createTracingContext(); + var observationContext = createChatModelObservationContext(tracingContext); + new ChatModelPromptContentObservationTraceHandler(AiObservationContentFormatterName.TEXT, -1) + .onStop(observationContext); + var otelSpan = TracingHelper.extractOtelSpan(tracingContext); + assertThat(otelSpan).isNotNull(); + var spanData = ((ReadableSpan) otelSpan).toSpanData(); + assertThat(spanData.getEvents().size()).isEqualTo(1); + assertThat(spanData.getEvents().get(0).getName()).isEqualTo(AiObservationEventNames.CONTENT_PROMPT.value()); + assertThat(spanData.getEvents() + .get(0) + .getAttributes() + .get(AttributeKey.stringArrayKey(AiObservationAttributes.PROMPT.value()))) + .containsOnly("supercalifragilisticexpialidocious"); + } + + @Test + void whenPromptWithLangfuseThenSpanEvent() { + var tracingContext = createTracingContext(); + var observationContext = createChatModelObservationContext(tracingContext); + new ChatModelPromptContentObservationTraceHandler(AiObservationContentFormatterName.LANGFUSE, -1) + .onStop(observationContext); + var otelSpan = TracingHelper.extractOtelSpan(tracingContext); + assertThat(otelSpan).isNotNull(); + var spanData = ((ReadableSpan) otelSpan).toSpanData(); + assertThat(spanData.getEvents().size()).isEqualTo(1); + assertThat(spanData.getEvents().get(0).getName()).isEqualTo(AiObservationEventNames.CONTENT_PROMPT.value()); + assertThat(spanData.getEvents() + .get(0) + .getAttributes() + .get(AttributeKey.stringArrayKey(AiObservationAttributes.PROMPT.value()))) + .containsOnly("{\"role\":\"user\",\"content\":\"supercalifragilisticexpialidocious\"}"); + } + +} diff --git a/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/trace/LangfuseMessageFormatterTest.java b/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/trace/LangfuseMessageFormatterTest.java new file mode 100644 index 00000000000..4f8cd955eec --- /dev/null +++ b/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/trace/LangfuseMessageFormatterTest.java @@ -0,0 +1,83 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.chat.observation.trace; + +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.*; + +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Unit tests for {@link LangfuseMessageFormatter}. + * + * @author tingchuan.li + */ +class LangfuseMessageFormatterTest { + + private static final String TEXT = "Hello World!"; + + private static final String ID = "test_id"; + + private static final String TYPE = "test_type"; + + private static final String NAME = "test_name"; + + private static final String ARGUMENTS = "test_arguments"; + + private static final String RESPONSE_DATA = "test_response_data"; + + private static final MessageFormatter FORMATTER = new LangfuseMessageFormatter(); + + @Test + void systemFormat() { + Message userMessage = new SystemMessage(TEXT); + assertThat(FORMATTER.format(userMessage)).isEqualTo("{\"role\":\"system\",\"content\":\"Hello World!\"}"); + } + + @Test + void userFormat() { + Message userMessage = new UserMessage(TEXT); + assertThat(FORMATTER.format(userMessage)).isEqualTo("{\"role\":\"user\",\"content\":\"Hello World!\"}"); + } + + @Test + void assistantFormat() { + Message assistantMessage = new AssistantMessage(TEXT); + assertThat(FORMATTER.format(assistantMessage)) + .isEqualTo("{\"role\":\"assistant\",\"content\":\"Hello World!\"}"); + } + + @Test + void assistantToolcallFormat() { + Message assistantMessage = new AssistantMessage("", Map.of(), + List.of(new AssistantMessage.ToolCall(ID, TYPE, NAME, ARGUMENTS))); + assertThat(FORMATTER.format(assistantMessage)).isEqualTo( + "{\"role\":\"assistant\",\"content\":[{\"id\":\"test_id\",\"type\":\"test_type\",\"name\":\"test_name\",\"arguments\":\"test_arguments\"}]}"); + } + + @Test + void toolResponseFormat() { + Message toolResponseMessage = new ToolResponseMessage( + List.of(new ToolResponseMessage.ToolResponse(ID, NAME, RESPONSE_DATA))); + assertThat(FORMATTER.format(toolResponseMessage)).isEqualTo( + "{\"role\":\"tool\",\"content\":[{\"id\":\"test_id\",\"name\":\"test_name\",\"responseData\":\"test_response_data\"}]}"); + } + +} diff --git a/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/trace/TextMessageFormatterTest.java b/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/trace/TextMessageFormatterTest.java new file mode 100644 index 00000000000..e8602b46cf8 --- /dev/null +++ b/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/trace/TextMessageFormatterTest.java @@ -0,0 +1,40 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.chat.observation.trace; + +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.UserMessage; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Unit tests for {@link TextMessageFormatter}. + * + * @author tingchuan.li + */ +class TextMessageFormatterTest { + + @Test + void format() { + String text = "Hello World!"; + Message message = new UserMessage(text); + MessageFormatter formatter = new TextMessageFormatter(); + assertThat(formatter.format(message)).isEqualTo(text); + } + +}