From 52f00a11a9169d0cde746e866c8c040080f65f0f Mon Sep 17 00:00:00 2001 From: Kayla Reopelle Date: Wed, 31 Jan 2024 10:38:34 -0800 Subject: [PATCH] Update attribute names to period-delimited strings Some attribute names are expected upstream to include periods. We can't use names with periods in setter/getter method names. To resolve this, a new ATTRIBUTE_NAME_EXCEPTIONS constant, which is accompanied by an attribute_name_exceptions method exists to store the string value of the attribute name. Then, replace_attr_with_string returns the correct string name when we're creating the final event_attributes hash. --- .../agent/llm/chat_completion_summary.rb | 14 +++++++++ lib/new_relic/agent/llm/embedding.rb | 13 ++++++-- lib/new_relic/agent/llm/llm_event.rb | 20 ++++++++++++- lib/new_relic/agent/llm/response_headers.rb | 9 ++++++ .../agent/llm/chat_completion_message_test.rb | 2 +- .../agent/llm/chat_completion_summary_test.rb | 30 +++++++++---------- test/new_relic/agent/llm/embedding_test.rb | 24 +++++++-------- test/new_relic/agent/llm/llm_event_test.rb | 4 ++- 8 files changed, 82 insertions(+), 34 deletions(-) diff --git a/lib/new_relic/agent/llm/chat_completion_summary.rb b/lib/new_relic/agent/llm/chat_completion_summary.rb index 15fa09e493..4f64fc3cee 100644 --- a/lib/new_relic/agent/llm/chat_completion_summary.rb +++ b/lib/new_relic/agent/llm/chat_completion_summary.rb @@ -16,6 +16,16 @@ class ChatCompletionSummary < LlmEvent response_usage_total_tokens response_usage_prompt_tokens response_usage_completion_tokens response_choices_finish_reason request_temperature duration error] + ATTRIBUTE_NAME_EXCEPTIONS = { + response_number_of_messages: 'response.number_of_messages', + request_model: 'request.model', + response_usage_total_tokens: 'response.usage.total_tokens', + response_usage_prompt_tokens: 'response.usage.prompt_tokens', + response_usage_completion_tokens: 'response.usage.completion_tokens', + response_choices_finish_reason: 'response.choices.finish_reason', + temperature: 'request.temperature' + } + EVENT_NAME = 'LlmChatCompletionSummary' attr_accessor(*ATTRIBUTES) @@ -24,6 +34,10 @@ def attributes LlmEvent::ATTRIBUTES + ChatCompletion::ATTRIBUTES + ResponseHeaders::ATTRIBUTES + ATTRIBUTES end + def attribute_name_exceptions + LlmEvent::ATTRIBUTE_NAME_EXCEPTIONS.merge(ResponseHeaders::ATTRIBUTE_NAME_EXCEPTIONS, ATTRIBUTE_NAME_EXCEPTIONS) + end + def event_name EVENT_NAME end diff --git a/lib/new_relic/agent/llm/embedding.rb b/lib/new_relic/agent/llm/embedding.rb index 773831211f..782dc786ce 100644 --- a/lib/new_relic/agent/llm/embedding.rb +++ b/lib/new_relic/agent/llm/embedding.rb @@ -9,8 +9,13 @@ class Embedding < LlmEvent include ResponseHeaders ATTRIBUTES = %i[input api_key_last_four_digits request_model - response_organization response_usage_total_tokens - response_usage_prompt_tokens duration error] + response_usage_total_tokens response_usage_prompt_tokens duration + error] + ATTRIBUTE_NAME_EXCEPTIONS = { + request_model: 'request.model', + response_usage_total_tokens: 'response.usage.total_tokens', + response_usage_prompt_tokens: 'response.usage.prompt_tokens' + } EVENT_NAME = 'LlmEmbedding' attr_accessor(*ATTRIBUTES) @@ -19,6 +24,10 @@ def attributes LlmEvent::ATTRIBUTES + ResponseHeaders::ATTRIBUTES + ATTRIBUTES end + def attribute_name_exceptions + LlmEvent::ATTRIBUTE_NAME_EXCEPTIONS.merge(ResponseHeaders::ATTRIBUTE_NAME_EXCEPTIONS, ATTRIBUTE_NAME_EXCEPTIONS) + end + def event_name EVENT_NAME end diff --git a/lib/new_relic/agent/llm/llm_event.rb b/lib/new_relic/agent/llm/llm_event.rb index 7869e7fa70..995e138706 100644 --- a/lib/new_relic/agent/llm/llm_event.rb +++ b/lib/new_relic/agent/llm/llm_event.rb @@ -12,6 +12,8 @@ class LlmEvent trace_id response_model vendor ingest_source] # These attributes should not be passed as arguments to initialize and will be set by the agent AGENT_DEFINED_ATTRIBUTES = %i[span_id transaction_id trace_id ingest_source] + ATTRIBUTE_NAME_EXCEPTIONS = {response_model: 'response.model'} + INGEST_SOURCE = 'Ruby' attr_accessor(*ATTRIBUTES) @@ -36,7 +38,7 @@ def initialize(opts = {}) # attributes and their values def event_attributes attributes.each_with_object({}) do |attr, hash| - hash[attr] = instance_variable_get(:"@#{attr}") + hash[replace_attr_with_string(attr)] = instance_variable_get(:"@#{attr}") end end @@ -50,9 +52,25 @@ def attributes def event_name end + # Some attribute names include periods, which aren't valid values for + # Ruby method names. This method returns a Hash with the key as the + # Ruby symbolized version of the attribute and the value as the + # period-delimited string expected upstream + def attribute_name_exceptions + ATTRIBUTE_NAME_EXCEPTIONS + end + def record NewRelic::Agent.record_custom_event(event_name, event_attributes) end + + private + + def replace_attr_with_string(attr) + return attribute_name_exceptions[attr] if attribute_name_exceptions.key?(attr) + + attr + end end end end diff --git a/lib/new_relic/agent/llm/response_headers.rb b/lib/new_relic/agent/llm/response_headers.rb index 892b7a1cf4..4d55f469e1 100644 --- a/lib/new_relic/agent/llm/response_headers.rb +++ b/lib/new_relic/agent/llm/response_headers.rb @@ -9,6 +9,15 @@ module ResponseHeaders ATTRIBUTES = %i[llm_version rate_limit_requests rate_limit_tokens rate_limit_remaining_requests rate_limit_remaining_tokens rate_limit_reset_requests rate_limit_reset_tokens] + ATTRIBUTE_NAME_EXCEPTIONS = { + llm_version: 'response.headers.llm_version', + rate_limit_requests: 'response.headers.ratelimitLimitRequests', + rate_limit_tokens: 'response.headers.ratelimitLimitTokens', + rate_limit_remaining_requests: 'response.headers.ratelimitRemainingRequests', + rate_limit_remaining_tokens: 'response.headers.ratelimitRemainingTokens', + rate_limit_reset_requests: 'response.headers.ratelimitResetRequests', + rate_limit_reset_tokens: 'response.headers.ratelimitResetTokens' + } OPENAI_VERSION = 'openai-version' X_RATELIMIT_LIMIT_REQUESTS = 'x-ratelimit-limit-requests' diff --git a/test/new_relic/agent/llm/chat_completion_message_test.rb b/test/new_relic/agent/llm/chat_completion_message_test.rb index 7695b6fc91..e0e4d4f657 100644 --- a/test/new_relic/agent/llm/chat_completion_message_test.rb +++ b/test/new_relic/agent/llm/chat_completion_message_test.rb @@ -76,7 +76,7 @@ def test_record_creates_an_event assert_equal txn.current_segment.guid, attributes['span_id'] assert_equal txn.guid, attributes['transaction_id'] assert_equal txn.trace_id, attributes['trace_id'] - assert_equal 'gpt-4', attributes['response_model'] + assert_equal 'gpt-4', attributes['response.model'] assert_equal 'OpenAI', attributes['vendor'] assert_equal 'Ruby', attributes['ingest_source'] assert_equal 'Red-Tailed Hawk', attributes['content'] diff --git a/test/new_relic/agent/llm/chat_completion_summary_test.rb b/test/new_relic/agent/llm/chat_completion_summary_test.rb index f496346a4e..62c070ba93 100644 --- a/test/new_relic/agent/llm/chat_completion_summary_test.rb +++ b/test/new_relic/agent/llm/chat_completion_summary_test.rb @@ -64,7 +64,6 @@ def test_record_creates_an_event summary.response_number_of_messages = 5 summary.request_model = 'gpt-4-turbo-preview' summary.response_model = 'gpt-4' - summary.response_organization = '98338' summary.response_usage_total_tokens = 20 summary.response_usage_prompt_tokens = '24' summary.response_usage_completion_tokens = '26' @@ -94,25 +93,24 @@ def test_record_creates_an_event assert_equal txn.trace_id, attributes['trace_id'] assert_equal 'sk-0713', attributes['api_key_last_four_digits'] assert_equal 500, attributes['request_max_tokens'] - assert_equal 5, attributes['response_number_of_messages'] - assert_equal 'gpt-4-turbo-preview', attributes['request_model'] - assert_equal 'gpt-4', attributes['response_model'] - assert_equal '98338', attributes['response_organization'] - assert_equal 20, attributes['response_usage_total_tokens'] - assert_equal '24', attributes['response_usage_prompt_tokens'] - assert_equal '26', attributes['response_usage_completion_tokens'] - assert_equal 'stop', attributes['response_choices_finish_reason'] + assert_equal 5, attributes['response.number_of_messages'] + assert_equal 'gpt-4-turbo-preview', attributes['request.model'] + assert_equal 'gpt-4', attributes['response.model'] + assert_equal 20, attributes['response.usage.total_tokens'] + assert_equal '24', attributes['response.usage.prompt_tokens'] + assert_equal '26', attributes['response.usage.completion_tokens'] + assert_equal 'stop', attributes['response.choices.finish_reason'] assert_equal 'OpenAI', attributes['vendor'] assert_equal 'Ruby', attributes['ingest_source'] assert_equal '500', attributes['duration'] assert_equal 'true', attributes['error'] - assert_equal '2022-01-01', attributes['llm_version'] - assert_equal '100', attributes['rate_limit_requests'] - assert_equal '101', attributes['rate_limit_tokens'] - assert_equal '102', attributes['rate_limit_reset_tokens'] - assert_equal '103', attributes['rate_limit_reset_requests'] - assert_equal '104', attributes['rate_limit_remaining_tokens'] - assert_equal '105', attributes['rate_limit_remaining_requests'] + assert_equal '2022-01-01', attributes['response.headers.llm_version'] + assert_equal '100', attributes['response.headers.ratelimitLimitRequests'] + assert_equal '101', attributes['response.headers.ratelimitLimitTokens'] + assert_equal '102', attributes['response.headers.ratelimitResetTokens'] + assert_equal '103', attributes['response.headers.ratelimitResetRequests'] + assert_equal '104', attributes['response.headers.ratelimitRemainingTokens'] + assert_equal '105', attributes['response.headers.ratelimitRemainingRequests'] end end end diff --git a/test/new_relic/agent/llm/embedding_test.rb b/test/new_relic/agent/llm/embedding_test.rb index e10eceadca..cf59578bcb 100644 --- a/test/new_relic/agent/llm/embedding_test.rb +++ b/test/new_relic/agent/llm/embedding_test.rb @@ -46,7 +46,6 @@ def test_record_creates_an_event embedding.request_id = '789' embedding.api_key_last_four_digits = 'sk-0126' embedding.response_model = 'text-embedding-3-large' - embedding.response_organization = '98338' embedding.response_usage_total_tokens = '20' embedding.response_usage_prompt_tokens = '24' embedding.vendor = 'OpenAI' @@ -73,22 +72,21 @@ def test_record_creates_an_event assert_equal txn.trace_id, attributes['trace_id'] assert_equal 'Bonjour', attributes['input'] assert_equal 'sk-0126', attributes['api_key_last_four_digits'] - assert_equal 'text-embedding-ada-002', attributes['request_model'] - assert_equal 'text-embedding-3-large', attributes['response_model'] - assert_equal '98338', attributes['response_organization'] - assert_equal '20', attributes['response_usage_total_tokens'] - assert_equal '24', attributes['response_usage_prompt_tokens'] + assert_equal 'text-embedding-ada-002', attributes['request.model'] + assert_equal 'text-embedding-3-large', attributes['response.model'] + assert_equal '20', attributes['response.usage.total_tokens'] + assert_equal '24', attributes['response.usage.prompt_tokens'] assert_equal 'OpenAI', attributes['vendor'] assert_equal 'Ruby', attributes['ingest_source'] assert_equal '500', attributes['duration'] assert_equal 'true', attributes['error'] - assert_equal '2022-01-01', attributes['llm_version'] - assert_equal '100', attributes['rate_limit_requests'] - assert_equal '101', attributes['rate_limit_tokens'] - assert_equal '102', attributes['rate_limit_reset_tokens'] - assert_equal '103', attributes['rate_limit_reset_requests'] - assert_equal '104', attributes['rate_limit_remaining_tokens'] - assert_equal '105', attributes['rate_limit_remaining_requests'] + assert_equal '2022-01-01', attributes['response.headers.llm_version'] + assert_equal '100', attributes['response.headers.ratelimitLimitRequests'] + assert_equal '101', attributes['response.headers.ratelimitLimitTokens'] + assert_equal '102', attributes['response.headers.ratelimitResetTokens'] + assert_equal '103', attributes['response.headers.ratelimitResetRequests'] + assert_equal '104', attributes['response.headers.ratelimitRemainingTokens'] + assert_equal '105', attributes['response.headers.ratelimitRemainingRequests'] end end end diff --git a/test/new_relic/agent/llm/llm_event_test.rb b/test/new_relic/agent/llm/llm_event_test.rb index c69c121724..6f958bc960 100644 --- a/test/new_relic/agent/llm/llm_event_test.rb +++ b/test/new_relic/agent/llm/llm_event_test.rb @@ -33,11 +33,13 @@ def test_args_passed_to_init_not_set_as_instance_vars_when_not_in_attributes_con def test_event_attributes_returns_a_hash_of_assigned_attributes_and_values event = NewRelic::Agent::Llm::LlmEvent.new(id: 123) event.vendor = 'OpenAI' + event.response_model = 'gpt-4' result = event.event_attributes - assert_equal(result.keys, NewRelic::Agent::Llm::LlmEvent::ATTRIBUTES) + assert_instance_of(Hash, result) assert_equal(123, result[:id]) assert_equal('OpenAI', result[:vendor]) + assert_equal('gpt-4', result['response.model']) end def test_record_does_not_create_an_event