From b48e24d685867256a4442f40b1c7e6b2b043f096 Mon Sep 17 00:00:00 2001 From: Hannah Ramadan Date: Thu, 22 Feb 2024 11:02:08 -0800 Subject: [PATCH 01/12] WIP: llm_custom_attributes --- .../ruby_openai/instrumentation.rb | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb b/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb index c3fb6d20f8..1dc3d3b280 100644 --- a/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb +++ b/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb @@ -77,13 +77,37 @@ def create_chat_completion_summary(parameters) end def create_embeddings_event(parameters) - NewRelic::Agent::Llm::Embedding.new( + event = NewRelic::Agent::Llm::Embedding.new( # TODO: POST-GA: Add metadata from add_custom_attributes if prefixed with 'llm.', except conversation_id vendor: VENDOR, input: parameters[:input] || parameters['input'], api_key_last_four_digits: parse_api_key, request_model: parameters[:model] || parameters['model'] ) + add_llm_custom_attributes(event) if llm_custom_attributes + end + + def llm_custom_attributes + binding.irb + NewRelic::Agent::Tracer.current_transaction.attributes.custom_attributes.select { |k,v| k.to_s.match(/llm.*/)} + end + + def add_llm_custom_attributes(event) + attributes = transform_custom_attributes(llm_custom_attributes) + # attributes = llm_custom_attributes + binding.irb + attributes.map {|k,v| event.instance_variable_set("@#{k}".to_sym, v)} + binding.irb + event + end + + def transform_custom_attributes(attributes) + binding.irb + # new = attributes.map {|k,v| + + attributes.each { |k, v| attributes[k.gsub('.', '_')] = v} + + new end def add_chat_completion_response_params(parameters, response, event) From 859069ff2bb5f3282128f5cf3bdfec9fcbcaa995 Mon Sep 17 00:00:00 2001 From: Hannah Ramadan Date: Tue, 27 Feb 2024 17:46:07 -0800 Subject: [PATCH 02/12] Refactor --- .../ruby_openai/instrumentation.rb | 32 ++++--------------- lib/new_relic/agent/llm/llm_event.rb | 5 ++- 2 files changed, 11 insertions(+), 26 deletions(-) diff --git a/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb b/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb index f916057d40..6693d5c67e 100644 --- a/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb +++ b/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb @@ -30,6 +30,7 @@ def embeddings_instrumentation(parameters) segment = NewRelic::Agent::Tracer.start_segment(name: EMBEDDINGS_SEGMENT_NAME) record_openai_metric event = create_embeddings_event(parameters) + event.metadata = llm_custom_attributes segment.llm_event = event begin response = NewRelic::Agent::Tracer.capture_segment_error(segment) { yield } @@ -46,6 +47,7 @@ def chat_completions_instrumentation(parameters) segment = NewRelic::Agent::Tracer.start_segment(name: CHAT_COMPLETIONS_SEGMENT_NAME) record_openai_metric event = create_chat_completion_summary(parameters) + event.metadata = llm_custom_attributes segment.llm_event = event messages = create_chat_completion_messages(parameters, event.id) @@ -76,36 +78,12 @@ def create_chat_completion_summary(parameters) end def create_embeddings_event(parameters) - event = NewRelic::Agent::Llm::Embedding.new( + NewRelic::Agent::Llm::Embedding.new( # TODO: POST-GA: Add metadata from add_custom_attributes if prefixed with 'llm.', except conversation_id vendor: VENDOR, input: parameters[:input] || parameters['input'], request_model: parameters[:model] || parameters['model'] ) - add_llm_custom_attributes(event) if llm_custom_attributes - end - - def llm_custom_attributes - binding.irb - NewRelic::Agent::Tracer.current_transaction.attributes.custom_attributes.select { |k,v| k.to_s.match(/llm.*/)} - end - - def add_llm_custom_attributes(event) - attributes = transform_custom_attributes(llm_custom_attributes) - # attributes = llm_custom_attributes - binding.irb - attributes.map {|k,v| event.instance_variable_set("@#{k}".to_sym, v)} - binding.irb - event - end - - def transform_custom_attributes(attributes) - binding.irb - # new = attributes.map {|k,v| - - attributes.each { |k, v| attributes[k.gsub('.', '_')] = v} - - new end def add_chat_completion_response_params(parameters, response, event) @@ -171,6 +149,10 @@ def update_chat_completion_messages(messages, response, summary) end end + def llm_custom_attributes + NewRelic::Agent::Tracer.current_transaction.attributes.custom_attributes.select { |k,v| k.to_s.match(/llm.*/)} + end + def record_openai_metric NewRelic::Agent.record_metric(nr_supportability_metric, 0.0) end diff --git a/lib/new_relic/agent/llm/llm_event.rb b/lib/new_relic/agent/llm/llm_event.rb index ebffddd8c1..d37d990b36 100644 --- a/lib/new_relic/agent/llm/llm_event.rb +++ b/lib/new_relic/agent/llm/llm_event.rb @@ -27,6 +27,7 @@ class LlmEvent PARAM_STRING = 'param' attr_accessor(*ATTRIBUTES) + attr_accessor :metadata def self.set_llm_agent_attribute_on_transaction NewRelic::Agent::Transaction.add_agent_attribute(LLM, true, NewRelic::Agent::AttributeFilter::DST_TRANSACTION_EVENTS) @@ -51,9 +52,10 @@ def initialize(opts = {}) # All subclasses use event_attributes to get a full hash of all # attributes and their values def event_attributes - attributes.each_with_object({}) do |attr, hash| + attributes_hash = attributes.each_with_object({}) do |attr, hash| hash[replace_attr_with_string(attr)] = instance_variable_get(:"@#{attr}") end + attributes_hash.merge!(metadata) end # Subclasses define an attributes method to concatenate attributes @@ -75,6 +77,7 @@ def attribute_name_exceptions end def record + binding.irb NewRelic::Agent.record_custom_event(event_name, event_attributes) end From 9c665cb36dad0f1da5f44a7a7b2f5cab439777a5 Mon Sep 17 00:00:00 2001 From: Hannah Ramadan Date: Wed, 28 Feb 2024 09:59:27 -0800 Subject: [PATCH 03/12] Do empty? check on metadata --- lib/new_relic/agent/llm/llm_event.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/new_relic/agent/llm/llm_event.rb b/lib/new_relic/agent/llm/llm_event.rb index d37d990b36..78dc4557c0 100644 --- a/lib/new_relic/agent/llm/llm_event.rb +++ b/lib/new_relic/agent/llm/llm_event.rb @@ -55,7 +55,7 @@ def event_attributes attributes_hash = attributes.each_with_object({}) do |attr, hash| hash[replace_attr_with_string(attr)] = instance_variable_get(:"@#{attr}") end - attributes_hash.merge!(metadata) + attributes_hash.merge!(metadata) if !metadata.empty? end # Subclasses define an attributes method to concatenate attributes @@ -77,7 +77,6 @@ def attribute_name_exceptions end def record - binding.irb NewRelic::Agent.record_custom_event(event_name, event_attributes) end From e4b0f95c657320a48e1a42a470676e64c59a7fa8 Mon Sep 17 00:00:00 2001 From: Hannah Ramadan Date: Wed, 28 Feb 2024 10:26:32 -0800 Subject: [PATCH 04/12] Add test --- lib/new_relic/agent/llm/llm_event.rb | 4 +++- test/new_relic/agent/llm/llm_event_test.rb | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/new_relic/agent/llm/llm_event.rb b/lib/new_relic/agent/llm/llm_event.rb index 78dc4557c0..e15f8af54a 100644 --- a/lib/new_relic/agent/llm/llm_event.rb +++ b/lib/new_relic/agent/llm/llm_event.rb @@ -55,7 +55,9 @@ def event_attributes attributes_hash = attributes.each_with_object({}) do |attr, hash| hash[replace_attr_with_string(attr)] = instance_variable_get(:"@#{attr}") end - attributes_hash.merge!(metadata) if !metadata.empty? + attributes_hash.merge!(metadata) if !metadata.nil? + + attributes_hash end # Subclasses define an attributes method to concatenate attributes diff --git a/test/new_relic/agent/llm/llm_event_test.rb b/test/new_relic/agent/llm/llm_event_test.rb index a28b27e963..4cd0040e36 100644 --- a/test/new_relic/agent/llm/llm_event_test.rb +++ b/test/new_relic/agent/llm/llm_event_test.rb @@ -46,6 +46,17 @@ def test_event_attributes_returns_a_hash_of_assigned_attributes_and_values assert_equal('gpt-4', result['response.model']) end + def test_event_attributes_adds_custom_attributes + event = NewRelic::Agent::Llm::LlmEvent.new(id: 123) + event.vendor = 'OpenAI' + event.response_model = 'gpt-4' + event.metadata = {'Marathon' => 26.2, 'Ultra Marathon' => 'Ouch'} + result = event.event_attributes + + assert_equal(26.2, result['Marathon']) + assert_equal('Ouch', result['Ultra Marathon']) + end + def test_record_does_not_create_an_event event = NewRelic::Agent::Llm::LlmEvent.new event.record From 1d3c799a5229ad42409e4511c41381246d14ec70 Mon Sep 17 00:00:00 2001 From: Hannah Ramadan Date: Wed, 28 Feb 2024 10:55:31 -0800 Subject: [PATCH 05/12] Rubocop --- .../agent/instrumentation/ruby_openai/instrumentation.rb | 2 +- test/new_relic/agent/llm/llm_event_test.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb b/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb index 6693d5c67e..74d6a864dc 100644 --- a/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb +++ b/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb @@ -150,7 +150,7 @@ def update_chat_completion_messages(messages, response, summary) end def llm_custom_attributes - NewRelic::Agent::Tracer.current_transaction.attributes.custom_attributes.select { |k,v| k.to_s.match(/llm.*/)} + NewRelic::Agent::Tracer.current_transaction.attributes.custom_attributes.select { |k, v| k.to_s.match(/llm.*/) } end def record_openai_metric diff --git a/test/new_relic/agent/llm/llm_event_test.rb b/test/new_relic/agent/llm/llm_event_test.rb index 4cd0040e36..19ae13595c 100644 --- a/test/new_relic/agent/llm/llm_event_test.rb +++ b/test/new_relic/agent/llm/llm_event_test.rb @@ -50,10 +50,10 @@ def test_event_attributes_adds_custom_attributes event = NewRelic::Agent::Llm::LlmEvent.new(id: 123) event.vendor = 'OpenAI' event.response_model = 'gpt-4' - event.metadata = {'Marathon' => 26.2, 'Ultra Marathon' => 'Ouch'} + event.metadata = {'Marathon' => '26.2', 'Ultra Marathon' => 'Ouch'} result = event.event_attributes - assert_equal(26.2, result['Marathon']) + assert_equal('26.2', result['Marathon']) assert_equal('Ouch', result['Ultra Marathon']) end From 2a6762e7f29113c46e09ec0030a179b7d486ffd4 Mon Sep 17 00:00:00 2001 From: Hannah Ramadan Date: Thu, 29 Feb 2024 10:32:56 -0800 Subject: [PATCH 06/12] Add tests: llm custom attibutes on all events --- .../ruby_openai/instrumentation.rb | 1 + .../ruby_openai_instrumentation_test.rb | 42 +++++++++++++++---- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb b/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb index 74d6a864dc..4e2438ea85 100644 --- a/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb +++ b/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb @@ -146,6 +146,7 @@ def update_chat_completion_messages(messages, response, summary) message.conversation_id = conversation_id message.request_id = summary.request_id message.response_model = response['model'] + message.metadata = llm_custom_attributes end end diff --git a/test/multiverse/suites/ruby_openai/ruby_openai_instrumentation_test.rb b/test/multiverse/suites/ruby_openai/ruby_openai_instrumentation_test.rb index 5dec72f4a0..6475b44304 100644 --- a/test/multiverse/suites/ruby_openai/ruby_openai_instrumentation_test.rb +++ b/test/multiverse/suites/ruby_openai/ruby_openai_instrumentation_test.rb @@ -130,10 +130,12 @@ def test_set_llm_agent_attribute_on_chat_transaction assert_truthy harvest_transaction_events![1][0][2][:llm] end - def test_conversation_id_added_to_summary_events - conversation_id = '12345' + def test_llm_custom_attributes_added_to_summary_events in_transaction do - NewRelic::Agent.add_custom_attributes({'llm.conversation_id' => conversation_id}) + NewRelic::Agent.add_custom_attributes({ + 'llm.conversation_id' => '1993', + 'llm.JurassicPark' => 'Steven Spielberg', + 'trex' => 'carnivore' }) stub_post_request do client.chat(parameters: chat_params) end @@ -142,24 +144,46 @@ def test_conversation_id_added_to_summary_events _, events = @aggregator.harvest! summary_event = events.find { |event| event[0]['type'] == NewRelic::Agent::Llm::ChatCompletionSummary::EVENT_NAME } - assert_equal conversation_id, summary_event[1]['conversation_id'] + assert_equal '1993', summary_event[1]['llm.conversation_id'] + assert_equal 'Steven Spielberg', summary_event[1]['llm.JurassicPark'] + refute summary_event[1]['trex'] end - def test_conversation_id_added_to_message_events - conversation_id = '12345' + def test_llm_custom_attributes_added_to_embedding_events + in_transaction do + NewRelic::Agent.add_custom_attributes({ + 'llm.conversation_id' => '1997', + 'llm.TheLostWorld' => 'Steven Spielberg', + 'triceratops' => 'herbivore' }) + stub_post_request do + client.embeddings(parameters: chat_params) + end + end + _, events = @aggregator.harvest! + embedding_event = events.find { |event| event[0]['type'] == NewRelic::Agent::Llm::Embedding::EVENT_NAME } + + assert_equal '1997', embedding_event[1]['llm.conversation_id'] + assert_equal 'Steven Spielberg', embedding_event[1]['llm.TheLostWorld'] + refute embedding_event[1]['fruit'] + end + def test_llm_custom_attributes_added_to_message_events in_transaction do - NewRelic::Agent.add_custom_attributes({'llm.conversation_id' => conversation_id}) + NewRelic::Agent.add_custom_attributes({ + 'llm.conversation_id' => '2001', + 'llm.JurassicParkIII' => 'Joe Johnston', + 'Pterosaur' => 'Can fly — scary!' }) stub_post_request do client.chat(parameters: chat_params) end end - _, events = @aggregator.harvest! message_events = events.filter { |event| event[0]['type'] == NewRelic::Agent::Llm::ChatCompletionMessage::EVENT_NAME } message_events.each do |event| - assert_equal conversation_id, event[1]['conversation_id'] + assert_equal '2001', event[1]['llm.conversation_id'] + assert_equal 'Joe Johnston', event[1]['llm.JurassicParkIII'] + refute event[1]['Pterosaur'] end end From fe43276aaf259445087d7e14fd931bc633133569 Mon Sep 17 00:00:00 2001 From: Hannah Ramadan Date: Thu, 29 Feb 2024 10:42:39 -0800 Subject: [PATCH 07/12] Rubocop --- .../ruby_openai/ruby_openai_instrumentation_test.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/multiverse/suites/ruby_openai/ruby_openai_instrumentation_test.rb b/test/multiverse/suites/ruby_openai/ruby_openai_instrumentation_test.rb index 6475b44304..a80df66aed 100644 --- a/test/multiverse/suites/ruby_openai/ruby_openai_instrumentation_test.rb +++ b/test/multiverse/suites/ruby_openai/ruby_openai_instrumentation_test.rb @@ -135,7 +135,8 @@ def test_llm_custom_attributes_added_to_summary_events NewRelic::Agent.add_custom_attributes({ 'llm.conversation_id' => '1993', 'llm.JurassicPark' => 'Steven Spielberg', - 'trex' => 'carnivore' }) + 'trex' => 'carnivore' + }) stub_post_request do client.chat(parameters: chat_params) end @@ -154,7 +155,8 @@ def test_llm_custom_attributes_added_to_embedding_events NewRelic::Agent.add_custom_attributes({ 'llm.conversation_id' => '1997', 'llm.TheLostWorld' => 'Steven Spielberg', - 'triceratops' => 'herbivore' }) + 'triceratops' => 'herbivore' + }) stub_post_request do client.embeddings(parameters: chat_params) end @@ -172,7 +174,8 @@ def test_llm_custom_attributes_added_to_message_events NewRelic::Agent.add_custom_attributes({ 'llm.conversation_id' => '2001', 'llm.JurassicParkIII' => 'Joe Johnston', - 'Pterosaur' => 'Can fly — scary!' }) + 'Pterosaur' => 'Can fly — scary!' + }) stub_post_request do client.chat(parameters: chat_params) end From 0a607671cb5a508f7104f2fdbf772961c2ae31e4 Mon Sep 17 00:00:00 2001 From: Hannah Ramadan <76922290+hannahramadan@users.noreply.github.com> Date: Thu, 29 Feb 2024 10:44:26 -0800 Subject: [PATCH 08/12] Update lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb Co-authored-by: James Bunch --- .../agent/instrumentation/ruby_openai/instrumentation.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb b/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb index 4e2438ea85..42e497cc51 100644 --- a/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb +++ b/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb @@ -151,7 +151,7 @@ def update_chat_completion_messages(messages, response, summary) end def llm_custom_attributes - NewRelic::Agent::Tracer.current_transaction.attributes.custom_attributes.select { |k, v| k.to_s.match(/llm.*/) } + NewRelic::Agent::Tracer.current_transaction.attributes.custom_attributes.select { |k| k.to_s.match(/llm.*/) } end def record_openai_metric From 949c101980b88f3bcbbb01f9f6e2926e01402070 Mon Sep 17 00:00:00 2001 From: Hannah Ramadan Date: Mon, 4 Mar 2024 14:05:27 -0800 Subject: [PATCH 09/12] Remove 'llm.' from keys and refactor metadata attr_accessor --- .../instrumentation/ruby_openai/instrumentation.rb | 12 +++++++----- lib/new_relic/agent/llm/llm_event.rb | 5 ++--- .../ruby_openai/ruby_openai_instrumentation_test.rb | 12 ++++++------ 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb b/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb index 42e497cc51..678d0481f0 100644 --- a/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb +++ b/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb @@ -30,7 +30,6 @@ def embeddings_instrumentation(parameters) segment = NewRelic::Agent::Tracer.start_segment(name: EMBEDDINGS_SEGMENT_NAME) record_openai_metric event = create_embeddings_event(parameters) - event.metadata = llm_custom_attributes segment.llm_event = event begin response = NewRelic::Agent::Tracer.capture_segment_error(segment) { yield } @@ -47,7 +46,6 @@ def chat_completions_instrumentation(parameters) segment = NewRelic::Agent::Tracer.start_segment(name: CHAT_COMPLETIONS_SEGMENT_NAME) record_openai_metric event = create_chat_completion_summary(parameters) - event.metadata = llm_custom_attributes segment.llm_event = event messages = create_chat_completion_messages(parameters, event.id) @@ -73,7 +71,8 @@ def create_chat_completion_summary(parameters) conversation_id: conversation_id, request_max_tokens: parameters[:max_tokens] || parameters['max_tokens'], request_model: parameters[:model] || parameters['model'], - temperature: parameters[:temperature] || parameters['temperature'] + temperature: parameters[:temperature] || parameters['temperature'], + metadata: llm_custom_attributes ) end @@ -82,7 +81,8 @@ def create_embeddings_event(parameters) # TODO: POST-GA: Add metadata from add_custom_attributes if prefixed with 'llm.', except conversation_id vendor: VENDOR, input: parameters[:input] || parameters['input'], - request_model: parameters[:model] || parameters['model'] + request_model: parameters[:model] || parameters['model'], + metadata: llm_custom_attributes ) end @@ -151,7 +151,9 @@ def update_chat_completion_messages(messages, response, summary) end def llm_custom_attributes - NewRelic::Agent::Tracer.current_transaction.attributes.custom_attributes.select { |k| k.to_s.match(/llm.*/) } + attributes = NewRelic::Agent::Tracer.current_transaction&.attributes&.custom_attributes&.select { |k| k.to_s.match(/llm.*/) } + + attributes.transform_keys! { |key| key[4..-1] } if !attributes.nil? end def record_openai_metric diff --git a/lib/new_relic/agent/llm/llm_event.rb b/lib/new_relic/agent/llm/llm_event.rb index e15f8af54a..9d1c01bb7a 100644 --- a/lib/new_relic/agent/llm/llm_event.rb +++ b/lib/new_relic/agent/llm/llm_event.rb @@ -9,7 +9,7 @@ class LlmEvent # Every subclass must define its own ATTRIBUTES constant, an array of symbols representing # that class's unique attributes ATTRIBUTES = %i[id request_id span_id trace_id response_model vendor - ingest_source] + ingest_source metadata] # These attributes should not be passed as arguments to initialize and will be set by the agent AGENT_DEFINED_ATTRIBUTES = %i[span_id trace_id ingest_source] # Some attributes have names that can't be written as symbols used for metaprogramming. @@ -27,7 +27,6 @@ class LlmEvent PARAM_STRING = 'param' attr_accessor(*ATTRIBUTES) - attr_accessor :metadata def self.set_llm_agent_attribute_on_transaction NewRelic::Agent::Transaction.add_agent_attribute(LLM, true, NewRelic::Agent::AttributeFilter::DST_TRANSACTION_EVENTS) @@ -55,7 +54,7 @@ def event_attributes attributes_hash = attributes.each_with_object({}) do |attr, hash| hash[replace_attr_with_string(attr)] = instance_variable_get(:"@#{attr}") end - attributes_hash.merge!(metadata) if !metadata.nil? + attributes_hash.merge!(metadata) && attributes_hash.delete(:metadata) attributes_hash end diff --git a/test/multiverse/suites/ruby_openai/ruby_openai_instrumentation_test.rb b/test/multiverse/suites/ruby_openai/ruby_openai_instrumentation_test.rb index a80df66aed..6e09f48878 100644 --- a/test/multiverse/suites/ruby_openai/ruby_openai_instrumentation_test.rb +++ b/test/multiverse/suites/ruby_openai/ruby_openai_instrumentation_test.rb @@ -145,8 +145,8 @@ def test_llm_custom_attributes_added_to_summary_events _, events = @aggregator.harvest! summary_event = events.find { |event| event[0]['type'] == NewRelic::Agent::Llm::ChatCompletionSummary::EVENT_NAME } - assert_equal '1993', summary_event[1]['llm.conversation_id'] - assert_equal 'Steven Spielberg', summary_event[1]['llm.JurassicPark'] + assert_equal '1993', summary_event[1]['conversation_id'] + assert_equal 'Steven Spielberg', summary_event[1]['JurassicPark'] refute summary_event[1]['trex'] end @@ -164,8 +164,8 @@ def test_llm_custom_attributes_added_to_embedding_events _, events = @aggregator.harvest! embedding_event = events.find { |event| event[0]['type'] == NewRelic::Agent::Llm::Embedding::EVENT_NAME } - assert_equal '1997', embedding_event[1]['llm.conversation_id'] - assert_equal 'Steven Spielberg', embedding_event[1]['llm.TheLostWorld'] + assert_equal '1997', embedding_event[1]['conversation_id'] + assert_equal 'Steven Spielberg', embedding_event[1]['TheLostWorld'] refute embedding_event[1]['fruit'] end @@ -184,8 +184,8 @@ def test_llm_custom_attributes_added_to_message_events message_events = events.filter { |event| event[0]['type'] == NewRelic::Agent::Llm::ChatCompletionMessage::EVENT_NAME } message_events.each do |event| - assert_equal '2001', event[1]['llm.conversation_id'] - assert_equal 'Joe Johnston', event[1]['llm.JurassicParkIII'] + assert_equal '2001', event[1]['conversation_id'] + assert_equal 'Joe Johnston', event[1]['JurassicParkIII'] refute event[1]['Pterosaur'] end end From 9a20ccb2b84583b80acd0e1da9281cb399b9f479 Mon Sep 17 00:00:00 2001 From: Hannah Ramadan Date: Mon, 4 Mar 2024 14:08:23 -0800 Subject: [PATCH 10/12] Use safe navigation --- .../agent/instrumentation/ruby_openai/instrumentation.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb b/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb index 678d0481f0..a3290d6eb0 100644 --- a/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb +++ b/lib/new_relic/agent/instrumentation/ruby_openai/instrumentation.rb @@ -153,7 +153,7 @@ def update_chat_completion_messages(messages, response, summary) def llm_custom_attributes attributes = NewRelic::Agent::Tracer.current_transaction&.attributes&.custom_attributes&.select { |k| k.to_s.match(/llm.*/) } - attributes.transform_keys! { |key| key[4..-1] } if !attributes.nil? + attributes&.transform_keys! { |key| key[4..-1] } end def record_openai_metric From 3d20ea8d7be9dd86632a15147845b99123694fa6 Mon Sep 17 00:00:00 2001 From: Hannah Ramadan Date: Mon, 4 Mar 2024 14:20:12 -0800 Subject: [PATCH 11/12] nil check! --- lib/new_relic/agent/llm/llm_event.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/new_relic/agent/llm/llm_event.rb b/lib/new_relic/agent/llm/llm_event.rb index 9d1c01bb7a..96dae0ccff 100644 --- a/lib/new_relic/agent/llm/llm_event.rb +++ b/lib/new_relic/agent/llm/llm_event.rb @@ -54,7 +54,7 @@ def event_attributes attributes_hash = attributes.each_with_object({}) do |attr, hash| hash[replace_attr_with_string(attr)] = instance_variable_get(:"@#{attr}") end - attributes_hash.merge!(metadata) && attributes_hash.delete(:metadata) + attributes_hash.merge!(metadata) && attributes_hash.delete(:metadata) if !metadata.nil? attributes_hash end From 7b1908192e0f445809ccb313fac635006dcfd764 Mon Sep 17 00:00:00 2001 From: Hannah Ramadan <76922290+hannahramadan@users.noreply.github.com> Date: Tue, 5 Mar 2024 09:21:19 -0800 Subject: [PATCH 12/12] Update test/new_relic/agent/llm/llm_event_test.rb Co-authored-by: Kayla Reopelle (she/her) <87386821+kaylareopelle@users.noreply.github.com> --- test/new_relic/agent/llm/llm_event_test.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/new_relic/agent/llm/llm_event_test.rb b/test/new_relic/agent/llm/llm_event_test.rb index 19ae13595c..5ce4a4ef26 100644 --- a/test/new_relic/agent/llm/llm_event_test.rb +++ b/test/new_relic/agent/llm/llm_event_test.rb @@ -55,6 +55,7 @@ def test_event_attributes_adds_custom_attributes assert_equal('26.2', result['Marathon']) assert_equal('Ouch', result['Ultra Marathon']) + assert_equal('OpenAI', result[:vendor]) end def test_record_does_not_create_an_event