diff --git a/newrelic/hooks/mlmodel_openai.py b/newrelic/hooks/mlmodel_openai.py index 392d6c453f..db58d00e3a 100644 --- a/newrelic/hooks/mlmodel_openai.py +++ b/newrelic/hooks/mlmodel_openai.py @@ -19,6 +19,7 @@ from newrelic.api.function_trace import FunctionTrace from newrelic.api.time_trace import get_trace_linking_metadata from newrelic.api.transaction import current_transaction +from newrelic.common.encoding_utils import json_decode from newrelic.common.object_wrapper import wrap_function_wrapper from newrelic.common.package_version_utils import get_package_version from newrelic.core.config import global_settings @@ -192,16 +193,36 @@ def wrap_chat_completion_sync(wrapped, instance, args, kwargs): try: return_val = wrapped(*args, **kwargs) except Exception as exc: - exc_organization = getattr(exc, "organization", "") + if OPENAI_V1: + response = getattr(exc, "response", "") + response_headers = getattr(response, "headers", "") + exc_organization = response_headers.get("openai-organization", "") if response_headers else "" + # There appears to be a bug here in openai v1 where despite having code, + # param, etc in the error response, they are not populated on the exception + # object so grab them from the response object instead. + content = getattr(response, "content", b"{}") + response = json_decode(content.decode("utf-8")).get("error", {}) + notice_error_attributes = { + "http.statusCode": getattr(exc, "status_code", "") or "", + "error.message": response.get("message", "") or "", + "error.code": response.get("code", "") or "", + "error.param": response.get("param", "") or "", + "completion_id": completion_id, + } + else: + exc_organization = getattr(exc, "organization", "") + notice_error_attributes = { + "http.statusCode": getattr(exc, "http_status", ""), + "error.message": getattr(exc, "_message", ""), + "error.code": getattr(getattr(exc, "error", ""), "code", ""), + "error.param": getattr(exc, "param", ""), + "completion_id": completion_id, + } + # Override the default message if it is not empty. + message = notice_error_attributes.pop("error.message") + if message: + exc._nr_message = message - notice_error_attributes = { - "http.statusCode": getattr(exc, "http_status", ""), - "error.message": getattr(exc, "_message", ""), - "error.code": getattr(getattr(exc, "error", ""), "code", ""), - "error.param": getattr(exc, "param", ""), - "completion_id": completion_id, - } - exc._nr_message = notice_error_attributes.pop("error.message") ft.notice_error( attributes=notice_error_attributes, ) @@ -617,16 +638,36 @@ async def wrap_chat_completion_async(wrapped, instance, args, kwargs): try: return_val = await wrapped(*args, **kwargs) except Exception as exc: - exc_organization = getattr(exc, "organization", "") + if OPENAI_V1: + response = getattr(exc, "response", "") + response_headers = getattr(response, "headers", "") + exc_organization = response_headers.get("openai-organization", "") if response_headers else "" + # There appears to be a bug here in openai v1 where despite having code, + # param, etc in the error response, they are not populated on the exception + # object so grab them from the response object instead. + content = getattr(response, "content", b"{}") + response = json_decode(content.decode("utf-8")).get("error", {}) + notice_error_attributes = { + "http.statusCode": getattr(exc, "status_code", "") or "", + "error.message": response.get("message", "") or "", + "error.code": response.get("code", "") or "", + "error.param": response.get("param", "") or "", + "completion_id": completion_id, + } + else: + exc_organization = getattr(exc, "organization", "") + notice_error_attributes = { + "http.statusCode": getattr(exc, "http_status", ""), + "error.message": getattr(exc, "_message", ""), + "error.code": getattr(getattr(exc, "error", ""), "code", ""), + "error.param": getattr(exc, "param", ""), + "completion_id": completion_id, + } + # Override the default message if it is not empty. + message = notice_error_attributes.pop("error.message") + if message: + exc._nr_message = message - notice_error_attributes = { - "http.statusCode": getattr(exc, "http_status", ""), - "error.message": getattr(exc, "_message", ""), - "error.code": getattr(getattr(exc, "error", ""), "code", ""), - "error.param": getattr(exc, "param", ""), - "completion_id": completion_id, - } - exc._nr_message = notice_error_attributes.pop("error.message") ft.notice_error( attributes=notice_error_attributes, ) diff --git a/tests/mlmodel_openai/_mock_external_openai_server.py b/tests/mlmodel_openai/_mock_external_openai_server.py index a5bcc174c0..edcfc47f35 100644 --- a/tests/mlmodel_openai/_mock_external_openai_server.py +++ b/tests/mlmodel_openai/_mock_external_openai_server.py @@ -107,6 +107,30 @@ "system_fingerprint": None, }, ], + "Invalid API key.": [ + {"content-type": "application/json; charset=utf-8", "x-request-id": "a51821b9fd83d8e0e04542bedc174310"}, + 401, + { + "error": { + "message": "Incorrect API key provided: DEADBEEF. You can find your API key at https://platform.openai.com/account/api-keys.", + "type": "invalid_request_error", + "param": None, + "code": "invalid_api_key", + } + }, + ], + "Model does not exist.": [ + {"content-type": "application/json; charset=utf-8", "x-request-id": "3b0f8e510ee8a67c08a227a98eadbbe6"}, + 404, + { + "error": { + "message": "The model `does-not-exist` does not exist", + "type": "invalid_request_error", + "param": None, + "code": "model_not_found", + } + }, + ], "This is an embedding test.": [ { "content-type": "application/json", diff --git a/tests/mlmodel_openai/conftest.py b/tests/mlmodel_openai/conftest.py index 859ee2bdf9..180bec9cc4 100644 --- a/tests/mlmodel_openai/conftest.py +++ b/tests/mlmodel_openai/conftest.py @@ -52,6 +52,7 @@ if get_openai_version() < (1, 0): collect_ignore = [ "test_chat_completion_v1.py", + "test_chat_completion_error_v1.py", "test_embeddings_v1.py", "test_get_llm_message_ids_v1.py", "test_chat_completion_error_v1.py", @@ -144,9 +145,9 @@ def set_info(): def openai_server( openai_version, # noqa: F811 openai_clients, - wrap_openai_base_client_process_response, wrap_openai_api_requestor_request, wrap_openai_api_requestor_interpret_response, + wrap_httpx_client_send, ): """ This fixture will either create a mocked backend for testing purposes, or will @@ -166,9 +167,7 @@ def openai_server( yield # Run tests else: # Apply function wrappers to record data - wrap_function_wrapper( - "openai._base_client", "BaseClient._process_response", wrap_openai_base_client_process_response - ) + wrap_function_wrapper("httpx._client", "Client.send", wrap_httpx_client_send) yield # Run tests # Write responses to audit log with open(OPENAI_AUDIT_LOG_FILE, "w") as audit_log_fp: @@ -178,6 +177,43 @@ def openai_server( yield +def bind_send_params(request, *, stream=False, **kwargs): + return request + + +@pytest.fixture(scope="session") +def wrap_httpx_client_send(extract_shortened_prompt): # noqa: F811 + def _wrap_httpx_client_send(wrapped, instance, args, kwargs): + request = bind_send_params(*args, **kwargs) + if not request: + return wrapped(*args, **kwargs) + + params = json.loads(request.content.decode("utf-8")) + prompt = extract_shortened_prompt(params) + + # Send request + response = wrapped(*args, **kwargs) + + if response.status_code >= 400 or response.status_code < 200: + prompt = "error" + + rheaders = getattr(response, "headers") + + headers = dict( + filter( + lambda k: k[0].lower() in RECORDED_HEADERS + or k[0].lower().startswith("openai") + or k[0].lower().startswith("x-ratelimit"), + rheaders.items(), + ) + ) + body = json.loads(response.content.decode("utf-8")) + OPENAI_AUDIT_LOG_CONTENTS[prompt] = headers, response.status_code, body # Append response data to log + return response + + return _wrap_httpx_client_send + + @pytest.fixture(scope="session") def wrap_openai_api_requestor_interpret_response(): def _wrap_openai_api_requestor_interpret_response(wrapped, instance, args, kwargs): @@ -236,39 +272,3 @@ def bind_request_params(method, url, params=None, *args, **kwargs): def bind_request_interpret_response_params(result, stream): return result.content.decode("utf-8"), result.status_code, result.headers - - -def bind_base_client_process_response( - cast_to, - options, - response, - stream, - stream_cls, -): - return options, response - - -@pytest.fixture(scope="session") -def wrap_openai_base_client_process_response(extract_shortened_prompt): # noqa: F811 - def _wrap_openai_base_client_process_response(wrapped, instance, args, kwargs): - options, response = bind_base_client_process_response(*args, **kwargs) - if not options: - return wrapped(*args, **kwargs) - - data = getattr(options, "json_data", {}) - prompt = extract_shortened_prompt(data) - rheaders = getattr(response, "headers") - - headers = dict( - filter( - lambda k: k[0].lower() in RECORDED_HEADERS - or k[0].lower().startswith("openai") - or k[0].lower().startswith("x-ratelimit"), - rheaders.items(), - ) - ) - body = json.loads(response.content.decode("utf-8")) - OPENAI_AUDIT_LOG_CONTENTS[prompt] = headers, response.status_code, body # Append response data to audit log - return wrapped(*args, **kwargs) - - return _wrap_openai_base_client_process_response diff --git a/tests/mlmodel_openai/test_chat_completion_error_v1.py b/tests/mlmodel_openai/test_chat_completion_error_v1.py new file mode 100644 index 0000000000..fc2fdaa103 --- /dev/null +++ b/tests/mlmodel_openai/test_chat_completion_error_v1.py @@ -0,0 +1,414 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import openai +import pytest +from testing_support.fixtures import ( + dt_enabled, + reset_core_stats_engine, + validate_custom_event_count, +) +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_error_trace_attributes import ( + validate_error_trace_attributes, +) +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.api.transaction import add_custom_attribute +from newrelic.common.object_names import callable_name + +_test_openai_chat_completion_messages = ( + {"role": "system", "content": "You are a scientist."}, + {"role": "user", "content": "What is 212 degrees Fahrenheit converted to Celsius?"}, +) + +expected_events_on_no_model_error = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "appName": "Python Agent Test (mlmodel_openai)", + "transaction_id": "transaction-id", + "conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "api_key_last_four_digits": "sk-CRET", + "duration": None, # Response time varies each test run + "request.model": "", # No model in this test case + "response.organization": "", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.number_of_messages": 2, + "vendor": "openAI", + "ingest_source": "Python", + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "appName": "Python Agent Test (mlmodel_openai)", + "conversation_id": "my-awesome-id", + "request_id": "", + "span_id": None, + "trace_id": "trace-id", + "transaction_id": "transaction-id", + "content": "You are a scientist.", + "role": "system", + "response.model": "", + "completion_id": None, + "sequence": 0, + "vendor": "openAI", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "appName": "Python Agent Test (mlmodel_openai)", + "conversation_id": "my-awesome-id", + "request_id": "", + "span_id": None, + "trace_id": "trace-id", + "transaction_id": "transaction-id", + "content": "What is 212 degrees Fahrenheit converted to Celsius?", + "role": "user", + "completion_id": None, + "response.model": "", + "sequence": 1, + "vendor": "openAI", + "ingest_source": "Python", + }, + ), +] + + +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(TypeError), + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": {}, + }, +) +@validate_span_events( + exact_agents={ + "error.message": "Missing required arguments; Expected either ('messages' and 'model') or ('messages', 'model' and 'stream') arguments to be given", + } +) +@validate_transaction_metrics( + "test_chat_completion_error_v1:test_chat_completion_invalid_request_error_no_model", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(expected_events_on_no_model_error) +@validate_custom_event_count(count=3) +@background_task() +def test_chat_completion_invalid_request_error_no_model(set_trace_info, sync_openai_client): + with pytest.raises(TypeError): + set_trace_info() + add_custom_attribute("conversation_id", "my-awesome-id") + sync_openai_client.chat.completions.create( + messages=_test_openai_chat_completion_messages, temperature=0.7, max_tokens=100 + ) + + +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(TypeError), + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": {}, + }, +) +@validate_span_events( + exact_agents={ + "error.message": "Missing required arguments; Expected either ('messages' and 'model') or ('messages', 'model' and 'stream') arguments to be given", + } +) +@validate_transaction_metrics( + "test_chat_completion_error_v1:test_chat_completion_invalid_request_error_no_model_async", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(expected_events_on_no_model_error) +@validate_custom_event_count(count=3) +@background_task() +def test_chat_completion_invalid_request_error_no_model_async(loop, set_trace_info, async_openai_client): + with pytest.raises(TypeError): + set_trace_info() + add_custom_attribute("conversation_id", "my-awesome-id") + loop.run_until_complete( + async_openai_client.chat.completions.create( + messages=_test_openai_chat_completion_messages, temperature=0.7, max_tokens=100 + ) + ) + + +expected_events_on_invalid_model_error = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "appName": "Python Agent Test (mlmodel_openai)", + "transaction_id": "transaction-id", + "conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "api_key_last_four_digits": "sk-CRET", + "duration": None, # Response time varies each test run + "request.model": "does-not-exist", + "response.organization": "", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.number_of_messages": 1, + "vendor": "openAI", + "ingest_source": "Python", + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "appName": "Python Agent Test (mlmodel_openai)", + "conversation_id": "my-awesome-id", + "request_id": "", + "span_id": None, + "trace_id": "trace-id", + "transaction_id": "transaction-id", + "content": "Model does not exist.", + "role": "user", + "response.model": "", + "completion_id": None, + "sequence": 0, + "vendor": "openAI", + "ingest_source": "Python", + }, + ), +] + + +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(openai.NotFoundError), + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "error.code": "model_not_found", + "http.statusCode": 404, + }, + }, +) +@validate_span_events( + exact_agents={ + "error.message": "The model `does-not-exist` does not exist", + } +) +@validate_transaction_metrics( + "test_chat_completion_error_v1:test_chat_completion_invalid_request_error_invalid_model", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(expected_events_on_invalid_model_error) +@validate_custom_event_count(count=2) +@background_task() +def test_chat_completion_invalid_request_error_invalid_model(set_trace_info, sync_openai_client): + with pytest.raises(openai.NotFoundError): + set_trace_info() + add_custom_attribute("conversation_id", "my-awesome-id") + sync_openai_client.chat.completions.create( + model="does-not-exist", + messages=({"role": "user", "content": "Model does not exist."},), + temperature=0.7, + max_tokens=100, + ) + + +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(openai.NotFoundError), + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "error.code": "model_not_found", + "http.statusCode": 404, + }, + }, +) +@validate_span_events( + exact_agents={ + "error.message": "The model `does-not-exist` does not exist", + } +) +@validate_transaction_metrics( + "test_chat_completion_error_v1:test_chat_completion_invalid_request_error_invalid_model_async", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(expected_events_on_invalid_model_error) +@validate_custom_event_count(count=2) +@background_task() +def test_chat_completion_invalid_request_error_invalid_model_async(loop, set_trace_info, async_openai_client): + with pytest.raises(openai.NotFoundError): + set_trace_info() + add_custom_attribute("conversation_id", "my-awesome-id") + loop.run_until_complete( + async_openai_client.chat.completions.create( + model="does-not-exist", + messages=({"role": "user", "content": "Model does not exist."},), + temperature=0.7, + max_tokens=100, + ) + ) + + +expected_events_on_wrong_api_key_error = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "appName": "Python Agent Test (mlmodel_openai)", + "transaction_id": "transaction-id", + "conversation_id": "", + "span_id": None, + "trace_id": "trace-id", + "api_key_last_four_digits": "sk-BEEF", + "duration": None, # Response time varies each test run + "request.model": "gpt-3.5-turbo", + "response.organization": "", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.number_of_messages": 1, + "vendor": "openAI", + "ingest_source": "Python", + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "appName": "Python Agent Test (mlmodel_openai)", + "conversation_id": "", + "request_id": "", + "span_id": None, + "trace_id": "trace-id", + "transaction_id": "transaction-id", + "content": "Invalid API key.", + "role": "user", + "completion_id": None, + "response.model": "", + "sequence": 0, + "vendor": "openAI", + "ingest_source": "Python", + }, + ), +] + + +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(openai.AuthenticationError), + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "http.statusCode": 401, + }, + }, +) +@validate_span_events( + exact_agents={ + "error.message": "Incorrect API key provided: DEADBEEF. You can find your API key at https://platform.openai.com/account/api-keys.", + } +) +@validate_transaction_metrics( + "test_chat_completion_error_v1:test_chat_completion_wrong_api_key_error", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(expected_events_on_wrong_api_key_error) +@validate_custom_event_count(count=2) +@background_task() +def test_chat_completion_wrong_api_key_error(monkeypatch, set_trace_info, sync_openai_client): + with pytest.raises(openai.AuthenticationError): + set_trace_info() + monkeypatch.setattr(sync_openai_client, "api_key", "DEADBEEF") + sync_openai_client.chat.completions.create( + model="gpt-3.5-turbo", + messages=({"role": "user", "content": "Invalid API key."},), + temperature=0.7, + max_tokens=100, + ) + + +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(openai.AuthenticationError), + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "http.statusCode": 401, + }, + }, +) +@validate_span_events( + exact_agents={ + "error.message": "Incorrect API key provided: DEADBEEF. You can find your API key at https://platform.openai.com/account/api-keys.", + } +) +@validate_transaction_metrics( + "test_chat_completion_error_v1:test_chat_completion_wrong_api_key_error_async", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(expected_events_on_wrong_api_key_error) +@validate_custom_event_count(count=2) +@background_task() +def test_chat_completion_wrong_api_key_error_async(loop, monkeypatch, set_trace_info, async_openai_client): + with pytest.raises(openai.AuthenticationError): + set_trace_info() + monkeypatch.setattr(async_openai_client, "api_key", "DEADBEEF") + loop.run_until_complete( + async_openai_client.chat.completions.create( + model="gpt-3.5-turbo", + messages=({"role": "user", "content": "Invalid API key."},), + temperature=0.7, + max_tokens=100, + ) + )