From c220c5452ab87bd1478287c2fe42c2d8f4a8fdbb Mon Sep 17 00:00:00 2001 From: Sameerlite Date: Mon, 15 Sep 2025 16:34:18 +0530 Subject: [PATCH 1/6] Add Support for Bedrock Guardrails to supportive selective Guarding --- .../prompt_templates/factory.py | 176 ++++-- .../bedrock/chat/converse_transformation.py | 88 ++- litellm/types/llms/bedrock.py | 32 +- .../chat/test_converse_transformation.py | 548 +++++++++++++++++- 4 files changed, 754 insertions(+), 90 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 2adddd52e749..161066900bf4 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -16,8 +16,8 @@ from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client from litellm.types.files import get_file_extension_from_mime_type from litellm.types.llms.anthropic import * -from litellm.types.llms.bedrock import MessageBlock as BedrockMessageBlock from litellm.types.llms.bedrock import CachePointBlock +from litellm.types.llms.bedrock import MessageBlock as BedrockMessageBlock from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.llms.ollama import OllamaVisionModelObject from litellm.types.llms.openai import ( @@ -1067,10 +1067,10 @@ def convert_to_gemini_tool_call_invoke( if tool_calls is not None: for tool in tool_calls: if "function" in tool: - gemini_function_call: Optional[VertexFunctionCall] = ( - _gemini_tool_call_invoke_helper( - function_call_params=tool["function"] - ) + gemini_function_call: Optional[ + VertexFunctionCall + ] = _gemini_tool_call_invoke_helper( + function_call_params=tool["function"] ) if gemini_function_call is not None: _parts_list.append( @@ -1589,9 +1589,9 @@ def anthropic_messages_pt( # noqa: PLR0915 ) if "cache_control" in _content_element: - _anthropic_content_element["cache_control"] = ( - _content_element["cache_control"] - ) + _anthropic_content_element[ + "cache_control" + ] = _content_element["cache_control"] user_content.append(_anthropic_content_element) elif m.get("type", "") == "text": m = cast(ChatCompletionTextObject, m) @@ -1629,9 +1629,9 @@ def anthropic_messages_pt( # noqa: PLR0915 ) if "cache_control" in _content_element: - _anthropic_content_text_element["cache_control"] = ( - _content_element["cache_control"] - ) + _anthropic_content_text_element[ + "cache_control" + ] = _content_element["cache_control"] user_content.append(_anthropic_content_text_element) @@ -2482,8 +2482,7 @@ def _validate_format(mime_type: str, image_format: str) -> str: if is_document: return BedrockImageProcessor._get_document_format( - mime_type=mime_type, - supported_doc_formats=supported_doc_formats + mime_type=mime_type, supported_doc_formats=supported_doc_formats ) else: @@ -2495,12 +2494,9 @@ def _validate_format(mime_type: str, image_format: str) -> str: f"Unsupported image format: {image_format}. Supported formats: {supported_image_and_video_formats}" ) return image_format - + @staticmethod - def _get_document_format( - mime_type: str, - supported_doc_formats: List[str] - ) -> str: + def _get_document_format(mime_type: str, supported_doc_formats: List[str]) -> str: """ Get the document format from the mime type @@ -2519,13 +2515,9 @@ def _get_document_format( The document format """ valid_extensions: Optional[List[str]] = None - potential_extensions = mimetypes.guess_all_extensions( - mime_type, strict=False - ) + potential_extensions = mimetypes.guess_all_extensions(mime_type, strict=False) valid_extensions = [ - ext[1:] - for ext in potential_extensions - if ext[1:] in supported_doc_formats + ext[1:] for ext in potential_extensions if ext[1:] in supported_doc_formats ] # Fallback to types/files.py if mimetypes doesn't return valid extensions @@ -2686,10 +2678,12 @@ def _convert_to_bedrock_tool_call_invoke( ) bedrock_content_block = BedrockContentBlock(toolUse=bedrock_tool) _parts_list.append(bedrock_content_block) - + # Check for cache_control and add a separate cachePoint block if tool.get("cache_control", None) is not None: - cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default")) + cache_point_block = BedrockContentBlock( + cachePoint=CachePointBlock(type="default") + ) _parts_list.append(cache_point_block) return _parts_list except Exception as e: @@ -2751,7 +2745,7 @@ def _convert_to_bedrock_tool_call_result( for content in content_list: if content["type"] == "text": content_str += content["text"] - + message.get("name", "") id = str(message.get("tool_call_id", str(uuid.uuid4()))) @@ -2760,7 +2754,7 @@ def _convert_to_bedrock_tool_call_result( content=[tool_result_content_block], toolUseId=id, ) - + content_block = BedrockContentBlock(toolResult=tool_result) return content_block @@ -3091,6 +3085,7 @@ async def _bedrock_converse_messages_pt_async( # noqa: PLR0915 assistant_continue_message: Optional[ Union[str, ChatCompletionAssistantMessage] ] = None, + guard_last_turn_only: bool = False, ) -> List[BedrockMessageBlock]: contents: List[BedrockMessageBlock] = [] msg_i = 0 @@ -3167,6 +3162,42 @@ async def _bedrock_converse_messages_pt_async( # noqa: PLR0915 msg_i += 1 if user_content: + # Apply guardrail wrapping if guard_last_turn_only is True and this is the last user message + if guard_last_turn_only: + # Check if this is the last user message by looking ahead + is_last_user_message = True + temp_msg_i = msg_i + while temp_msg_i < len(messages): + if messages[temp_msg_i]["role"] == "user": + is_last_user_message = False + break + temp_msg_i += 1 + + if is_last_user_message: + # Wrap only the last text block in GuardrailConverseContent + wrapped_content = [] + text_blocks = [ + block for block in user_content if "text" in block + ] + + for content_block in user_content: + if "text" in content_block: + # Only wrap the last text block + if content_block == text_blocks[-1]: + guardrail_block = BedrockContentBlock( + guardrailConverseContent={ + "text": content_block["text"] + } + ) + wrapped_content.append(guardrail_block) + else: + # Keep other text blocks as-is + wrapped_content.append(content_block) + else: + # Keep non-text content blocks as-is + wrapped_content.append(content_block) + user_content = wrapped_content + if len(contents) > 0 and contents[-1]["role"] == "user": if ( assistant_continue_message is not None @@ -3196,26 +3227,29 @@ async def _bedrock_converse_messages_pt_async( # noqa: PLR0915 current_message = messages[msg_i] tool_call_result = _convert_to_bedrock_tool_call_result(current_message) tool_content.append(tool_call_result) - + # Check if we need to add a separate cachePoint block has_cache_control = False - + # Check for message-level cache_control if current_message.get("cache_control", None) is not None: has_cache_control = True # Check for content-level cache_control in list content elif isinstance(current_message.get("content"), list): for content_element in current_message["content"]: - if (isinstance(content_element, dict) and - content_element.get("cache_control", None) is not None): + if ( + isinstance(content_element, dict) + and content_element.get("cache_control", None) is not None + ): has_cache_control = True break - + # Add a separate cachePoint block if cache_control is present if has_cache_control: - cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default")) + cache_point_block = BedrockContentBlock( + cachePoint=CachePointBlock(type="default") + ) tool_content.append(cache_point_block) - msg_i += 1 if tool_content: @@ -3296,7 +3330,7 @@ async def _bedrock_converse_messages_pt_async( # noqa: PLR0915 image_url=image_url ) assistants_parts.append(assistants_part) - # Add cache point block for assistant content elements + # Add cache point block for assistant content elements _cache_point_block = ( litellm.AmazonConverseConfig()._get_cache_point_block( message_block=cast( @@ -3308,8 +3342,12 @@ async def _bedrock_converse_messages_pt_async( # noqa: PLR0915 if _cache_point_block is not None: assistants_parts.append(_cache_point_block) assistant_content.extend(assistants_parts) - elif _assistant_content is not None and isinstance(_assistant_content, str): - assistant_content.append(BedrockContentBlock(text=_assistant_content)) + elif _assistant_content is not None and isinstance( + _assistant_content, str + ): + assistant_content.append( + BedrockContentBlock(text=_assistant_content) + ) # Add cache point block for assistant string content _cache_point_block = ( litellm.AmazonConverseConfig()._get_cache_point_block( @@ -3442,6 +3480,7 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 assistant_continue_message: Optional[ Union[str, ChatCompletionAssistantMessage] ] = None, + guard_last_turn_only: bool = False, ) -> List[BedrockMessageBlock]: """ Converts given messages from OpenAI format to Bedrock format @@ -3536,6 +3575,40 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 msg_i += 1 if user_content: + # Apply guardrail wrapping if guard_last_turn_only is True and this is the last user message + if guard_last_turn_only: + # Check if this is the last user message by looking ahead + is_last_user_message = True + temp_msg_i = msg_i + while temp_msg_i < len(messages): + if messages[temp_msg_i]["role"] == "user": + is_last_user_message = False + break + temp_msg_i += 1 + + if is_last_user_message: + # Wrap only the last text block in GuardrailConverseContent + wrapped_content = [] + text_blocks = [block for block in user_content if "text" in block] + + for content_block in user_content: + if "text" in content_block: + # Only wrap the last text block + if content_block == text_blocks[-1]: + guardrail_block = BedrockContentBlock( + guardrailConverseContent={ + "text": content_block["text"] + } + ) + wrapped_content.append(guardrail_block) + else: + # Keep other text blocks as-is + wrapped_content.append(content_block) + else: + # Keep non-text content blocks as-is + wrapped_content.append(content_block) + user_content = wrapped_content + if len(contents) > 0 and contents[-1]["role"] == "user": if ( assistant_continue_message is not None @@ -3562,29 +3635,33 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 while msg_i < len(messages) and messages[msg_i]["role"] == "tool": tool_call_result = _convert_to_bedrock_tool_call_result(messages[msg_i]) current_message = messages[msg_i] - + # Add the tool result first tool_content.append(tool_call_result) - + # Check if we need to add a separate cachePoint block has_cache_control = False - + # Check for message-level cache_control if current_message.get("cache_control", None) is not None: has_cache_control = True # Check for content-level cache_control in list content elif isinstance(current_message.get("content"), list): for content_element in current_message["content"]: - if (isinstance(content_element, dict) and - content_element.get("cache_control", None) is not None): + if ( + isinstance(content_element, dict) + and content_element.get("cache_control", None) is not None + ): has_cache_control = True break - + # Add a separate cachePoint block if cache_control is present if has_cache_control: - cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default")) + cache_point_block = BedrockContentBlock( + cachePoint=CachePointBlock(type="default") + ) tool_content.append(cache_point_block) - + msg_i += 1 if tool_content: # if last message was a 'user' message, then add a blank assistant message (bedrock requires alternating roles) @@ -3849,10 +3926,9 @@ def function_call_prompt(messages: list, functions: list): if isinstance(message["content"], str): message["content"] += f""" {function_prompt}""" else: - message["content"].append({ - "type": "text", - "text": f""" {function_prompt}""" - }) + message["content"].append( + {"type": "text", "text": f""" {function_prompt}"""} + ) function_added_to_prompt = True if function_added_to_prompt is False: diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index e3d65be8bbf7..0f83a9dbb3ad 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -501,7 +501,6 @@ def _translate_response_format_param( ) and not is_thinking_enabled ): - optional_params["tool_choice"] = ToolChoiceValuesBlock( tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME) ) @@ -632,7 +631,7 @@ def _handle_top_k_value(self, model: str, inference_params: dict) -> dict: return {} - def _transform_request_helper( + def _transform_request_helper( # noqa: PLR0915 self, model: str, system_content_blocks: List[SystemContentBlock], @@ -673,6 +672,10 @@ def _transform_request_helper( ) inference_params.pop("json_mode", None) # used for handling json_schema + # Remove raw_blocks and guard_last_turn_only from inference_params as they are handled in transformation + inference_params.pop("raw_blocks", None) + inference_params.pop("guard_last_turn_only", None) + # keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params' additional_request_params = { k: v for k, v in inference_params.items() if k not in total_supported_params @@ -780,14 +783,30 @@ async def _async_transform_request( headers=headers, ) - bedrock_messages = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + # Check for raw_blocks parameter (advanced control) + raw_blocks = optional_params.pop("raw_blocks", None) + if raw_blocks is not None: + # Use raw blocks directly if provided + bedrock_messages: List[MessageBlock] = raw_blocks + else: + # Use standard transformation with guard_last_turn_only support + guard_last_turn_only = optional_params.pop("guard_last_turn_only", False) + + # Validate that we have messages when not using raw_blocks + if not messages: + raise litellm.BadRequestError( + message="Invalid Message: bedrock requires at least one non-system message when not using raw_blocks", + model=model, + llm_provider="bedrock_converse", + ) + + bedrock_messages = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( messages=messages, model=model, llm_provider="bedrock_converse", user_continue_message=litellm_params.pop("user_continue_message", None), + guard_last_turn_only=guard_last_turn_only, ) - ) data: RequestObject = {"messages": bedrock_messages, **_data} @@ -831,12 +850,30 @@ def _transform_request( ) ## TRANSFORMATION ## - bedrock_messages: List[MessageBlock] = _bedrock_converse_messages_pt( - messages=messages, - model=model, - llm_provider="bedrock_converse", - user_continue_message=litellm_params.pop("user_continue_message", None), - ) + # Check for raw_blocks parameter (advanced control) + raw_blocks = optional_params.pop("raw_blocks", None) + if raw_blocks is not None: + # Use raw blocks directly if provided + bedrock_messages: List[MessageBlock] = raw_blocks + else: + # Use standard transformation with guard_last_turn_only support + guard_last_turn_only = optional_params.pop("guard_last_turn_only", False) + + # Validate that we have messages when not using raw_blocks + if not messages: + raise litellm.BadRequestError( + message="Invalid Message: bedrock requires at least one non-system message when not using raw_blocks", + model=model, + llm_provider="bedrock_converse", + ) + + bedrock_messages = _bedrock_converse_messages_pt( + messages=messages, + model=model, + llm_provider="bedrock_converse", + user_continue_message=litellm_params.pop("user_continue_message", None), + guard_last_turn_only=guard_last_turn_only, + ) data: RequestObject = {"messages": bedrock_messages, **_data} @@ -995,7 +1032,9 @@ def apply_tool_call_transformation_if_needed( return message, returned_finish_reason - def _translate_message_content(self, content_blocks: List[ContentBlock]) -> Tuple[ + def _translate_message_content( + self, content_blocks: List[ContentBlock] + ) -> Tuple[ str, List[ChatCompletionToolCallChunk], Optional[List[BedrockConverseReasoningContentBlock]], @@ -1010,9 +1049,9 @@ def _translate_message_content(self, content_blocks: List[ContentBlock]) -> Tupl """ content_str = "" tools: List[ChatCompletionToolCallChunk] = [] - reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = ( - None - ) + reasoningContentBlocks: Optional[ + List[BedrockConverseReasoningContentBlock] + ] = None for idx, content in enumerate(content_blocks): """ - Content is either a tool response or text @@ -1133,9 +1172,9 @@ def _transform_response( chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"} content_str = "" tools: List[ChatCompletionToolCallChunk] = [] - reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = ( - None - ) + reasoningContentBlocks: Optional[ + List[BedrockConverseReasoningContentBlock] + ] = None if message is not None: ( @@ -1148,12 +1187,12 @@ def _transform_response( chat_completion_message["provider_specific_fields"] = { "reasoningContentBlocks": reasoningContentBlocks, } - chat_completion_message["reasoning_content"] = ( - self._transform_reasoning_content(reasoningContentBlocks) - ) - chat_completion_message["thinking_blocks"] = ( - self._transform_thinking_blocks(reasoningContentBlocks) - ) + chat_completion_message[ + "reasoning_content" + ] = self._transform_reasoning_content(reasoningContentBlocks) + chat_completion_message[ + "thinking_blocks" + ] = self._transform_thinking_blocks(reasoningContentBlocks) chat_completion_message["content"] = content_str if ( json_mode is True @@ -1171,7 +1210,6 @@ def _transform_response( # Bedrock returns the response wrapped in a "properties" object # We need to extract the actual content from this wrapper try: - response_data = json.loads(json_mode_content_str) # If Bedrock wrapped the response in "properties", extract the content diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index baa7c205204f..a829a6b94b90 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -3,14 +3,9 @@ from typing_extensions import ( TYPE_CHECKING, - Protocol, Required, - Self, TypedDict, - TypeGuard, - get_origin, override, - runtime_checkable, ) from .openai import ChatCompletionToolCallChunk @@ -93,6 +88,12 @@ class BedrockConverseReasoningContentBlockDelta(TypedDict, total=False): text: str +class GuardrailConverseContentBlock(TypedDict, total=False): + """Content block for selective guardrail evaluation in Bedrock Converse API""" + + text: str + + class ContentBlock(TypedDict, total=False): text: str image: ImageBlock @@ -102,6 +103,7 @@ class ContentBlock(TypedDict, total=False): toolUse: ToolUseBlock cachePoint: CachePointBlock reasoningContent: BedrockConverseReasoningContentBlock + guardrailConverseContent: GuardrailConverseContentBlock class MessageBlock(TypedDict): @@ -581,30 +583,35 @@ class AmazonDeepSeekR1StreamingResponse(TypedDict): class BedrockS3InputDataConfig(TypedDict): """S3 input data configuration for Bedrock batch jobs.""" + s3Uri: str class BedrockInputDataConfig(TypedDict): """Input data configuration for Bedrock batch jobs.""" + s3InputDataConfig: BedrockS3InputDataConfig class BedrockS3OutputDataConfig(TypedDict): """S3 output data configuration for Bedrock batch jobs.""" + s3Uri: str class BedrockOutputDataConfig(TypedDict): """Output data configuration for Bedrock batch jobs.""" + s3OutputDataConfig: BedrockS3OutputDataConfig class BedrockCreateBatchRequest(TypedDict, total=False): """ Request structure for creating a Bedrock batch inference job. - + Reference: https://docs.aws.amazon.com/bedrock/latest/APIReference/API_CreateModelInvocationJob.html """ + jobName: str roleArn: str modelId: str @@ -616,21 +623,17 @@ class BedrockCreateBatchRequest(TypedDict, total=False): BedrockBatchJobStatus = Literal[ - "Submitted", - "InProgress", - "Completed", - "Failed", - "Stopping", - "Stopped" + "Submitted", "InProgress", "Completed", "Failed", "Stopping", "Stopped" ] class BedrockCreateBatchResponse(TypedDict): """ Response structure from creating a Bedrock batch inference job. - + Reference: https://docs.aws.amazon.com/bedrock/latest/APIReference/API_CreateModelInvocationJob.html """ + jobArn: str jobName: str status: BedrockBatchJobStatus @@ -639,9 +642,10 @@ class BedrockCreateBatchResponse(TypedDict): class BedrockGetBatchResponse(TypedDict, total=False): """ Response structure from getting a Bedrock batch inference job. - + Reference: https://docs.aws.amazon.com/bedrock/latest/APIReference/API_GetModelInvocationJob.html """ + jobArn: str jobName: str modelId: str diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 2fc710664e68..7597c3475de8 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -1589,4 +1589,550 @@ async def test_no_cache_control_no_cache_point(): # Tool message should only have tool result, no cachePoint tool_content = result[2]["content"] assert len(tool_content) == 1 - assert "toolResult" in tool_content[0] \ No newline at end of file + assert "toolResult" in tool_content[0] + + +# ============================================================================ +# GuardrailConverseContent Feature Tests +# ============================================================================ + +def test_guard_last_turn_only_wraps_last_user_message(): + """Test that guard_last_turn_only=True wraps only the last user message in GuardrailConverseContent.""" + from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_converse_messages_pt + + messages = [ + {"role": "user", "content": "Hello, how are you?"}, + {"role": "assistant", "content": "I'm doing well, thank you!"}, + {"role": "user", "content": "What's the weather like?"}, + {"role": "assistant", "content": "I don't have access to real-time weather data."}, + {"role": "user", "content": "Tell me about AI safety and potential risks"} + ] + + result = _bedrock_converse_messages_pt( + messages=messages, + model="us.amazon.nova-pro-v1:0", + llm_provider="bedrock_converse", + guard_last_turn_only=True + ) + + # Should have 5 messages + assert len(result) == 5 + + # First 4 messages should be normal (no GuardrailConverseContent) + for i in range(4): + content = result[i]["content"] + if result[i]["role"] == "user": + # User messages should have text content, not GuardrailConverseContent + assert "text" in content[0] + assert "guardrailConverseContent" not in content[0] + + # Last message (user) should have GuardrailConverseContent + last_message = result[-1] + assert last_message["role"] == "user" + last_content = last_message["content"] + assert len(last_content) == 1 + assert "guardrailConverseContent" in last_content[0] + assert last_content[0]["guardrailConverseContent"]["text"] == "Tell me about AI safety and potential risks" + + +def test_guard_last_turn_only_with_multiple_consecutive_user_messages(): + """Test that guard_last_turn_only=True wraps only the last user message when there are consecutive user messages.""" + from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_converse_messages_pt + + messages = [ + {"role": "user", "content": "First user message"}, + {"role": "user", "content": "Second user message"}, + {"role": "user", "content": "Third user message (should be guarded)"} + ] + + result = _bedrock_converse_messages_pt( + messages=messages, + model="us.amazon.nova-pro-v1:0", + llm_provider="bedrock_converse", + guard_last_turn_only=True + ) + + # Should have 1 message (consecutive user messages are merged) + assert len(result) == 1 + assert result[0]["role"] == "user" + + # Should have 3 content blocks + content = result[0]["content"] + assert len(content) == 3 + + # First two should be normal text + assert "text" in content[0] + assert content[0]["text"] == "First user message" + assert "text" in content[1] + assert content[1]["text"] == "Second user message" + + # Last one should be GuardrailConverseContent + assert "guardrailConverseContent" in content[2] + assert content[2]["guardrailConverseContent"]["text"] == "Third user message (should be guarded)" + + +def test_guard_last_turn_only_false_does_not_wrap(): + """Test that guard_last_turn_only=False does not wrap any messages in GuardrailConverseContent.""" + from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_converse_messages_pt + + messages = [ + {"role": "user", "content": "Hello, how are you?"}, + {"role": "assistant", "content": "I'm doing well, thank you!"}, + {"role": "user", "content": "Tell me about AI safety and potential risks"} + ] + + result = _bedrock_converse_messages_pt( + messages=messages, + model="us.amazon.nova-pro-v1:0", + llm_provider="bedrock_converse", + guard_last_turn_only=False + ) + + # Should have 3 messages + assert len(result) == 3 + + # No message should have GuardrailConverseContent + for message in result: + content = message["content"] + for block in content: + assert "guardrailConverseContent" not in block + + +def test_raw_blocks_used_directly(): + """Test that raw_blocks are used directly as messages without transformation.""" + config = AmazonConverseConfig() + + raw_blocks = [ + { + "role": "user", + "content": [ + {"text": "System prompt"}, + {"text": "Un-guarded history"}, + { + "guardrailConverseContent": { + "text": "Only this gets moderated" + } + } + ] + } + ] + + optional_params = { + "raw_blocks": raw_blocks, + "guardrailConfig": { + "guardrailIdentifier": "gr-abc123", + "guardrailVersion": "DRAFT" + } + } + + result = config._transform_request( + model="us.amazon.nova-pro-v1:0", + messages=[], # Empty messages + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + # Messages should be exactly the raw_blocks + assert "messages" in result + assert result["messages"] == raw_blocks + + # additionalModelRequestFields should not contain raw_blocks + assert "additionalModelRequestFields" in result + assert "raw_blocks" not in result["additionalModelRequestFields"] + + # GuardrailConfig should be present + assert "guardrailConfig" in result + assert result["guardrailConfig"]["guardrailIdentifier"] == "gr-abc123" + + +def test_raw_blocks_with_complex_structure(): + """Test raw_blocks with complex message structure.""" + config = AmazonConverseConfig() + + raw_blocks = [ + { + "role": "user", + "content": [ + {"text": "System: You are a helpful AI assistant."}, + {"text": "Previous conversation context..."}, + { + "guardrailConverseContent": { + "text": "Please help me with this sensitive topic" + } + } + ] + }, + { + "role": "assistant", + "content": [ + {"text": "I'd be happy to help you with that topic."} + ] + } + ] + + optional_params = { + "raw_blocks": raw_blocks, + "guardrailConfig": { + "guardrailIdentifier": "gr-abc123", + "guardrailVersion": "DRAFT" + } + } + + result = config._transform_request( + model="us.amazon.nova-pro-v1:0", + messages=[], # Empty messages + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + # Messages should be exactly the raw_blocks + assert "messages" in result + assert result["messages"] == raw_blocks + + # Verify GuardrailConverseContent is preserved + user_message = result["messages"][0] + assert user_message["role"] == "user" + content = user_message["content"] + assert len(content) == 3 + assert "guardrailConverseContent" in content[2] + assert content[2]["guardrailConverseContent"]["text"] == "Please help me with this sensitive topic" + + +def test_empty_messages_with_guard_last_turn_only_raises_error(): + """Test that empty messages with guard_last_turn_only=True raises appropriate error.""" + config = AmazonConverseConfig() + + optional_params = { + "guardrailConfig": { + "guardrailIdentifier": "gr-abc123", + "guardrailVersion": "DRAFT" + }, + "guard_last_turn_only": True + } + + with pytest.raises(litellm.BadRequestError) as exc_info: + config._transform_request( + model="us.amazon.nova-pro-v1:0", + messages=[], # Empty messages + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + assert "bedrock requires at least one non-system message when not using raw_blocks" in str(exc_info.value) + + +def test_empty_messages_with_raw_blocks_works(): + """Test that empty messages with raw_blocks works correctly.""" + config = AmazonConverseConfig() + + raw_blocks = [ + { + "role": "user", + "content": [ + {"text": "Hello from raw_blocks!"} + ] + } + ] + + optional_params = { + "raw_blocks": raw_blocks, + "guardrailConfig": { + "guardrailIdentifier": "gr-abc123", + "guardrailVersion": "DRAFT" + } + } + + # Should not raise an error + result = config._transform_request( + model="us.amazon.nova-pro-v1:0", + messages=[], # Empty messages + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + assert "messages" in result + assert result["messages"] == raw_blocks + + +def test_guard_last_turn_only_with_system_messages(): + """Test guard_last_turn_only with system messages using the full transformation.""" + config = AmazonConverseConfig() + + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"}, + {"role": "assistant", "content": "Hi there!"}, + {"role": "user", "content": "Tell me about AI safety"} + ] + + optional_params = { + "guardrailConfig": { + "guardrailIdentifier": "gr-abc123", + "guardrailVersion": "DRAFT" + }, + "guard_last_turn_only": True + } + + result = config._transform_request( + model="us.amazon.nova-pro-v1:0", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + # Should have 3 messages (system message is extracted to system blocks) + assert "messages" in result + assert len(result["messages"]) == 3 + + # Last user message should have GuardrailConverseContent + last_message = result["messages"][-1] + assert last_message["role"] == "user" + content = last_message["content"] + assert len(content) == 1 + assert "guardrailConverseContent" in content[0] + assert content[0]["guardrailConverseContent"]["text"] == "Tell me about AI safety" + + # System content should be in system blocks + assert "system" in result + assert len(result["system"]) == 1 + assert result["system"][0]["text"] == "You are a helpful assistant." + + +def test_guard_last_turn_only_with_mixed_content_types(): + """Test guard_last_turn_only with user messages containing mixed content types.""" + from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_converse_messages_pt + + messages = [ + {"role": "user", "content": "Hello!"}, + {"role": "assistant", "content": "Hi there!"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "Regular text"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,test"}}, + {"type": "text", "text": "This should be guarded"} + ] + } + ] + + result = _bedrock_converse_messages_pt( + messages=messages, + model="us.amazon.nova-pro-v1:0", + llm_provider="bedrock_converse", + guard_last_turn_only=True + ) + + # Should have 3 messages + assert len(result) == 3 + + # Last user message should have GuardrailConverseContent for text content only + last_message = result[-1] + assert last_message["role"] == "user" + content = last_message["content"] + assert len(content) == 3 + + # First two should be normal content + assert "text" in content[0] + assert content[0]["text"] == "Regular text" + assert "image" in content[1] # image_url gets transformed to image + + # Last one should be GuardrailConverseContent + assert "guardrailConverseContent" in content[2] + assert content[2]["guardrailConverseContent"]["text"] == "This should be guarded" + + +def test_async_guard_last_turn_only(): + """Test async version of guard_last_turn_only.""" + from litellm.litellm_core_utils.prompt_templates.factory import BedrockConverseMessagesProcessor + + messages = [ + {"role": "user", "content": "Hello!"}, + {"role": "assistant", "content": "Hi there!"}, + {"role": "user", "content": "Tell me about AI safety"} + ] + + async def run_test(): + result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="us.amazon.nova-pro-v1:0", + llm_provider="bedrock_converse", + guard_last_turn_only=True + ) + + # Should have 3 messages + assert len(result) == 3 + + # Last user message should have GuardrailConverseContent + last_message = result[-1] + assert last_message["role"] == "user" + content = last_message["content"] + assert len(content) == 1 + assert "guardrailConverseContent" in content[0] + assert content[0]["guardrailConverseContent"]["text"] == "Tell me about AI safety" + + # Run the async test + import asyncio + asyncio.run(run_test()) + + +def test_async_raw_blocks(): + """Test async version of raw_blocks.""" + config = AmazonConverseConfig() + + raw_blocks = [ + { + "role": "user", + "content": [ + {"text": "Hello from async raw_blocks!"} + ] + } + ] + + optional_params = { + "raw_blocks": raw_blocks, + "guardrailConfig": { + "guardrailIdentifier": "gr-abc123", + "guardrailVersion": "DRAFT" + } + } + + async def run_test(): + result = await config._async_transform_request( + model="us.amazon.nova-pro-v1:0", + messages=[], # Empty messages + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + assert "messages" in result + assert result["messages"] == raw_blocks + + # Run the async test + import asyncio + asyncio.run(run_test()) + + +def test_guard_last_turn_only_with_tool_calls(): + """Test guard_last_turn_only with tool calls in the conversation.""" + from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_converse_messages_pt + + messages = [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "function": {"name": "get_weather", "arguments": "{}"} + } + ] + }, + { + "role": "tool", + "tool_call_id": "call_123", + "content": "It's sunny and 25°C" + }, + {"role": "user", "content": "Tell me about AI safety"} + ] + + result = _bedrock_converse_messages_pt( + messages=messages, + model="us.amazon.nova-pro-v1:0", + llm_provider="bedrock_converse", + guard_last_turn_only=True + ) + + # Should have 3 messages (tool message is merged with user message) + assert len(result) == 3 + + # Last user message should have GuardrailConverseContent + last_message = result[-1] + assert last_message["role"] == "user" + content = last_message["content"] + # Should have tool result and guardrail content + assert len(content) == 2 + assert "toolResult" in content[0] + assert "guardrailConverseContent" in content[1] + assert content[1]["guardrailConverseContent"]["text"] == "Tell me about AI safety" + + +def test_guardrail_config_preserved_in_request(): + """Test that guardrailConfig is properly preserved in the request.""" + config = AmazonConverseConfig() + + messages = [ + {"role": "user", "content": "Tell me about AI safety"} + ] + + optional_params = { + "guardrailConfig": { + "guardrailIdentifier": "gr-abc123", + "guardrailVersion": "DRAFT" + }, + "guard_last_turn_only": True + } + + result = config._transform_request( + model="us.amazon.nova-pro-v1:0", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + # GuardrailConfig should be present at top level + assert "guardrailConfig" in result + assert result["guardrailConfig"]["guardrailIdentifier"] == "gr-abc123" + assert result["guardrailConfig"]["guardrailVersion"] == "DRAFT" + + # GuardrailConfig should also be in inferenceConfig + assert "inferenceConfig" in result + assert "guardrailConfig" in result["inferenceConfig"] + assert result["inferenceConfig"]["guardrailConfig"]["guardrailIdentifier"] == "gr-abc123" + + +def test_raw_blocks_guardrail_config_preserved(): + """Test that guardrailConfig is preserved when using raw_blocks.""" + config = AmazonConverseConfig() + + raw_blocks = [ + { + "role": "user", + "content": [ + {"text": "Hello from raw_blocks!"} + ] + } + ] + + optional_params = { + "raw_blocks": raw_blocks, + "guardrailConfig": { + "guardrailIdentifier": "gr-abc123", + "guardrailVersion": "DRAFT" + } + } + + result = config._transform_request( + model="us.amazon.nova-pro-v1:0", + messages=[], + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + # GuardrailConfig should be present at top level + assert "guardrailConfig" in result + assert result["guardrailConfig"]["guardrailIdentifier"] == "gr-abc123" + + # GuardrailConfig should also be in inferenceConfig + assert "inferenceConfig" in result + assert "guardrailConfig" in result["inferenceConfig"] + assert result["inferenceConfig"]["guardrailConfig"]["guardrailIdentifier"] == "gr-abc123" \ No newline at end of file From 202d55bec47bc4031c182c999027adcc142ccc47 Mon Sep 17 00:00:00 2001 From: Sameerlite Date: Mon, 15 Sep 2025 22:41:19 +0530 Subject: [PATCH 2/6] Add method for better handling --- .../prompt_templates/factory.py | 122 +++++++++--------- 1 file changed, 58 insertions(+), 64 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 161066900bf4..c86f08980cdf 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -3076,6 +3076,54 @@ def _initial_message_setup( messages.append(DEFAULT_USER_CONTINUE_MESSAGE) return messages + @staticmethod + def _apply_guardrail_wrapping( + user_content: List[BedrockContentBlock], messages: List[dict], msg_i: int + ) -> List[BedrockContentBlock]: + """ + Apply guardrail wrapping to the last text block in user content if this is the last user message. + + Args: + user_content: List of content blocks for the current user message + messages: Full list of messages + msg_i: Current message index + + Returns: + Modified user_content with guardrail wrapping applied if needed + """ + # Check if this is the last user message by looking ahead + is_last_user_message = True + temp_msg_i = msg_i + while temp_msg_i < len(messages): + if messages[temp_msg_i]["role"] == "user": + is_last_user_message = False + break + temp_msg_i += 1 + + if not is_last_user_message: + return user_content + + # Wrap only the last text block in GuardrailConverseContent + wrapped_content = [] + text_blocks = [block for block in user_content if "text" in block] + + for content_block in user_content: + if "text" in content_block: + # Only wrap the last text block + if content_block == text_blocks[-1]: + guardrail_block = BedrockContentBlock( + guardrailConverseContent={"text": content_block["text"]} + ) + wrapped_content.append(guardrail_block) + else: + # Keep other text blocks as-is + wrapped_content.append(content_block) + else: + # Keep non-text content blocks as-is + wrapped_content.append(content_block) + + return wrapped_content + @staticmethod async def _bedrock_converse_messages_pt_async( # noqa: PLR0915 messages: List, @@ -3164,39 +3212,11 @@ async def _bedrock_converse_messages_pt_async( # noqa: PLR0915 if user_content: # Apply guardrail wrapping if guard_last_turn_only is True and this is the last user message if guard_last_turn_only: - # Check if this is the last user message by looking ahead - is_last_user_message = True - temp_msg_i = msg_i - while temp_msg_i < len(messages): - if messages[temp_msg_i]["role"] == "user": - is_last_user_message = False - break - temp_msg_i += 1 - - if is_last_user_message: - # Wrap only the last text block in GuardrailConverseContent - wrapped_content = [] - text_blocks = [ - block for block in user_content if "text" in block - ] - - for content_block in user_content: - if "text" in content_block: - # Only wrap the last text block - if content_block == text_blocks[-1]: - guardrail_block = BedrockContentBlock( - guardrailConverseContent={ - "text": content_block["text"] - } - ) - wrapped_content.append(guardrail_block) - else: - # Keep other text blocks as-is - wrapped_content.append(content_block) - else: - # Keep non-text content blocks as-is - wrapped_content.append(content_block) - user_content = wrapped_content + user_content = ( + BedrockConverseMessagesProcessor._apply_guardrail_wrapping( + user_content=user_content, messages=messages, msg_i=msg_i + ) + ) if len(contents) > 0 and contents[-1]["role"] == "user": if ( @@ -3577,37 +3597,11 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 if user_content: # Apply guardrail wrapping if guard_last_turn_only is True and this is the last user message if guard_last_turn_only: - # Check if this is the last user message by looking ahead - is_last_user_message = True - temp_msg_i = msg_i - while temp_msg_i < len(messages): - if messages[temp_msg_i]["role"] == "user": - is_last_user_message = False - break - temp_msg_i += 1 - - if is_last_user_message: - # Wrap only the last text block in GuardrailConverseContent - wrapped_content = [] - text_blocks = [block for block in user_content if "text" in block] - - for content_block in user_content: - if "text" in content_block: - # Only wrap the last text block - if content_block == text_blocks[-1]: - guardrail_block = BedrockContentBlock( - guardrailConverseContent={ - "text": content_block["text"] - } - ) - wrapped_content.append(guardrail_block) - else: - # Keep other text blocks as-is - wrapped_content.append(content_block) - else: - # Keep non-text content blocks as-is - wrapped_content.append(content_block) - user_content = wrapped_content + user_content = ( + BedrockConverseMessagesProcessor._apply_guardrail_wrapping( + user_content=user_content, messages=messages, msg_i=msg_i + ) + ) if len(contents) > 0 and contents[-1]["role"] == "user": if ( From 8cfbf2fb4ab8ab4e60c04c73113557aa9226788f Mon Sep 17 00:00:00 2001 From: Sameerlite Date: Tue, 16 Sep 2025 23:42:55 +0530 Subject: [PATCH 3/6] Add guarded_text content type --- Dockerfile | 2 +- docker-compose.yml | 17 +- docs/my-website/docs/providers/bedrock.md | 46 +- .../prompt_templates/factory.py | 75 +-- .../bedrock/chat/converse_transformation.py | 58 +- litellm/types/llms/openai.py | 1 + .../chat/test_converse_transformation.py | 504 ++++-------------- 7 files changed, 197 insertions(+), 506 deletions(-) diff --git a/Dockerfile b/Dockerfile index addc109e10c3..b319edbbae7d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,4 +81,4 @@ COPY docker/supervisord.conf /etc/supervisord.conf ENTRYPOINT ["docker/prod_entrypoint.sh"] # Append "--detailed_debug" to the end of CMD to view detailed debug logs -CMD ["--port", "4000"] +CMD ["--port", "4000", "--detailed_debug"] diff --git a/docker-compose.yml b/docker-compose.yml index 366fbe51b5ac..df2088c93926 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,18 +5,21 @@ services: args: target: runtime image: ghcr.io/berriai/litellm:main-stable - ######################################### - ## Uncomment these lines to start proxy with a config.yaml file ## - # volumes: - # - ./config.yaml:/app/config.yaml <<- this is missing in the docker-compose file currently - # command: - # - "--config=/app/config.yaml" - ############################################## + ######################################## + # Uncomment these lines to start proxy with a config.yaml file ## + volumes: + - ./config.yaml:/app/config.yaml <<- this is missing in the docker-compose file currently + command: + - "--config=/app/config.yaml" + ############################################# ports: - "4000:4000" # Map the container port to the host, change the host port if necessary environment: DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm" STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI + LITELLM_LOG: "DEBUG" + LITELLM_LOG_LEVEL: "DEBUG" + PYTHONUNBUFFERED: "1" env_file: - .env # Load local .env file depends_on: diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index c191b742268c..165ef1d12f73 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -889,6 +889,19 @@ curl http://0.0.0.0:4000/v1/chat/completions \ Example of using [Bedrock Guardrails with LiteLLM](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-use-converse-api.html) +### Selective Content Moderation with `guarded_text` + +LiteLLM supports selective content moderation using the `guarded_text` content type. This allows you to wrap only specific content that should be moderated by Bedrock Guardrails, rather than evaluating the entire conversation. + +**How it works:** +- Content with `type: "guarded_text"` gets automatically wrapped in `guardrailConverseContent` blocks +- Only the wrapped content is evaluated by Bedrock Guardrails +- Regular content with `type: "text"` bypasses guardrail evaluation + +:::note +If `guarded_text` is not used, the entire conversation history will be sent to the guardrail for evaluation, which can increase latency and costs. +::: + @@ -915,6 +928,24 @@ response = completion( "trace": "disabled", # The trace behavior for the guardrail. Can either be "disabled" or "enabled" }, ) + +# Selective guardrail usage with guarded_text - only specific content is evaluated +response_guard = completion( + model="anthropic.claude-v2", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is the main topic of this legal document?"}, + {"type": "guarded_text", "text": "This document contains sensitive legal information that should be moderated by guardrails."} + ] + } + ], + guardrailConfig={ + "guardrailIdentifier": "gr-abc123", + "guardrailVersion": "DRAFT" + } +) ``` @@ -993,7 +1024,20 @@ response = client.chat.completions.create(model="bedrock-claude-v1", messages = temperature=0.7 ) -print(response) +# For adding selective guardrail usage with guarded_text +response_guard = client.chat.completions.create(model="bedrock-claude-v1", messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is the main topic of this legal document?"}, + {"type": "guarded_text", "text": "This document contains sensitive legal information that should be moderated by guardrails."} + ] + } +], +temperature=0.7 +) + +print(response_guard) ``` diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index c86f08980cdf..f372e74b31f5 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -3076,53 +3076,6 @@ def _initial_message_setup( messages.append(DEFAULT_USER_CONTINUE_MESSAGE) return messages - @staticmethod - def _apply_guardrail_wrapping( - user_content: List[BedrockContentBlock], messages: List[dict], msg_i: int - ) -> List[BedrockContentBlock]: - """ - Apply guardrail wrapping to the last text block in user content if this is the last user message. - - Args: - user_content: List of content blocks for the current user message - messages: Full list of messages - msg_i: Current message index - - Returns: - Modified user_content with guardrail wrapping applied if needed - """ - # Check if this is the last user message by looking ahead - is_last_user_message = True - temp_msg_i = msg_i - while temp_msg_i < len(messages): - if messages[temp_msg_i]["role"] == "user": - is_last_user_message = False - break - temp_msg_i += 1 - - if not is_last_user_message: - return user_content - - # Wrap only the last text block in GuardrailConverseContent - wrapped_content = [] - text_blocks = [block for block in user_content if "text" in block] - - for content_block in user_content: - if "text" in content_block: - # Only wrap the last text block - if content_block == text_blocks[-1]: - guardrail_block = BedrockContentBlock( - guardrailConverseContent={"text": content_block["text"]} - ) - wrapped_content.append(guardrail_block) - else: - # Keep other text blocks as-is - wrapped_content.append(content_block) - else: - # Keep non-text content blocks as-is - wrapped_content.append(content_block) - - return wrapped_content @staticmethod async def _bedrock_converse_messages_pt_async( # noqa: PLR0915 @@ -3133,7 +3086,6 @@ async def _bedrock_converse_messages_pt_async( # noqa: PLR0915 assistant_continue_message: Optional[ Union[str, ChatCompletionAssistantMessage] ] = None, - guard_last_turn_only: bool = False, ) -> List[BedrockMessageBlock]: contents: List[BedrockMessageBlock] = [] msg_i = 0 @@ -3168,6 +3120,12 @@ async def _bedrock_converse_messages_pt_async( # noqa: PLR0915 if element["type"] == "text": _part = BedrockContentBlock(text=element["text"]) _parts.append(_part) + elif element["type"] == "guarded_text": + # Wrap guarded_text in guardrailConverseContent block + _part = BedrockContentBlock( + guardrailConverseContent={"text": element["text"]} + ) + _parts.append(_part) elif element["type"] == "image_url": format: Optional[str] = None if isinstance(element["image_url"], dict): @@ -3210,13 +3168,6 @@ async def _bedrock_converse_messages_pt_async( # noqa: PLR0915 msg_i += 1 if user_content: - # Apply guardrail wrapping if guard_last_turn_only is True and this is the last user message - if guard_last_turn_only: - user_content = ( - BedrockConverseMessagesProcessor._apply_guardrail_wrapping( - user_content=user_content, messages=messages, msg_i=msg_i - ) - ) if len(contents) > 0 and contents[-1]["role"] == "user": if ( @@ -3500,7 +3451,6 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 assistant_continue_message: Optional[ Union[str, ChatCompletionAssistantMessage] ] = None, - guard_last_turn_only: bool = False, ) -> List[BedrockMessageBlock]: """ Converts given messages from OpenAI format to Bedrock format @@ -3552,6 +3502,12 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 if element["type"] == "text": _part = BedrockContentBlock(text=element["text"]) _parts.append(_part) + elif element["type"] == "guarded_text": + # Wrap guarded_text in guardrailConverseContent block + _part = BedrockContentBlock( + guardrailConverseContent={"text": element["text"]} + ) + _parts.append(_part) elif element["type"] == "image_url": format: Optional[str] = None if isinstance(element["image_url"], dict): @@ -3595,13 +3551,6 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 msg_i += 1 if user_content: - # Apply guardrail wrapping if guard_last_turn_only is True and this is the last user message - if guard_last_turn_only: - user_content = ( - BedrockConverseMessagesProcessor._apply_guardrail_wrapping( - user_content=user_content, messages=messages, msg_i=msg_i - ) - ) if len(contents) > 0 and contents[-1]["role"] == "user": if ( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 0f83a9dbb3ad..88103a86cf31 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -631,7 +631,7 @@ def _handle_top_k_value(self, model: str, inference_params: dict) -> dict: return {} - def _transform_request_helper( # noqa: PLR0915 + def _transform_request_helper( self, model: str, system_content_blocks: List[SystemContentBlock], @@ -672,10 +672,6 @@ def _transform_request_helper( # noqa: PLR0915 ) inference_params.pop("json_mode", None) # used for handling json_schema - # Remove raw_blocks and guard_last_turn_only from inference_params as they are handled in transformation - inference_params.pop("raw_blocks", None) - inference_params.pop("guard_last_turn_only", None) - # keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params' additional_request_params = { k: v for k, v in inference_params.items() if k not in total_supported_params @@ -783,30 +779,14 @@ async def _async_transform_request( headers=headers, ) - # Check for raw_blocks parameter (advanced control) - raw_blocks = optional_params.pop("raw_blocks", None) - if raw_blocks is not None: - # Use raw blocks directly if provided - bedrock_messages: List[MessageBlock] = raw_blocks - else: - # Use standard transformation with guard_last_turn_only support - guard_last_turn_only = optional_params.pop("guard_last_turn_only", False) - - # Validate that we have messages when not using raw_blocks - if not messages: - raise litellm.BadRequestError( - message="Invalid Message: bedrock requires at least one non-system message when not using raw_blocks", - model=model, - llm_provider="bedrock_converse", - ) - - bedrock_messages = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + bedrock_messages = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( messages=messages, model=model, llm_provider="bedrock_converse", user_continue_message=litellm_params.pop("user_continue_message", None), - guard_last_turn_only=guard_last_turn_only, ) + ) data: RequestObject = {"messages": bedrock_messages, **_data} @@ -850,30 +830,12 @@ def _transform_request( ) ## TRANSFORMATION ## - # Check for raw_blocks parameter (advanced control) - raw_blocks = optional_params.pop("raw_blocks", None) - if raw_blocks is not None: - # Use raw blocks directly if provided - bedrock_messages: List[MessageBlock] = raw_blocks - else: - # Use standard transformation with guard_last_turn_only support - guard_last_turn_only = optional_params.pop("guard_last_turn_only", False) - - # Validate that we have messages when not using raw_blocks - if not messages: - raise litellm.BadRequestError( - message="Invalid Message: bedrock requires at least one non-system message when not using raw_blocks", - model=model, - llm_provider="bedrock_converse", - ) - - bedrock_messages = _bedrock_converse_messages_pt( - messages=messages, - model=model, - llm_provider="bedrock_converse", - user_continue_message=litellm_params.pop("user_continue_message", None), - guard_last_turn_only=guard_last_turn_only, - ) + bedrock_messages: List[MessageBlock] = _bedrock_converse_messages_pt( + messages=messages, + model=model, + llm_provider="bedrock_converse", + user_continue_message=litellm_params.pop("user_continue_message", None), + ) data: RequestObject = {"messages": bedrock_messages, **_data} diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 79e3c73dbc15..4adb751d9059 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -723,6 +723,7 @@ class GenericChatCompletionMessage(TypedDict, total=False): "input_audio", "audio_url", "document", + "guarded_text", "video_url", "file", ] # used for validating user messages. Prevent users from accidentally sending anthropic messages. diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 7597c3475de8..df003850f702 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -1593,66 +1593,31 @@ async def test_no_cache_control_no_cache_point(): # ============================================================================ -# GuardrailConverseContent Feature Tests +# Guarded Text Feature Tests # ============================================================================ -def test_guard_last_turn_only_wraps_last_user_message(): - """Test that guard_last_turn_only=True wraps only the last user message in GuardrailConverseContent.""" +def test_guarded_text_wraps_in_guardrail_converse_content(): + """Test that guarded_text content type gets wrapped in guardrailConverseContent blocks.""" from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_converse_messages_pt messages = [ - {"role": "user", "content": "Hello, how are you?"}, - {"role": "assistant", "content": "I'm doing well, thank you!"}, - {"role": "user", "content": "What's the weather like?"}, - {"role": "assistant", "content": "I don't have access to real-time weather data."}, - {"role": "user", "content": "Tell me about AI safety and potential risks"} - ] - - result = _bedrock_converse_messages_pt( - messages=messages, - model="us.amazon.nova-pro-v1:0", - llm_provider="bedrock_converse", - guard_last_turn_only=True - ) - - # Should have 5 messages - assert len(result) == 5 - - # First 4 messages should be normal (no GuardrailConverseContent) - for i in range(4): - content = result[i]["content"] - if result[i]["role"] == "user": - # User messages should have text content, not GuardrailConverseContent - assert "text" in content[0] - assert "guardrailConverseContent" not in content[0] - - # Last message (user) should have GuardrailConverseContent - last_message = result[-1] - assert last_message["role"] == "user" - last_content = last_message["content"] - assert len(last_content) == 1 - assert "guardrailConverseContent" in last_content[0] - assert last_content[0]["guardrailConverseContent"]["text"] == "Tell me about AI safety and potential risks" - - -def test_guard_last_turn_only_with_multiple_consecutive_user_messages(): - """Test that guard_last_turn_only=True wraps only the last user message when there are consecutive user messages.""" - from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_converse_messages_pt - - messages = [ - {"role": "user", "content": "First user message"}, - {"role": "user", "content": "Second user message"}, - {"role": "user", "content": "Third user message (should be guarded)"} + { + "role": "user", + "content": [ + {"type": "text", "text": "Regular text content"}, + {"type": "guarded_text", "text": "This should be guarded"}, + {"type": "text", "text": "More regular text"} + ] + } ] result = _bedrock_converse_messages_pt( messages=messages, model="us.amazon.nova-pro-v1:0", - llm_provider="bedrock_converse", - guard_last_turn_only=True + llm_provider="bedrock_converse" ) - # Should have 1 message (consecutive user messages are merged) + # Should have 1 message assert len(result) == 1 assert result[0]["role"] == "user" @@ -1660,65 +1625,33 @@ def test_guard_last_turn_only_with_multiple_consecutive_user_messages(): content = result[0]["content"] assert len(content) == 3 - # First two should be normal text + # First and third should be regular text assert "text" in content[0] - assert content[0]["text"] == "First user message" - assert "text" in content[1] - assert content[1]["text"] == "Second user message" - - # Last one should be GuardrailConverseContent - assert "guardrailConverseContent" in content[2] - assert content[2]["guardrailConverseContent"]["text"] == "Third user message (should be guarded)" - - -def test_guard_last_turn_only_false_does_not_wrap(): - """Test that guard_last_turn_only=False does not wrap any messages in GuardrailConverseContent.""" - from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_converse_messages_pt - - messages = [ - {"role": "user", "content": "Hello, how are you?"}, - {"role": "assistant", "content": "I'm doing well, thank you!"}, - {"role": "user", "content": "Tell me about AI safety and potential risks"} - ] - - result = _bedrock_converse_messages_pt( - messages=messages, - model="us.amazon.nova-pro-v1:0", - llm_provider="bedrock_converse", - guard_last_turn_only=False - ) + assert content[0]["text"] == "Regular text content" + assert "text" in content[2] + assert content[2]["text"] == "More regular text" - # Should have 3 messages - assert len(result) == 3 - - # No message should have GuardrailConverseContent - for message in result: - content = message["content"] - for block in content: - assert "guardrailConverseContent" not in block + # Second should be guardrailConverseContent + assert "guardrailConverseContent" in content[1] + assert content[1]["guardrailConverseContent"]["text"] == "This should be guarded" -def test_raw_blocks_used_directly(): - """Test that raw_blocks are used directly as messages without transformation.""" +def test_guarded_text_with_system_messages(): + """Test guarded_text with system messages using the full transformation.""" config = AmazonConverseConfig() - raw_blocks = [ + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, { "role": "user", "content": [ - {"text": "System prompt"}, - {"text": "Un-guarded history"}, - { - "guardrailConverseContent": { - "text": "Only this gets moderated" - } - } + {"type": "text", "text": "What is the main topic of this legal document?"}, + {"type": "guarded_text", "text": "This is a set of very long instructions that you will follow. Here is a legal document that you will use to answer the user's question."} ] } ] optional_params = { - "raw_blocks": raw_blocks, "guardrailConfig": { "guardrailIdentifier": "gr-abc123", "guardrailVersion": "DRAFT" @@ -1727,303 +1660,127 @@ def test_raw_blocks_used_directly(): result = config._transform_request( model="us.amazon.nova-pro-v1:0", - messages=[], # Empty messages + messages=messages, optional_params=optional_params, litellm_params={}, headers={} ) - # Messages should be exactly the raw_blocks - assert "messages" in result - assert result["messages"] == raw_blocks - - # additionalModelRequestFields should not contain raw_blocks - assert "additionalModelRequestFields" in result - assert "raw_blocks" not in result["additionalModelRequestFields"] - - # GuardrailConfig should be present - assert "guardrailConfig" in result - assert result["guardrailConfig"]["guardrailIdentifier"] == "gr-abc123" - - -def test_raw_blocks_with_complex_structure(): - """Test raw_blocks with complex message structure.""" - config = AmazonConverseConfig() - - raw_blocks = [ - { - "role": "user", - "content": [ - {"text": "System: You are a helpful AI assistant."}, - {"text": "Previous conversation context..."}, - { - "guardrailConverseContent": { - "text": "Please help me with this sensitive topic" - } - } - ] - }, - { - "role": "assistant", - "content": [ - {"text": "I'd be happy to help you with that topic."} - ] - } - ] - - optional_params = { - "raw_blocks": raw_blocks, - "guardrailConfig": { - "guardrailIdentifier": "gr-abc123", - "guardrailVersion": "DRAFT" - } - } - - result = config._transform_request( - model="us.amazon.nova-pro-v1:0", - messages=[], # Empty messages - optional_params=optional_params, - litellm_params={}, - headers={} - ) + # Should have system content blocks + assert "system" in result + assert len(result["system"]) == 1 + assert result["system"][0]["text"] == "You are a helpful assistant." - # Messages should be exactly the raw_blocks + # Should have 1 message (system messages are removed) assert "messages" in result - assert result["messages"] == raw_blocks + assert len(result["messages"]) == 1 - # Verify GuardrailConverseContent is preserved + # User message should have both regular text and guarded text user_message = result["messages"][0] assert user_message["role"] == "user" content = user_message["content"] - assert len(content) == 3 - assert "guardrailConverseContent" in content[2] - assert content[2]["guardrailConverseContent"]["text"] == "Please help me with this sensitive topic" - - -def test_empty_messages_with_guard_last_turn_only_raises_error(): - """Test that empty messages with guard_last_turn_only=True raises appropriate error.""" - config = AmazonConverseConfig() - - optional_params = { - "guardrailConfig": { - "guardrailIdentifier": "gr-abc123", - "guardrailVersion": "DRAFT" - }, - "guard_last_turn_only": True - } + assert len(content) == 2 - with pytest.raises(litellm.BadRequestError) as exc_info: - config._transform_request( - model="us.amazon.nova-pro-v1:0", - messages=[], # Empty messages - optional_params=optional_params, - litellm_params={}, - headers={} - ) + # First should be regular text + assert "text" in content[0] + assert content[0]["text"] == "What is the main topic of this legal document?" - assert "bedrock requires at least one non-system message when not using raw_blocks" in str(exc_info.value) + # Second should be guardrailConverseContent + assert "guardrailConverseContent" in content[1] + assert content[1]["guardrailConverseContent"]["text"] == "This is a set of very long instructions that you will follow. Here is a legal document that you will use to answer the user's question." -def test_empty_messages_with_raw_blocks_works(): - """Test that empty messages with raw_blocks works correctly.""" - config = AmazonConverseConfig() +def test_guarded_text_with_mixed_content_types(): + """Test guarded_text with mixed content types including images.""" + from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_converse_messages_pt - raw_blocks = [ + messages = [ { "role": "user", "content": [ - {"text": "Hello from raw_blocks!"} + {"type": "text", "text": "Look at this image"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,test"}}, + {"type": "guarded_text", "text": "This sensitive content should be guarded"} ] } ] - optional_params = { - "raw_blocks": raw_blocks, - "guardrailConfig": { - "guardrailIdentifier": "gr-abc123", - "guardrailVersion": "DRAFT" - } - } - - # Should not raise an error - result = config._transform_request( + result = _bedrock_converse_messages_pt( + messages=messages, model="us.amazon.nova-pro-v1:0", - messages=[], # Empty messages - optional_params=optional_params, - litellm_params={}, - headers={} + llm_provider="bedrock_converse" ) - assert "messages" in result - assert result["messages"] == raw_blocks - - -def test_guard_last_turn_only_with_system_messages(): - """Test guard_last_turn_only with system messages using the full transformation.""" - config = AmazonConverseConfig() - - messages = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"}, - {"role": "assistant", "content": "Hi there!"}, - {"role": "user", "content": "Tell me about AI safety"} - ] - - optional_params = { - "guardrailConfig": { - "guardrailIdentifier": "gr-abc123", - "guardrailVersion": "DRAFT" - }, - "guard_last_turn_only": True - } + # Should have 1 message + assert len(result) == 1 + assert result[0]["role"] == "user" - result = config._transform_request( - model="us.amazon.nova-pro-v1:0", - messages=messages, - optional_params=optional_params, - litellm_params={}, - headers={} - ) + # Should have 3 content blocks + content = result[0]["content"] + assert len(content) == 3 - # Should have 3 messages (system message is extracted to system blocks) - assert "messages" in result - assert len(result["messages"]) == 3 + # First should be regular text + assert "text" in content[0] + assert content[0]["text"] == "Look at this image" - # Last user message should have GuardrailConverseContent - last_message = result["messages"][-1] - assert last_message["role"] == "user" - content = last_message["content"] - assert len(content) == 1 - assert "guardrailConverseContent" in content[0] - assert content[0]["guardrailConverseContent"]["text"] == "Tell me about AI safety" + # Second should be image + assert "image" in content[1] - # System content should be in system blocks - assert "system" in result - assert len(result["system"]) == 1 - assert result["system"][0]["text"] == "You are a helpful assistant." + # Third should be guardrailConverseContent + assert "guardrailConverseContent" in content[2] + assert content[2]["guardrailConverseContent"]["text"] == "This sensitive content should be guarded" -def test_guard_last_turn_only_with_mixed_content_types(): - """Test guard_last_turn_only with user messages containing mixed content types.""" - from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_converse_messages_pt +@pytest.mark.asyncio +async def test_async_guarded_text(): + """Test async version of guarded_text processing.""" + from litellm.litellm_core_utils.prompt_templates.factory import BedrockConverseMessagesProcessor messages = [ - {"role": "user", "content": "Hello!"}, - {"role": "assistant", "content": "Hi there!"}, { - "role": "user", + "role": "user", "content": [ - {"type": "text", "text": "Regular text"}, - {"type": "image_url", "image_url": {"url": "data:image/png;base64,test"}}, - {"type": "text", "text": "This should be guarded"} + {"type": "text", "text": "Hello"}, + {"type": "guarded_text", "text": "This should be guarded"} ] } ] - result = _bedrock_converse_messages_pt( + result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( messages=messages, model="us.amazon.nova-pro-v1:0", - llm_provider="bedrock_converse", - guard_last_turn_only=True + llm_provider="bedrock_converse" ) - # Should have 3 messages - assert len(result) == 3 + # Should have 1 message + assert len(result) == 1 + assert result[0]["role"] == "user" - # Last user message should have GuardrailConverseContent for text content only - last_message = result[-1] - assert last_message["role"] == "user" - content = last_message["content"] - assert len(content) == 3 + # Should have 2 content blocks + content = result[0]["content"] + assert len(content) == 2 - # First two should be normal content + # First should be regular text assert "text" in content[0] - assert content[0]["text"] == "Regular text" - assert "image" in content[1] # image_url gets transformed to image + assert content[0]["text"] == "Hello" - # Last one should be GuardrailConverseContent - assert "guardrailConverseContent" in content[2] - assert content[2]["guardrailConverseContent"]["text"] == "This should be guarded" + # Second should be guardrailConverseContent + assert "guardrailConverseContent" in content[1] + assert content[1]["guardrailConverseContent"]["text"] == "This should be guarded" -def test_async_guard_last_turn_only(): - """Test async version of guard_last_turn_only.""" - from litellm.litellm_core_utils.prompt_templates.factory import BedrockConverseMessagesProcessor +def test_guarded_text_with_tool_calls(): + """Test guarded_text with tool calls in the conversation.""" + from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_converse_messages_pt messages = [ - {"role": "user", "content": "Hello!"}, - {"role": "assistant", "content": "Hi there!"}, - {"role": "user", "content": "Tell me about AI safety"} - ] - - async def run_test(): - result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="us.amazon.nova-pro-v1:0", - llm_provider="bedrock_converse", - guard_last_turn_only=True - ) - - # Should have 3 messages - assert len(result) == 3 - - # Last user message should have GuardrailConverseContent - last_message = result[-1] - assert last_message["role"] == "user" - content = last_message["content"] - assert len(content) == 1 - assert "guardrailConverseContent" in content[0] - assert content[0]["guardrailConverseContent"]["text"] == "Tell me about AI safety" - - # Run the async test - import asyncio - asyncio.run(run_test()) - - -def test_async_raw_blocks(): - """Test async version of raw_blocks.""" - config = AmazonConverseConfig() - - raw_blocks = [ { "role": "user", "content": [ - {"text": "Hello from async raw_blocks!"} + {"type": "text", "text": "What's the weather?"}, + {"type": "guarded_text", "text": "Please be careful with sensitive information"} ] - } - ] - - optional_params = { - "raw_blocks": raw_blocks, - "guardrailConfig": { - "guardrailIdentifier": "gr-abc123", - "guardrailVersion": "DRAFT" - } - } - - async def run_test(): - result = await config._async_transform_request( - model="us.amazon.nova-pro-v1:0", - messages=[], # Empty messages - optional_params=optional_params, - litellm_params={}, - headers={} - ) - - assert "messages" in result - assert result["messages"] == raw_blocks - - # Run the async test - import asyncio - asyncio.run(run_test()) - - -def test_guard_last_turn_only_with_tool_calls(): - """Test guard_last_turn_only with tool calls in the conversation.""" - from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_converse_messages_pt - - messages = [ - {"role": "user", "content": "What's the weather?"}, + }, { "role": "assistant", "content": None, @@ -2039,81 +1796,54 @@ def test_guard_last_turn_only_with_tool_calls(): "role": "tool", "tool_call_id": "call_123", "content": "It's sunny and 25°C" - }, - {"role": "user", "content": "Tell me about AI safety"} + } ] result = _bedrock_converse_messages_pt( messages=messages, model="us.amazon.nova-pro-v1:0", - llm_provider="bedrock_converse", - guard_last_turn_only=True + llm_provider="bedrock_converse" ) - # Should have 3 messages (tool message is merged with user message) + # Should have 3 messages assert len(result) == 3 - # Last user message should have GuardrailConverseContent - last_message = result[-1] - assert last_message["role"] == "user" - content = last_message["content"] - # Should have tool result and guardrail content + # First message (user) should have both text and guarded_text + user_message = result[0] + assert user_message["role"] == "user" + content = user_message["content"] assert len(content) == 2 - assert "toolResult" in content[0] - assert "guardrailConverseContent" in content[1] - assert content[1]["guardrailConverseContent"]["text"] == "Tell me about AI safety" - - -def test_guardrail_config_preserved_in_request(): - """Test that guardrailConfig is properly preserved in the request.""" - config = AmazonConverseConfig() - - messages = [ - {"role": "user", "content": "Tell me about AI safety"} - ] - - optional_params = { - "guardrailConfig": { - "guardrailIdentifier": "gr-abc123", - "guardrailVersion": "DRAFT" - }, - "guard_last_turn_only": True - } - result = config._transform_request( - model="us.amazon.nova-pro-v1:0", - messages=messages, - optional_params=optional_params, - litellm_params={}, - headers={} - ) + # First should be regular text + assert "text" in content[0] + assert content[0]["text"] == "What's the weather?" - # GuardrailConfig should be present at top level - assert "guardrailConfig" in result - assert result["guardrailConfig"]["guardrailIdentifier"] == "gr-abc123" - assert result["guardrailConfig"]["guardrailVersion"] == "DRAFT" + # Second should be guardrailConverseContent + assert "guardrailConverseContent" in content[1] + assert content[1]["guardrailConverseContent"]["text"] == "Please be careful with sensitive information" - # GuardrailConfig should also be in inferenceConfig - assert "inferenceConfig" in result - assert "guardrailConfig" in result["inferenceConfig"] - assert result["inferenceConfig"]["guardrailConfig"]["guardrailIdentifier"] == "gr-abc123" + # Other messages should not have guardrailConverseContent + for i in range(1, 3): + content = result[i]["content"] + for block in content: + assert "guardrailConverseContent" not in block -def test_raw_blocks_guardrail_config_preserved(): - """Test that guardrailConfig is preserved when using raw_blocks.""" +def test_guarded_text_guardrail_config_preserved(): + """Test that guardrailConfig is preserved when using guarded_text.""" config = AmazonConverseConfig() - raw_blocks = [ + messages = [ { "role": "user", "content": [ - {"text": "Hello from raw_blocks!"} + {"type": "text", "text": "Hello"}, + {"type": "guarded_text", "text": "This should be guarded"} ] } ] optional_params = { - "raw_blocks": raw_blocks, "guardrailConfig": { "guardrailIdentifier": "gr-abc123", "guardrailVersion": "DRAFT" @@ -2122,7 +1852,7 @@ def test_raw_blocks_guardrail_config_preserved(): result = config._transform_request( model="us.amazon.nova-pro-v1:0", - messages=[], + messages=messages, optional_params=optional_params, litellm_params={}, headers={} @@ -2135,4 +1865,6 @@ def test_raw_blocks_guardrail_config_preserved(): # GuardrailConfig should also be in inferenceConfig assert "inferenceConfig" in result assert "guardrailConfig" in result["inferenceConfig"] - assert result["inferenceConfig"]["guardrailConfig"]["guardrailIdentifier"] == "gr-abc123" \ No newline at end of file + assert result["inferenceConfig"]["guardrailConfig"]["guardrailIdentifier"] == "gr-abc123" + + From 7b66c55f8ccd5a2c13f63351562e68bfcf54e435 Mon Sep 17 00:00:00 2001 From: Sameerlite Date: Tue, 16 Sep 2025 23:47:21 +0530 Subject: [PATCH 4/6] Add guarded_text content type --- Dockerfile | 2 +- docker-compose.yml | 17 +++++++---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index b319edbbae7d..77dc049c7981 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,4 +81,4 @@ COPY docker/supervisord.conf /etc/supervisord.conf ENTRYPOINT ["docker/prod_entrypoint.sh"] # Append "--detailed_debug" to the end of CMD to view detailed debug logs -CMD ["--port", "4000", "--detailed_debug"] +CMD ["--port", "4000"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index df2088c93926..366fbe51b5ac 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,21 +5,18 @@ services: args: target: runtime image: ghcr.io/berriai/litellm:main-stable - ######################################## - # Uncomment these lines to start proxy with a config.yaml file ## - volumes: - - ./config.yaml:/app/config.yaml <<- this is missing in the docker-compose file currently - command: - - "--config=/app/config.yaml" - ############################################# + ######################################### + ## Uncomment these lines to start proxy with a config.yaml file ## + # volumes: + # - ./config.yaml:/app/config.yaml <<- this is missing in the docker-compose file currently + # command: + # - "--config=/app/config.yaml" + ############################################## ports: - "4000:4000" # Map the container port to the host, change the host port if necessary environment: DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm" STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI - LITELLM_LOG: "DEBUG" - LITELLM_LOG_LEVEL: "DEBUG" - PYTHONUNBUFFERED: "1" env_file: - .env # Load local .env file depends_on: From cb664bf2d6f1ab4d3d3ec71bfeaac4e809218955 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 16 Sep 2025 23:49:37 +0530 Subject: [PATCH 5/6] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 77dc049c7981..3dfaeae03ecd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,4 +81,4 @@ COPY docker/supervisord.conf /etc/supervisord.conf ENTRYPOINT ["docker/prod_entrypoint.sh"] # Append "--detailed_debug" to the end of CMD to view detailed debug logs -CMD ["--port", "4000"] \ No newline at end of file +CMD ["--port", "4000"] From f8f671494e53965f132a56f86ebfbfdd602914bf Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 16 Sep 2025 23:50:01 +0530 Subject: [PATCH 6/6] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3dfaeae03ecd..addc109e10c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,4 +81,4 @@ COPY docker/supervisord.conf /etc/supervisord.conf ENTRYPOINT ["docker/prod_entrypoint.sh"] # Append "--detailed_debug" to the end of CMD to view detailed debug logs -CMD ["--port", "4000"] +CMD ["--port", "4000"]