From beb16b2c810a87b28e7b8a7aa29d26f842f654b9 Mon Sep 17 00:00:00 2001 From: Travis Johnson Date: Tue, 10 Dec 2024 03:27:11 -0700 Subject: [PATCH] [Bugfix] Handle <|tool_call|> token in granite tool parser (#11039) Signed-off-by: Travis Johnson --- vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py index b5854ca39ab47..00917c866e496 100644 --- a/vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py @@ -35,11 +35,13 @@ class GraniteToolParser(ToolParser): def __init__(self, tokenizer: AnyTokenizer): super().__init__(tokenizer) + self.bot_token = "<|tool_call|>" def extract_tool_calls( self, model_output: str, request: ChatCompletionRequest) -> ExtractedToolCallInformation: - stripped = model_output.strip() + # remove whitespace and the BOT token if it exists + stripped = model_output.strip().removeprefix(self.bot_token).lstrip() if not stripped or stripped[0] != '[': return ExtractedToolCallInformation(tools_called=False, tool_calls=[],