From 1325872ec8c97d797c18f490bdb6be7f4def5aa8 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Sat, 19 Oct 2024 04:21:01 +0100 Subject: [PATCH] [Frontend] Avoid creating guided decoding LogitsProcessor unnecessarily (#9521) --- vllm/sampling_params.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 4f2ae75e65f3a..9993cec13d649 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -49,14 +49,17 @@ class GuidedDecodingParams: @staticmethod def from_optional( - json: Optional[Union[Dict, BaseModel, str]], + json: Optional[Union[Dict, BaseModel, str]] = None, regex: Optional[str] = None, choice: Optional[List[str]] = None, grammar: Optional[str] = None, json_object: Optional[bool] = None, backend: Optional[str] = None, whitespace_pattern: Optional[str] = None, - ) -> "GuidedDecodingParams": + ) -> Optional["GuidedDecodingParams"]: + if all(arg is None + for arg in (json, regex, choice, grammar, json_object)): + return None # Extract json schemas from pydantic models if isinstance(json, (BaseModel, type(BaseModel))): json = json.model_json_schema()