From 9a99273b482a3e90431069f37858d60827983e2f Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Tue, 26 Nov 2024 13:44:01 -0500 Subject: [PATCH] [Bugfix] Fix using `-O[0,3]` with LLM entrypoint (#10677) Signed-off-by: mgoin --- vllm/engine/arg_utils.py | 5 ++++- vllm/entrypoints/llm.py | 10 ++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 60ad5ee54a2f2..90b4798f17a13 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -206,7 +206,10 @@ def __post_init__(self): # support `EngineArgs(compilation_config={...})` # without having to manually construct a # CompilationConfig object - if isinstance(self.compilation_config, (int, dict)): + if isinstance(self.compilation_config, (int)): + self.compilation_config = CompilationConfig.from_cli( + str(self.compilation_config)) + elif isinstance(self.compilation_config, (dict)): self.compilation_config = CompilationConfig.from_cli( json.dumps(self.compilation_config)) diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index e07f4c04abd84..1551a9a998160 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -185,8 +185,14 @@ def __init__( kwargs["disable_log_stats"] = True if compilation_config is not None: - compilation_config_instance = CompilationConfig.from_cli( - json.dumps(compilation_config)) + if isinstance(compilation_config, (int)): + compilation_config_instance = CompilationConfig.from_cli( + str(compilation_config)) + elif isinstance(compilation_config, (dict)): + compilation_config_instance = CompilationConfig.from_cli( + json.dumps(compilation_config)) + else: + compilation_config_instance = compilation_config else: compilation_config_instance = None