Move copy down into guided decoding case

Signed-off-by: mgoin <[email protected]>
vllm-project · Dec 3, 2024 · 975e040 · 975e040
1 parent 9f97093
commit 975e040
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 5 deletions.
diff --git a/vllm/engine/async_llm_engine.py b/vllm/engine/async_llm_engine.py
@@ -1,4 +1,5 @@
 import asyncio
+import copy
 import time
 import weakref
 from functools import partial
@@ -533,9 +534,14 @@ async def build_guided_decoding_logits_processor_async(
     those fields and adds the constructed logits processors to the
     logits_processors field. Modifies sampling params in-place and returns
     the modified sampling params."""
-    if (guided_decoding := sampling_params.guided_decoding) is None:
+    if sampling_params.guided_decoding is None:
         return sampling_params
 
+    # Defensively copy sampling params since guided decoding logits
+    # processors can have different state for each request
+    sampling_params = copy.copy(sampling_params)
+    guided_decoding = sampling_params.guided_decoding
+
     logger.debug("Building guided decoding logits processor. "
                  "Params: %s", guided_decoding)
 

diff --git a/vllm/engine/llm_engine.py b/vllm/engine/llm_engine.py
@@ -1,3 +1,4 @@
+import copy
 import time
 from collections import Counter as collectionsCounter
 from collections import deque
@@ -2035,7 +2036,11 @@ def _build_logits_processors(
 
         logits_processors = []
 
-        if (guided_decoding := sampling_params.guided_decoding) is not None:
+        if sampling_params.guided_decoding is not None:
+            # Defensively copy sampling params since guided decoding logits
+            # processors can have different state for each request
+            sampling_params = copy.copy(sampling_params)
+            guided_decoding = sampling_params.guided_decoding
 
             logger.debug(
                 "Building guided decoding logits processor in "

diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py
@@ -1,4 +1,3 @@
-import copy
 import itertools
 import json
 import warnings
@@ -1038,8 +1037,7 @@ def _validate_and_add_requests(
         for i, prompt in enumerate(prompts):
             self._add_request(
                 prompt,
-                params[i]
-                if isinstance(params, Sequence) else copy.copy(params),
+                params[i] if isinstance(params, Sequence) else params,
                 lora_request=lora_request[i] if isinstance(
                     lora_request, Sequence) else lora_request,
                 prompt_adapter_request=prompt_adapter_request,