enforce eager mode with bnb quantization temporarily (vllm-project#6846)

KuntaiDu · Jul 27, 2024 · 7c1b18a · 7c1b18a
1 parent b3af188
commit 7c1b18a
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/vllm/config.py b/vllm/config.py
@@ -282,6 +282,10 @@ def verify_with_parallel_config(
             raise ValueError(
                 "BitAndBytes quantization with TP or PP is not supported yet.")
 
+        if self.quantization == "bitsandbytes" and self.enforce_eager is False:
+            raise ValueError(
+                "BitAndBytes with enforce_eager = False is not supported yet.")
+
     def get_hf_config_sliding_window(self) -> Optional[int]:
         """Get the sliding window size, or None if disabled."""