fix sliding window

vllm-project · Dec 20, 2024 · a0cf8b5 · a0cf8b5
1 parent e2fa044
commit a0cf8b5
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 435 deletions.
diff --git a/vllm/model_executor/models/commandr.py b/vllm/model_executor/models/commandr.py
@@ -183,14 +183,15 @@ def __init__(
 
         self.sliding_window = (sliding_window
                                if layer_has_sliding_window else None)
+        cache_config = copy.deepcopy(cache_config)
+        cache_config.sliding_window = self.sliding_window
 
         self.attn = Attention(self.num_heads,
                               self.head_dim,
                               self.scaling,
                               num_kv_heads=self.num_kv_heads,
                               cache_config=cache_config,
                               quant_config=quant_config,
-                              per_layer_sliding_window=self.sliding_window,
                               prefix=f"{prefix}.attn")
         if self.use_qk_norm:
             self.q_norm = LayerNorm(param_shape=(self.num_heads,