Skip to content

Commit

Permalink
Fix mamba logic
Browse files Browse the repository at this point in the history
Signed-off-by: mzusman <[email protected]>
  • Loading branch information
mzusman committed Dec 9, 2024
1 parent 3bc3823 commit 0f70131
Showing 1 changed file with 5 additions and 2 deletions.
7 changes: 5 additions & 2 deletions vllm/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -696,9 +696,12 @@ def get_num_layers_by_block_type(
# This function relies on 'layers_block_type' in hf_config,
# for hybrid/attention-free models w/o this attribute,
# we will need to have workarounds like so
if self.is_attention_free and block_type == LayerBlockType.attention:
attn_block_type = block_type == LayerBlockType.attention
is_full_attn_model = not self.is_hybrid and not self.is_attention_free

if self.is_attention_free and attn_block_type:
return 0
if not self.is_hybrid and block_type != LayerBlockType.attention:
if is_full_attn_model and attn_block_type:
return 0

start, end = self.get_layers_start_end_indices(parallel_config)
Expand Down

0 comments on commit 0f70131

Please sign in to comment.