fix moe mistral num_hidden_layers error

InternLM · Jul 30, 2024 · 8061975 · 8061975
1 parent e58b9bb
commit 8061975
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/huggingface_model/DiscoResearch/mixtral_7b_8expert/configuration_moe_mistral.py b/huggingface_model/DiscoResearch/mixtral_7b_8expert/configuration_moe_mistral.py
@@ -106,7 +106,7 @@ def __init__(
         vocab_size=32000,
         hidden_size=4096,
         intermediate_size=14336,
-        num_hidden_layers=4,
+        num_hidden_layers=32,
         num_attention_heads=32,
         num_key_value_heads=8,
         hidden_act="silu",