Update openmoe_34b_config.json

Orion-Zheng · Jan 3, 2024 · 50c885f · 50c885f
1 parent 616611f
commit 50c885f
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/examples/language/openmoe/model/openmoe_34b_config.json b/examples/language/openmoe/model/openmoe_34b_config.json
@@ -27,7 +27,7 @@
   "max_position_embeddings": 2048,
   "mlp_gated": true,
   "model_type": "llama",
-  "moe_layer_interval": 3,
+  "moe_layer_interval": 4,
   "num_attention_heads": 24,
   "num_experts": 32,
   "num_hidden_layers": 32,