From a271ceabe2b603485c6d17c51f7768640ca663e3 Mon Sep 17 00:00:00 2001 From: Iz Beltagy Date: Sun, 8 Oct 2023 09:13:52 -0700 Subject: [PATCH] fix config --- olmo/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/olmo/model.py b/olmo/model.py index 48cb77fb3..266dc4a7e 100644 --- a/olmo/model.py +++ b/olmo/model.py @@ -340,7 +340,7 @@ def __init__(self, layer_id: int, config: ModelConfig): size=config.d_model // config.n_heads if config.multi_query_attention else None, elementwise_affine=config.attention_layer_norm_with_affine, ) - self.q_norm = LayerNormBase.build(config, elementwise_affine=True) + self.q_norm = LayerNormBase.build(config, elementwise_affine=config.attention_layer_norm_with_affine) # Activation function. self.act = Activation.build(config)