From 170e1e4e976a21610443d97bf0b2031edc4cd81b Mon Sep 17 00:00:00 2001 From: wenxindongwork Date: Tue, 10 Sep 2024 13:44:27 -0700 Subject: [PATCH] remove default attention type --- MaxText/configs/models/gemma2-27b.yml | 1 - MaxText/configs/models/gemma2-2b.yml | 1 - MaxText/configs/models/gemma2-9b.yml | 1 - 3 files changed, 3 deletions(-) diff --git a/MaxText/configs/models/gemma2-27b.yml b/MaxText/configs/models/gemma2-27b.yml index 90b9436a9..84a8fb98b 100644 --- a/MaxText/configs/models/gemma2-27b.yml +++ b/MaxText/configs/models/gemma2-27b.yml @@ -25,7 +25,6 @@ vocab_size: 256128 decoder_block: "gemma2" normalization_layer_epsilon: 1.e-06 logits_via_embedding: True -attention: "dot_product" final_logits_soft_cap: 30.0 attn_logits_soft_cap: 50.0 sliding_window_size: 4096 diff --git a/MaxText/configs/models/gemma2-2b.yml b/MaxText/configs/models/gemma2-2b.yml index f4a49819a..8647196b2 100644 --- a/MaxText/configs/models/gemma2-2b.yml +++ b/MaxText/configs/models/gemma2-2b.yml @@ -25,7 +25,6 @@ vocab_size: 256128 decoder_block: "gemma2" normalization_layer_epsilon: 1.e-06 logits_via_embedding: True -attention: "dot_product" final_logits_soft_cap: 30.0 attn_logits_soft_cap: 50.0 sliding_window_size: 4096 diff --git a/MaxText/configs/models/gemma2-9b.yml b/MaxText/configs/models/gemma2-9b.yml index 24bda4783..d54352b95 100644 --- a/MaxText/configs/models/gemma2-9b.yml +++ b/MaxText/configs/models/gemma2-9b.yml @@ -25,7 +25,6 @@ vocab_size: 256128 decoder_block: "gemma2" normalization_layer_epsilon: 1.e-06 logits_via_embedding: True -attention: "dot_product" final_logits_soft_cap: 30.0 attn_logits_soft_cap: 50.0 sliding_window_size: 4096