gemmav2 attention softclamp value

lucidrains · Jul 19, 2024 · 02b0190 · 02b0190
1 parent 89270f9
commit 02b0190
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 setup(
   name = 'x-transformers',
   packages = find_packages(exclude=['examples']),
-  version = '1.31.12',
+  version = '1.31.14',
   license='MIT',
   description = 'X-Transformers - Pytorch',
   author = 'Phil Wang',

diff --git a/x_transformers/x_transformers.py b/x_transformers/x_transformers.py
@@ -898,7 +898,7 @@ def __init__(
         cope_soft_onehot_pos = False,
         cope_talking_heads = False,
         softclamp_logits = False,
-        logit_softclamp_value = 30.,
+        logit_softclamp_value = 50.,
         onnxable = False
     ):
         super().__init__()