set a default value for the transformer output softclamp value, and a…

…llow for boolean toggle
lucidrains · Jun 30, 2024 · 59cee27 · 59cee27
1 parent 9078611
commit 59cee27
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 3 deletions.
diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 setup(
   name = 'x-transformers',
   packages = find_packages(exclude=['examples']),
-  version = '1.31.5',
+  version = '1.31.6',
   license='MIT',
   description = 'X-Transformers - Pytorch',
   author = 'Phil Wang',

diff --git a/x_transformers/x_transformers.py b/x_transformers/x_transformers.py
@@ -1267,7 +1267,8 @@ def __init__(
         scale_residual_constant = 1.,
         shift_tokens = 0,
         sandwich_norm = False,
-        softclamp_output_value: float | None = None,
+        softclamp_output = False,
+        softclamp_output_value = 50.,
         resi_dual = False,
         resi_dual_scale = 1.,
         zero_init_branch_output = False,
@@ -1484,6 +1485,7 @@ def __init__(
         # optional soft clamping just before the final norm
         # used in gemma 2
 
+        self.softclamp_output = softclamp_output
         self.softclamp_output_value = softclamp_output_value
 
         # whether it has post norm
@@ -1717,7 +1719,7 @@ def forward(
         if return_hiddens:
             layer_hiddens.append(x)
 
-        if exists(self.softclamp_output_value):
+        if self.softclamp_output:
             x = softclamp(x, self.softclamp_output_value)
 
         final_norm = self.final_norm