From 80be13468065a720fb5ca92cb6d6dbcf5a204913 Mon Sep 17 00:00:00 2001 From: lucidrains Date: Sat, 6 Jul 2024 07:40:41 -0700 Subject: [PATCH] go with Gemma2 softclamp values --- setup.py | 2 +- x_transformers/attend.py | 2 +- x_transformers/x_transformers.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 9f56293b..d1a65e79 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name = 'x-transformers', packages = find_packages(exclude=['examples']), - version = '1.31.9', + version = '1.31.10', license='MIT', description = 'X-Transformers - Pytorch', author = 'Phil Wang', diff --git a/x_transformers/attend.py b/x_transformers/attend.py index cd30ae24..053b7ccf 100644 --- a/x_transformers/attend.py +++ b/x_transformers/attend.py @@ -82,7 +82,7 @@ def __init__( qk_norm = False, flash = False, softclamp_logits = False, - logit_softclamp_value = 30., + logit_softclamp_value = 50., add_zero_kv = False, cope = None, onnxable = False, diff --git a/x_transformers/x_transformers.py b/x_transformers/x_transformers.py index 667d3b14..5200dffa 100644 --- a/x_transformers/x_transformers.py +++ b/x_transformers/x_transformers.py @@ -1272,7 +1272,7 @@ def __init__( shift_tokens = 0, sandwich_norm = False, softclamp_output = False, - softclamp_output_value = 50., + softclamp_output_value = 30., resi_dual = False, resi_dual_scale = 1., zero_init_branch_output = False,