fixes #1

near-zero initialization
hmohebbi · Feb 25, 2021 · e38e00d · e38e00d
1 parent 71045a2
commit e38e00d
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/modeling_tf_adapter_bert.py b/modeling_tf_adapter_bert.py
@@ -20,13 +20,13 @@ def __init__(self, input_size, bottleneck_size, non_linearity, *inputs, **kwargs
 
     self.down_project = tf.keras.layers.Dense(
                         bottleneck_size,
-                        kernel_initializer=tf.keras.initializers.TruncatedNormal(stddev=0.02),
+                        kernel_initializer=tf.keras.initializers.TruncatedNormal(stddev=1e-3),
                         bias_initializer="zeros",
                         name="feedforward_downproject")
 
     self.up_project = tf.keras.layers.Dense(
                         input_size,
-                        kernel_initializer=tf.keras.initializers.TruncatedNormal(stddev=0.02),
+                        kernel_initializer=tf.keras.initializers.TruncatedNormal(stddev=1e-3),
                         bias_initializer="zeros",
                         name="feedforward_upproject")