From 921ea3c4d0b443e1d8f9e4b73763e6279b189bf4 Mon Sep 17 00:00:00 2001 From: Erin George Date: Fri, 10 Feb 2023 11:10:12 -0800 Subject: [PATCH] Correct gradient update for bias terms Adds gradient clipping and step size for bias update terms. --- src/glove.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/glove.c b/src/glove.c index 548d0b1..d0b7a6c 100644 --- a/src/glove.c +++ b/src/glove.c @@ -224,8 +224,8 @@ void *glove_thread(void *vid) { } // updates for bias terms - W[vector_size + l1] -= check_nan(fdiff / sqrt(gradsq[vector_size + l1])); - W[vector_size + l2] -= check_nan(fdiff / sqrt(gradsq[vector_size + l2])); + W[vector_size + l1] -= check_nan(fmin(fmax(fdiff, -grad_clip_value), grad_clip_value) / sqrt(gradsq[vector_size + l1])) * eta; + W[vector_size + l2] -= check_nan(fmin(fmax(fdiff, -grad_clip_value), grad_clip_value) / sqrt(gradsq[vector_size + l2])) * eta; fdiff *= fdiff; gradsq[vector_size + l1] += fdiff; gradsq[vector_size + l2] += fdiff;