From 756acd0d13303da080439046a9d5a263d92f7c38 Mon Sep 17 00:00:00 2001 From: ASEM000 Date: Sun, 6 Aug 2023 09:47:14 +0300 Subject: [PATCH] minor --- serket/nn/recurrent.py | 12 ++++++------ tests/test_rnn.py | 22 ++++++++++++---------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/serket/nn/recurrent.py b/serket/nn/recurrent.py index 366b43a..2356b01 100644 --- a/serket/nn/recurrent.py +++ b/serket/nn/recurrent.py @@ -170,8 +170,8 @@ def __init__( dtype=dtype, ) - self.ih2h_weight = jnp.concatenate([i2h.weight, h2h.weight], axis=0) - self.ih2h_bias = i2h.bias + self.in_hidden_to_hidden_weight = jnp.concatenate([i2h.weight, h2h.weight]) + self.in_hidden_to_hidden_bias = i2h.bias @ft.partial(maybe_lazy_call, is_lazy=is_lazy_call, updates=updates) @ft.partial(validate_spatial_ndim, attribute_name="spatial_ndim") @@ -181,7 +181,7 @@ def __call__(self, x: jax.Array, state: SimpleRNNState, **k) -> SimpleRNNState: raise TypeError(f"Expected {state=} to be an instance of `SimpleRNNState`") ih = jnp.concatenate([x, state.hidden_state], axis=-1) - h = ih @ self.ih2h_weight + self.ih2h_bias + h = ih @ self.in_hidden_to_hidden_weight + self.in_hidden_to_hidden_bias return SimpleRNNState(self.act_func(h)) @property @@ -368,8 +368,8 @@ def __init__( dtype=dtype, ) - self.ih2h_weight = jnp.concatenate([i2h.weight, h2h.weight], axis=0) - self.ih2h_bias = i2h.bias + self.in_hidden_to_hidden_weight = jnp.concatenate([i2h.weight, h2h.weight]) + self.in_hidden_to_hidden_bias = i2h.bias @ft.partial(maybe_lazy_call, is_lazy=is_lazy_call, updates=updates) @ft.partial(validate_spatial_ndim, attribute_name="spatial_ndim") @@ -380,7 +380,7 @@ def __call__(self, x: jax.Array, state: LSTMState, **k) -> LSTMState: h, c = state.hidden_state, state.cell_state ih = jnp.concatenate([x, h], axis=-1) - h = ih @ self.ih2h_weight + self.ih2h_bias + h = ih @ self.in_hidden_to_hidden_weight + self.in_hidden_to_hidden_bias i, f, g, o = jnp.split(h, 4, axis=-1) i = self.recurrent_act_func(i) f = self.recurrent_act_func(f) diff --git a/tests/test_rnn.py b/tests/test_rnn.py index e6af153..313341d 100644 --- a/tests/test_rnn.py +++ b/tests/test_rnn.py @@ -115,7 +115,7 @@ def test_vanilla_rnn(): ) w_combined = jnp.concatenate([w_in_to_hidden, w_hidden_to_hidden], axis=0) - cell = cell.at["ih2h_weight"].set(w_combined) + cell = cell.at["in_hidden_to_hidden_weight"].set(w_combined) sk_layer = ScanRNN(cell) y = jnp.array([0.9637042, -0.8282256, 0.7314449]) npt.assert_allclose(sk_layer(x), y) @@ -229,8 +229,8 @@ def test_lstm(): recurrent_weight_init="glorot_uniform", ) w_combined = jnp.concatenate([w_in_to_hidden, w_hidden_to_hidden], axis=0) - cell = cell.at["ih2h_weight"].set(w_combined) - cell = cell.at["ih2h_bias"].set(b_hidden_to_hidden) + cell = cell.at["in_hidden_to_hidden_weight"].set(w_combined) + cell = cell.at["in_hidden_to_hidden_bias"].set(b_hidden_to_hidden) sk_layer = ScanRNN(cell, return_sequences=False) @@ -329,8 +329,8 @@ def test_lstm(): w_combined = jnp.concatenate([w_in_to_hidden, w_hidden_to_hidden], axis=0) - cell = cell.at["ih2h_weight"].set(w_combined) - cell = cell.at["ih2h_bias"].set(b_hidden_to_hidden) + cell = cell.at["in_hidden_to_hidden_weight"].set(w_combined) + cell = cell.at["in_hidden_to_hidden_bias"].set(b_hidden_to_hidden) sk_layer = ScanRNN(cell, return_sequences=True) @@ -751,14 +751,16 @@ def test_bilstm(): b_hidden_to_hidden_reverse = jnp.array([0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0]) combined_w = jnp.concatenate([w_in_to_hidden, w_hidden_to_hidden], axis=0) - cell = cell.at["ih2h_weight"].set(combined_w) - cell = cell.at["ih2h_bias"].set(b_hidden_to_hidden) + cell = cell.at["in_hidden_to_hidden_weight"].set(combined_w) + cell = cell.at["in_hidden_to_hidden_bias"].set(b_hidden_to_hidden) combined_w_reverse = jnp.concatenate( - [w_in_to_hidden_reverse, w_hidden_to_hidden_reverse], axis=0 + [w_in_to_hidden_reverse, w_hidden_to_hidden_reverse] + ) + reverse_cell = reverse_cell.at["in_hidden_to_hidden_weight"].set(combined_w_reverse) + reverse_cell = reverse_cell.at["in_hidden_to_hidden_bias"].set( + b_hidden_to_hidden_reverse ) - reverse_cell = reverse_cell.at["ih2h_weight"].set(combined_w_reverse) - reverse_cell = reverse_cell.at["ih2h_bias"].set(b_hidden_to_hidden_reverse) res = ScanRNN(cell, reverse_cell, return_sequences=False)(x)