From cb3241866917024a59955a5099a8eab461357d9c Mon Sep 17 00:00:00 2001 From: marka17 Date: Tue, 8 Jan 2019 21:14:54 -0800 Subject: [PATCH] Add element-wise multiplication in formulas (#15834) Summary: Absence of element-wise multiplication can confused some beginners Pull Request resolved: https://github.com/pytorch/pytorch/pull/15834 Differential Revision: D13603369 Pulled By: soumith fbshipit-source-id: 1d5c17c57778ddbb4b201122d826d1d6437204d1 --- torch/nn/modules/rnn.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/torch/nn/modules/rnn.py b/torch/nn/modules/rnn.py index 09844fa..6578f8f 100644 --- a/torch/nn/modules/rnn.py +++ b/torch/nn/modules/rnn.py @@ -363,8 +363,8 @@ class LSTM(RNNBase): f_t = \sigma(W_{if} x_t + b_{if} + W_{hf} h_{(t-1)} + b_{hf}) \\ g_t = \tanh(W_{ig} x_t + b_{ig} + W_{hg} h_{(t-1)} + b_{hg}) \\ o_t = \sigma(W_{io} x_t + b_{io} + W_{ho} h_{(t-1)} + b_{ho}) \\ - c_t = f_t c_{(t-1)} + i_t g_t \\ - h_t = o_t \tanh(c_t) \\ + c_t = f_t * c_{(t-1)} + i_t * g_t \\ + h_t = o_t * \tanh(c_t) \\ \end{array} where :math:`h_t` is the hidden state at time `t`, :math:`c_t` is the cell @@ -469,7 +469,7 @@ class GRU(RNNBase): r_t = \sigma(W_{ir} x_t + b_{ir} + W_{hr} h_{(t-1)} + b_{hr}) \\ z_t = \sigma(W_{iz} x_t + b_{iz} + W_{hz} h_{(t-1)} + b_{hz}) \\ n_t = \tanh(W_{in} x_t + b_{in} + r_t (W_{hn} h_{(t-1)}+ b_{hn})) \\ - h_t = (1 - z_t) n_t + z_t h_{(t-1)} + h_t = (1 - z_t) * n_t + z_t * h_{(t-1)} \end{array} where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the input @@ -700,7 +700,7 @@ class LSTMCell(RNNCellBase): g = \tanh(W_{ig} x + b_{ig} + W_{hg} h + b_{hg}) \\ o = \sigma(W_{io} x + b_{io} + W_{ho} h + b_{ho}) \\ c' = f * c + i * g \\ - h' = o \tanh(c') \\ + h' = o * \tanh(c') \\ \end{array} where :math:`\sigma` is the sigmoid function. -- 2.7.4