[bugfix] Bugfix on tensor sum

 - Check beta is null when copy the origin data on sum function and added unittest
 - Remove argument beta on outplace sum function
 - Makes bias_hh to zero instead of bias_ih in recurrent models when convert pytorch to nntrainer
 - Fix typos

Self evaluation:

Build test: [X]Passed [ ]Failed [ ]Skipped
Run test: [X]Passed [ ]Failed [ ]Skipped

Signed-off-by: hyeonseok lee <hs89.lee@samsung.com>

diff --git a/nntrainer/compiler/recurrent_realizer.cpp b/nntrainer/compiler/recurrent_realizer.cpp
index d4daa4c..a5c3923 100644 (file)
--- a/nntrainer/compiler/recurrent_realizer.cpp
+++ b/nntrainer/compiler/recurrent_realizer.cpp
@@ -20,7 +20,6 @@
 #include <nntrainer_error.h>
 #include <node_exporter.h>
 #include <remap_realizer.h>
-#include <rnn.h>
 #include <rnncell.h>
 #include <util_func.h>
 

diff --git a/nntrainer/layers/gru.cpp b/nntrainer/layers/gru.cpp
index ea1a0da..59d35d4 100644 (file)
--- a/nntrainer/layers/gru.cpp
+++ b/nntrainer/layers/gru.cpp
@@ -19,7 +19,7 @@
  *          |  |     |d16    | d7              |d8
  *          |  |    [+]      [+]              [+]
  *          |  |    / \d16   |  \ d7          / \ d8
- *          |  |  Wxhr Whhr Wxhz Whhz       Wxhg Whhg
+ *          |  |  Whhr Wxhr Whhz Wxhz       Whhg Wxhg
  *          |  |  |d17  |d13 |d12 |d11       |d10 | d9
  *          +- |--+------|---+    |          |    |
  *             +---------|--------|----------+    |
@@ -48,8 +48,6 @@ enum GRUParams {
   dropout_mask
 };
 
-#define NUM_GATE 3
-
 GRULayer::GRULayer() :
   LayerImpl(),
   gru_props(props::Unit(), props::HiddenStateActivation(),
@@ -61,11 +59,11 @@ GRULayer::GRULayer() :
   epsilon(1e-3) {}
 
 // - weight_xh ( input to hidden )
-//  : [1, 1, input_size, unit (hidden_size) x NUM_GATE] -> f, g, i, o
+//  : [1, 1, input_size, unit (hidden_size) x NUM_GATE] -> z, r, g
 // - weight_hh ( hidden to hidden )
-//  : [1, 1, unit (hidden_size) , unit (hidden_size) x NUM_GATE] -> f, g, i, o
+//  : [1, 1, unit (hidden_size) , unit (hidden_size) x NUM_GATE] -> z, r, g
 // - bias_h ( hidden bias )
-//  : [1, 1, 1, unit (hidden_size) x NUM_GATE] -> f, g, i, o
+//  : [1, 1, 1, unit (hidden_size) x NUM_GATE] -> z, r, g
 void GRULayer::finalize(InitLayerContext &context) {
   auto &weight_regularizer =
     std::get<props::WeightRegularizer>(*layer_impl_props);
@@ -100,7 +98,7 @@ void GRULayer::finalize(InitLayerContext &context) {
 
   if (dropout_rate > epsilon) {
     wt_idx[GRUParams::dropout_mask] = context.requestTensor(
-      output_dim, "GRU:dropout_mask", Tensor::Initializer::NONE, false,
+      output_dim, "dropout_mask", Tensor::Initializer::NONE, false,
       TensorLifespan::ITERATION_LIFESPAN);
   }
 
@@ -123,7 +121,7 @@ void GRULayer::finalize(InitLayerContext &context) {
   dim_hh.width(unit * NUM_GATE);
   dim_hh.batch(1);
 
-  // weight_initializer can be set sepeartely. weight_xh initializer,
+  // weight_initializer can be set seperately. weight_xh initializer,
   // weight_hh initializer kernel initializer & recurrent_initializer in keras
   // for now, it is set same way.
   wt_idx[GRUParams::weight_xh] =

diff --git a/nntrainer/layers/gru.h b/nntrainer/layers/gru.h
index e74eb5f..c66c7ae 100644 (file)
--- a/nntrainer/layers/gru.h
+++ b/nntrainer/layers/gru.h
@@ -98,6 +98,8 @@ public:
   inline static const std::string type = "gru";
 
 private:
+  static constexpr unsigned int NUM_GATE = 3;
+
   /**
    * Unit: number of output neurons
    * HiddenStateActivation: activation type for hidden state. default is tanh
@@ -123,7 +125,7 @@ private:
   ActiFunc recurrent_acti_func;
 
   /**
-   * @brief     to pretect overflow
+   * @brief     to protect overflow
    */
   float epsilon;
 };

diff --git a/nntrainer/layers/lstm.cpp b/nntrainer/layers/lstm.cpp
index 07ab30f..25f3c4e 100644 (file)
--- a/nntrainer/layers/lstm.cpp
+++ b/nntrainer/layers/lstm.cpp
@@ -33,8 +33,6 @@ enum LSTMParams {
   dropout_mask
 };
 
-#define NUM_GATE 4
-
 LSTMLayer::LSTMLayer() :
   LayerImpl(),
   lstm_props(props::Unit(), props::HiddenStateActivation(),
@@ -111,7 +109,7 @@ void LSTMLayer::finalize(InitLayerContext &context) {
   dim_hh.width(unit * NUM_GATE);
   dim_hh.batch(1);
 
-  // weight_initializer can be set sepeartely. weight_xh initializer,
+  // weight_initializer can be set seperately. weight_xh initializer,
   // weight_hh initializer kernel initializer & recurrent_initializer in keras
   // for now, it is set same way.
   wt_idx[LSTMParams::weight_xh] =

diff --git a/nntrainer/layers/lstm.h b/nntrainer/layers/lstm.h
index 668e045..9e63584 100644 (file)
--- a/nntrainer/layers/lstm.h
+++ b/nntrainer/layers/lstm.h
@@ -98,6 +98,8 @@ public:
   inline static const std::string type = "lstm";
 
 private:
+  static constexpr unsigned int NUM_GATE = 4;
+
   /**
    * Unit: number of output neurons
    * HiddenStateActivation: activation type for hidden state. default is tanh
@@ -124,7 +126,7 @@ private:
   ActiFunc recurrent_acti_func;
 
   /**
-   * @brief     to pretect overflow
+   * @brief     to protect overflow
    */
   float epsilon;
 };

diff --git a/nntrainer/layers/lstmcell.cpp b/nntrainer/layers/lstmcell.cpp
index 24927ff..588b4a8 100644 (file)
--- a/nntrainer/layers/lstmcell.cpp
+++ b/nntrainer/layers/lstmcell.cpp
@@ -33,8 +33,6 @@ enum LSTMParams {
   dropout_mask
 };
 
-#define NUM_GATE 4
-
 LSTMCellLayer::LSTMCellLayer() :
   LayerImpl(),
   lstm_props(props::Unit(), props::HiddenStateActivation(),
@@ -110,7 +108,7 @@ void LSTMCellLayer::finalize(InitLayerContext &context) {
   dim_hh.width(unit * NUM_GATE);
   dim_hh.batch(1);
 
-  // weight_initializer can be set sepeartely. weight_xh initializer,
+  // weight_initializer can be set seperately. weight_xh initializer,
   // weight_hh initializer kernel initializer & recurrent_initializer in keras
   // for now, it is set same way.
   wt_idx[LSTMParams::weight_xh] =

diff --git a/nntrainer/layers/lstmcell.h b/nntrainer/layers/lstmcell.h
index bff553d..cc6dcf4 100644 (file)
--- a/nntrainer/layers/lstmcell.h
+++ b/nntrainer/layers/lstmcell.h
@@ -85,6 +85,8 @@ public:
   inline static const std::string type = "lstmcell";
 
 private:
+  static constexpr unsigned int NUM_GATE = 4;
+
   /**
    * Unit: number of output neurons
    * HiddenStateActivation: activation type for hidden state. default is tanh
@@ -110,7 +112,7 @@ private:
   ActiFunc recurrent_acti_func;
 
   /**
-   * @brief     to pretect overflow
+   * @brief     to protect overflow
    */
   float epsilon;
 };

diff --git a/nntrainer/layers/rnncell.cpp b/nntrainer/layers/rnncell.cpp
index 6dd86d7..df23084 100644 (file)
--- a/nntrainer/layers/rnncell.cpp
+++ b/nntrainer/layers/rnncell.cpp
@@ -4,7 +4,7 @@
  *
  * @file   rnncell.cpp
  * @date   29 Oct 2021
- * @brief  This is Recurrent Layer Cell Class of Neural Network
+ * @brief  This is Recurrent Cell Layer Class of Neural Network
  * @see    https://github.com/nnstreamer/nntrainer
  * @author hyeonseok lee <hs89.lee@samsung.com>
  * @bug    No known bugs except for NYI items

diff --git a/nntrainer/tensor/tensor.cpp b/nntrainer/tensor/tensor.cpp
index fb14cb3..812a081 100644 (file)
--- a/nntrainer/tensor/tensor.cpp
+++ b/nntrainer/tensor/tensor.cpp
@@ -120,8 +120,7 @@ public:
   SrcSharedTensor() : src(nullptr), off(0) {}
 
   SrcSharedTensor(const Tensor *tensor, unsigned int offset) :
-    src(tensor),
-    off(offset) {}
+    src(tensor), off(offset) {}
 
   /**
    * @brief   Get the allocated src tensor
@@ -779,9 +778,9 @@ Tensor Tensor::sum_by_batch() const {
 /**
  * @brief Calculate sum according to the axis.
  */
-Tensor Tensor::sum(unsigned int axis, float alpha, float beta) const {
+Tensor Tensor::sum(unsigned int axis, float alpha) const {
   Tensor ret;
-  return sum(axis, ret, alpha, beta);
+  return sum(axis, ret, alpha, 0);
 }
 Tensor &Tensor::sum(unsigned int axis, Tensor &ret, float alpha,
                     float beta) const {
@@ -793,7 +792,7 @@ Tensor &Tensor::sum(unsigned int axis, Tensor &ret, float alpha,
   if (axis >= 4)
     throw std::out_of_range("Error: axis is invalid");
 
-  if (dim.getDim()[axis] == 1 and alpha == 1.0) {
+  if (dim.getDim()[axis] == 1 and alpha == 1.0 and !beta) {
     CREATE_IF_EMPTY_DIMS(ret, dim);
     ret.copy(this->getData());
     return ret;

diff --git a/nntrainer/tensor/tensor.h b/nntrainer/tensor/tensor.h
index 76cf86b..d170b18 100644 (file)
--- a/nntrainer/tensor/tensor.h
+++ b/nntrainer/tensor/tensor.h
@@ -620,7 +620,7 @@ public:
    * @param[in] alpha Scale the sum by this value
    * @retval    Calculated Tensor
    */
-  Tensor sum(unsigned int axis, float alpha = 1.0, float beta = 0.0) const;
+  Tensor sum(unsigned int axis, float alpha = 1.0) const;
 
   /**
    * @brief     sum all the Tensor elements according to the axis
@@ -1114,8 +1114,8 @@ public:
    * @brief     return current stride of tensor.
    * @retval    int[MAXDIM] strides
    */
-  const std::array<unsigned int, TensorDim::MAXDIM> getStrides() const
-    noexcept {
+  const std::array<unsigned int, TensorDim::MAXDIM>
+  getStrides() const noexcept {
     return strides;
   }
   /**

diff --git a/test/input_gen/genModelsRecurrent_v2.py b/test/input_gen/genModelsRecurrent_v2.py
index 6a48787..e4d58d1 100644 (file)
--- a/test/input_gen/genModelsRecurrent_v2.py
+++ b/test/input_gen/genModelsRecurrent_v2.py
@@ -38,17 +38,12 @@ class RNNCellStacked(torch.nn.Module):
             ]
         )
         for rnn in self.rnns:
-            rnn.bias_ih.data.fill_(0.0)
-            rnn.bias_ih.requires_grad=False
+            rnn.bias_hh.data.fill_(0.0)
+            rnn.bias_hh.requires_grad=False
         self.unroll_for = unroll_for
         self.loss = torch.nn.MSELoss()
 
     def forward(self, inputs, labels):
-        # second bias is always set to make it always zero grad.
-        # this is because that we are only keeping one bias
-        for rnn in self.rnns:
-            rnn.bias_ih.data.fill_(0.0)
-
         hs = [torch.zeros_like(inputs[0]) for _ in self.rnns]
         out = inputs[0]
         ret = []
@@ -72,21 +67,13 @@ class LSTMStacked(torch.nn.Module):
                 for _ in range(num_lstm)
             ]
         )
-        # self.lstm.weight_hh.data.fill_(1.0)
-        # self.lstm.weight_ih.data.fill_(1.0)
-        # self.lstm.bias_hh.data.fill_(1.0)
         for lstm in self.lstms:
-            lstm.bias_ih.data.fill_(0.0)
-            lstm.bias_ih.requires_grad=False
+            lstm.bias_hh.data.fill_(0.0)
+            lstm.bias_hh.requires_grad=False
         self.unroll_for = unroll_for
         self.loss = torch.nn.MSELoss()
 
     def forward(self, inputs, labels):
-        # second bias is always set to make it always zero grad.
-        # this is because that we are only keeping one bias
-        for lstm in self.lstms:
-            lstm.bias_ih.data.fill_(0.0)
-
         hs = [torch.zeros_like(inputs[0]) for _ in self.lstms]
         cs = [torch.zeros_like(inputs[0]) for _ in self.lstms]
         out = inputs[0]

diff --git a/test/unittest/unittest_nntrainer_tensor.cpp b/test/unittest/unittest_nntrainer_tensor.cpp
index 5d8c3a0..dfd4bcb 100644 (file)
--- a/test/unittest/unittest_nntrainer_tensor.cpp
+++ b/test/unittest/unittest_nntrainer_tensor.cpp
@@ -1542,34 +1542,41 @@ TEST(nntrainer_Tensor, sum_02_n) {
 }
 
 TEST(nntrainer_Tensor, sum_02_p) {
-  int status = ML_ERROR_NONE;
   int batch = 3;
   int channel = 2;
   int height = 2;
   int width = 10;
 
-  float ans0[1][2][2][10] = {{{{39, 42, 45, 48, 51, 54, 57, 60, 63, 66},
-                               {69, 72, 75, 78, 81, 84, 87, 90, 93, 96}},
-                              {{57, 60, 63, 66, 69, 72, 75, 78, 81, 84},
-                               {87, 90, 93, 96, 99, 102, 105, 108, 111, 114}}}};
-
-  float ans1[3][1][2][10] = {{{{8, 10, 12, 14, 16, 18, 20, 22, 24, 26},
-                               {28, 30, 32, 34, 36, 38, 40, 42, 44, 46}}},
-                             {{{32, 34, 36, 38, 40, 42, 44, 46, 48, 50},
-                               {52, 54, 56, 58, 60, 62, 64, 66, 68, 70}}},
-                             {{{56, 58, 60, 62, 64, 66, 68, 70, 72, 74},
-                               {76, 78, 80, 82, 84, 86, 88, 90, 92, 94}}}};
-
-  float ans2[3][2][1][10] = {{{{12, 14, 16, 18, 20, 22, 24, 26, 28, 30}},
-                              {{24, 26, 28, 30, 32, 34, 36, 38, 40, 42}}},
-                             {{{36, 38, 40, 42, 44, 46, 48, 50, 52, 54}},
-                              {{48, 50, 52, 54, 56, 58, 60, 62, 64, 66}}},
-                             {{{60, 62, 64, 66, 68, 70, 72, 74, 76, 78}},
-                              {{72, 74, 76, 78, 80, 82, 84, 86, 88, 90}}}};
-
-  float ans3[3][2][2][1] = {{{{55}, {155}}, {{115}, {215}}},
-                            {{{175}, {275}}, {{235}, {335}}},
-                            {{{295}, {395}}, {{355}, {455}}}};
+  nntrainer::Tensor ans0(
+    std::vector<std::vector<std::vector<std::vector<float>>>>(
+      {{{{39, 42, 45, 48, 51, 54, 57, 60, 63, 66},
+         {69, 72, 75, 78, 81, 84, 87, 90, 93, 96}},
+        {{57, 60, 63, 66, 69, 72, 75, 78, 81, 84},
+         {87, 90, 93, 96, 99, 102, 105, 108, 111, 114}}}}));
+
+  nntrainer::Tensor ans1(
+    std::vector<std::vector<std::vector<std::vector<float>>>>(
+      {{{{8, 10, 12, 14, 16, 18, 20, 22, 24, 26},
+         {28, 30, 32, 34, 36, 38, 40, 42, 44, 46}}},
+       {{{32, 34, 36, 38, 40, 42, 44, 46, 48, 50},
+         {52, 54, 56, 58, 60, 62, 64, 66, 68, 70}}},
+       {{{56, 58, 60, 62, 64, 66, 68, 70, 72, 74},
+         {76, 78, 80, 82, 84, 86, 88, 90, 92, 94}}}}));
+
+  nntrainer::Tensor ans2(
+    std::vector<std::vector<std::vector<std::vector<float>>>>(
+      {{{{12, 14, 16, 18, 20, 22, 24, 26, 28, 30}},
+        {{24, 26, 28, 30, 32, 34, 36, 38, 40, 42}}},
+       {{{36, 38, 40, 42, 44, 46, 48, 50, 52, 54}},
+        {{48, 50, 52, 54, 56, 58, 60, 62, 64, 66}}},
+       {{{60, 62, 64, 66, 68, 70, 72, 74, 76, 78}},
+        {{72, 74, 76, 78, 80, 82, 84, 86, 88, 90}}}}));
+
+  nntrainer::Tensor ans3(
+    std::vector<std::vector<std::vector<std::vector<float>>>>(
+      {{{{55}, {155}}, {{115}, {215}}},
+       {{{175}, {275}}, {{235}, {335}}},
+       {{{295}, {395}}, {{355}, {455}}}}));
 
   nntrainer::Tensor input(batch, channel, height, width);
   GEN_TEST_INPUT(input, i * (batch * height * channel) + j * (batch * height) +
@@ -1580,63 +1587,227 @@ TEST(nntrainer_Tensor, sum_02_p) {
   nntrainer::Tensor result2 = input.sum(2);
   nntrainer::Tensor result3 = input.sum(3);
 
-  for (unsigned int i = 0; i < result0.batch(); ++i) {
-    for (unsigned int l = 0; l < result0.channel(); ++l) {
-      for (unsigned int j = 0; j < result0.height(); ++j) {
-        for (unsigned int k = 0; k < result0.width(); ++k) {
-          if (ans0[i][l][j][k] != result0.getValue(i, l, j, k)) {
-            status = ML_ERROR_RESULT_OUT_OF_RANGE;
-            goto end_test;
-          }
-        }
-      }
-    }
-  }
+  EXPECT_EQ(ans0, result0);
+  EXPECT_EQ(ans1, result1);
+  EXPECT_EQ(ans2, result2);
+  EXPECT_EQ(ans3, result3);
+}
 
-  for (unsigned int i = 0; i < result1.batch(); ++i) {
-    for (unsigned int l = 0; l < result1.channel(); ++l) {
-      for (unsigned int j = 0; j < result1.height(); ++j) {
-        for (unsigned int k = 0; k < result1.width(); ++k) {
-          if (ans1[i][l][j][k] != result1.getValue(i, l, j, k)) {
-            status = ML_ERROR_RESULT_OUT_OF_RANGE;
-            goto end_test;
-          }
-        }
-      }
-    }
-  }
+TEST(nntrainer_Tensor, sum_03_p) {
+  const int batch = 3;
+  const int channel = 2;
+  const int height = 1;
+  const int width = 10;
 
-  for (unsigned int i = 0; i < result2.batch(); ++i) {
-    for (unsigned int l = 0; l < result2.channel(); ++l) {
-      for (unsigned int j = 0; j < result2.height(); ++j) {
-        for (unsigned int k = 0; k < result2.width(); ++k) {
-          if (ans2[i][l][j][k] != result2.getValue(i, l, j, k)) {
-            status = ML_ERROR_RESULT_OUT_OF_RANGE;
-            goto end_test;
-          }
-        }
-      }
+  nntrainer::Tensor input(batch, channel, height, width);
+  GEN_TEST_INPUT(input, i * (height * channel * width) + j * (height * width) +
+                          k * (width) + l + 1);
+  // Test for alpha == 1 and beta == 0 and dimension of reduced axis == 1
+  {
+    nntrainer::Tensor ans_0_1_0(
+      std::vector<std::vector<std::vector<std::vector<float>>>>(
+        {{{{63, 66, 69, 72, 75, 78, 81, 84, 87, 90}},
+          {{93, 96, 99, 102, 105, 108, 111, 114, 117, 120}}}}));
+
+    nntrainer::Tensor ans_1_1_0(
+      std::vector<std::vector<std::vector<std::vector<float>>>>(
+        {{{{12, 14, 16, 18, 20, 22, 24, 26, 28, 30}}},
+         {{{52, 54, 56, 58, 60, 62, 64, 66, 68, 70}}},
+         {{{92, 94, 96, 98, 100, 102, 104, 106, 108, 110}}}}));
+
+    nntrainer::Tensor ans_2_1_0(
+      std::vector<std::vector<std::vector<std::vector<float>>>>(
+        {{{{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}},
+          {{11, 12, 13, 14, 15, 16, 17, 18, 19, 20}}},
+         {{{21, 22, 23, 24, 25, 26, 27, 28, 29, 30}},
+          {{31, 32, 33, 34, 35, 36, 37, 38, 39, 40}}},
+         {{{41, 42, 43, 44, 45, 46, 47, 48, 49, 50}},
+          {{51, 52, 53, 54, 55, 56, 57, 58, 59, 60}}}}));
+
+    nntrainer::Tensor ans_3_1_0(
+      std::vector<std::vector<std::vector<std::vector<float>>>>(
+        {{{{55}}, {{155}}}, {{{255}}, {{355}}}, {{{455}}, {{555}}}}));
+
+    nntrainer::Tensor result_0_1_0 = input.sum(0, 1);
+    nntrainer::Tensor result_1_1_0 = input.sum(1, 1);
+    nntrainer::Tensor result_2_1_0 = input.sum(2, 1);
+    nntrainer::Tensor result_3_1_0 = input.sum(3, 1);
+
+    EXPECT_EQ(ans_0_1_0, result_0_1_0);
+    EXPECT_EQ(ans_1_1_0, result_1_1_0);
+    EXPECT_EQ(ans_2_1_0, result_2_1_0);
+    EXPECT_EQ(ans_3_1_0, result_3_1_0);
+  }
+
+  // Test for alpha == 1 and beta == 2 and dimension of reduced axis == 1
+  {
+    nntrainer::Tensor ans_0_1_2(
+      std::vector<std::vector<std::vector<std::vector<float>>>>(
+        {{{{65, 70, 75, 80, 85, 90, 95, 100, 105, 110}},
+          {{115, 120, 125, 130, 135, 140, 145, 150, 155, 160}}}}));
+
+    nntrainer::Tensor ans_1_1_2(
+      std::vector<std::vector<std::vector<std::vector<float>>>>(
+        {{{{14, 18, 22, 26, 30, 34, 38, 42, 46, 50}}},
+         {{{74, 78, 82, 86, 90, 94, 98, 102, 106, 110}}},
+         {{{134, 138, 142, 146, 150, 154, 158, 162, 166, 170}}}}));
+
+    nntrainer::Tensor ans_2_1_2(
+      std::vector<std::vector<std::vector<std::vector<float>>>>(
+        {{{{3, 6, 9, 12, 15, 18, 21, 24, 27, 30}},
+          {{33, 36, 39, 42, 45, 48, 51, 54, 57, 60}}},
+         {{{63, 66, 69, 72, 75, 78, 81, 84, 87, 90}},
+          {{93, 96, 99, 102, 105, 108, 111, 114, 117, 120}}},
+         {{{123, 126, 129, 132, 135, 138, 141, 144, 147, 150}},
+          {{153, 156, 159, 162, 165, 168, 171, 174, 177, 180}}}}));
+
+    nntrainer::Tensor ans_3_1_2(
+      std::vector<std::vector<std::vector<std::vector<float>>>>(
+        {{{{57}}, {{159}}}, {{{261}}, {{363}}}, {{{465}}, {{567}}}}));
+
+    nntrainer::Tensor output_0_1_2(1, channel, height, width);
+    {
+      const int batch = 1;
+      GEN_TEST_INPUT(output_0_1_2, i * (channel * height * width) +
+                                     j * (height * width) + k * (width) + l +
+                                     1);
     }
-  }
-
-  for (unsigned int i = 0; i < result3.batch(); ++i) {
-    for (unsigned int l = 0; l < result3.channel(); ++l) {
-      for (unsigned int j = 0; j < result3.height(); ++j) {
-        for (unsigned int k = 0; k < result3.width(); ++k) {
-          if (ans3[i][l][j][k] != result3.getValue(i, l, j, k)) {
-            status = ML_ERROR_RESULT_OUT_OF_RANGE;
-            goto end_test;
-          }
-        }
-      }
+    nntrainer::Tensor output_1_1_2(batch, 1, height, width);
+    {
+      const int channel = 1;
+      GEN_TEST_INPUT(output_1_1_2, i * (channel * height * width) +
+                                     j * (height * width) + k * (width) + l +
+                                     1);
     }
-  }
+    nntrainer::Tensor output_2_1_2(batch, channel, 1, width);
+    {
+      const int height = 1;
+      GEN_TEST_INPUT(output_2_1_2, i * (channel * height * width) +
+                                     j * (height * width) + k * (width) + l +
+                                     1);
+    }
+    nntrainer::Tensor output_3_1_2(batch, channel, height, 1);
+    {
+      const int width = 1;
+      GEN_TEST_INPUT(output_3_1_2, i * (channel * height * width) +
+                                     j * (height * width) + k * (width) + l +
+                                     1);
+    }
+    nntrainer::Tensor result_0_1_2 = input.sum(0, output_0_1_2, 1, 2);
+    nntrainer::Tensor result_1_1_2 = input.sum(1, output_1_1_2, 1, 2);
+    nntrainer::Tensor result_2_1_2 = input.sum(2, output_2_1_2, 1, 2);
+    nntrainer::Tensor result_3_1_2 = input.sum(3, output_3_1_2, 1, 2);
+
+    EXPECT_EQ(ans_0_1_2, result_0_1_2);
+    EXPECT_EQ(ans_1_1_2, result_1_1_2);
+    EXPECT_EQ(ans_2_1_2, result_2_1_2);
+    EXPECT_EQ(ans_3_1_2, result_3_1_2);
+  }
+
+  // Test for alpha == 2 and beta == 0
+  {
+    nntrainer::Tensor ans_0_2_0(
+      std::vector<std::vector<std::vector<std::vector<float>>>>(
+        {{{{126, 132, 138, 144, 150, 156, 162, 168, 174, 180}},
+          {{186, 192, 198, 204, 210, 216, 222, 228, 234, 240}}}}));
+
+    nntrainer::Tensor ans_1_2_0(
+      std::vector<std::vector<std::vector<std::vector<float>>>>(
+        {{{{24, 28, 32, 36, 40, 44, 48, 52, 56, 60}}},
+         {{{104, 108, 112, 116, 120, 124, 128, 132, 136, 140}}},
+         {{{184, 188, 192, 196, 200, 204, 208, 212, 216, 220}}}}));
+
+    nntrainer::Tensor ans_2_2_0(
+      std::vector<std::vector<std::vector<std::vector<float>>>>(
+        {{{{2, 4, 6, 8, 10, 12, 14, 16, 18, 20}},
+          {{22, 24, 26, 28, 30, 32, 34, 36, 38, 40}}},
+         {{{42, 44, 46, 48, 50, 52, 54, 56, 58, 60}},
+          {{62, 64, 66, 68, 70, 72, 74, 76, 78, 80}}},
+         {{{82, 84, 86, 88, 90, 92, 94, 96, 98, 100}},
+          {{102, 104, 106, 108, 110, 112, 114, 116, 118, 120}}}}));
+
+    nntrainer::Tensor ans_3_2_0(
+      std::vector<std::vector<std::vector<std::vector<float>>>>(
+        {{{{110}}, {{310}}}, {{{510}}, {{710}}}, {{{910}}, {{1110}}}}));
+
+    nntrainer::Tensor result_0_2_0 = input.sum(0, 2);
+    nntrainer::Tensor result_1_2_0 = input.sum(1, 2);
+    nntrainer::Tensor result_2_2_0 = input.sum(2, 2);
+    nntrainer::Tensor result_3_2_0 = input.sum(3, 2);
+
+    EXPECT_EQ(ans_0_2_0, result_0_2_0);
+    EXPECT_EQ(ans_1_2_0, result_1_2_0);
+    EXPECT_EQ(ans_2_2_0, result_2_2_0);
+    EXPECT_EQ(ans_3_2_0, result_3_2_0);
+  }
+
+  // Test for alpha == 2 and beta == 2
+  {
+    nntrainer::Tensor ans_0_2_2(
+      std::vector<std::vector<std::vector<std::vector<float>>>>(
+        {{{{128, 136, 144, 152, 160, 168, 176, 184, 192, 200}},
+          {{208, 216, 224, 232, 240, 248, 256, 264, 272, 280}}}}));
+
+    nntrainer::Tensor ans_1_2_2(
+      std::vector<std::vector<std::vector<std::vector<float>>>>(
+        {{{{26, 32, 38, 44, 50, 56, 62, 68, 74, 80}}},
+         {{{126, 132, 138, 144, 150, 156, 162, 168, 174, 180}}},
+         {{{226, 232, 238, 244, 250, 256, 262, 268, 274, 280}}}}));
+
+    nntrainer::Tensor ans_2_2_2(
+      std::vector<std::vector<std::vector<std::vector<float>>>>(
+        {{{{4, 8, 12, 16, 20, 24, 28, 32, 36, 40}},
+          {{44, 48, 52, 56, 60, 64, 68, 72, 76, 80}}},
+         {{{84, 88, 92, 96, 100, 104, 108, 112, 116, 120}},
+          {{124, 128, 132, 136, 140, 144, 148, 152, 156, 160}}},
+         {{{164, 168, 172, 176, 180, 184, 188, 192, 196, 200}},
+          {{204, 208, 212, 216, 220, 224, 228, 232, 236, 240}}}}));
+
+    nntrainer::Tensor ans_3_2_2(
+      std::vector<std::vector<std::vector<std::vector<float>>>>(
+        {{{{112}}, {{314}}}, {{{516}}, {{718}}}, {{{920}}, {{1122}}}}));
+
+    nntrainer::Tensor output_0_2_2(1, channel, height, width);
+    {
+      const int batch = 1;
+      GEN_TEST_INPUT(output_0_2_2, i * (channel * height * width) +
+                                     j * (height * width) + k * (width) + l +
+                                     1);
+    }
+    nntrainer::Tensor output_1_2_2(batch, 1, height, width);
+    {
+      const int channel = 1;
+      GEN_TEST_INPUT(output_1_2_2, i * (channel * height * width) +
+                                     j * (height * width) + k * (width) + l +
+                                     1);
+    }
+    nntrainer::Tensor output_2_2_2(batch, channel, 1, width);
+    {
+      const int height = 1;
+      GEN_TEST_INPUT(output_2_2_2, i * (channel * height * width) +
+                                     j * (height * width) + k * (width) + l +
+                                     1);
+    }
+    nntrainer::Tensor output_3_2_2(batch, channel, height, 1);
+    {
+      const int width = 1;
+      GEN_TEST_INPUT(output_3_2_2, i * (channel * height * width) +
+                                     j * (height * width) + k * (width) + l +
+                                     1);
+    }
+    nntrainer::Tensor result_0_2_2 = input.sum(0, output_0_2_2, 2, 2);
+    nntrainer::Tensor result_1_2_2 = input.sum(1, output_1_2_2, 2, 2);
+    nntrainer::Tensor result_2_2_2 = input.sum(2, output_2_2_2, 2, 2);
+    nntrainer::Tensor result_3_2_2 = input.sum(3, output_3_2_2, 2, 2);
 
-end_test:
-  EXPECT_EQ(status, ML_ERROR_NONE);
+    EXPECT_EQ(ans_0_2_2, result_0_2_2);
+    EXPECT_EQ(ans_1_2_2, result_1_2_2);
+    EXPECT_EQ(ans_2_2_2, result_2_2_2);
+    EXPECT_EQ(ans_3_2_2, result_3_2_2);
+  }
 }
 
-TEST(nntrainer_Tensor, sum_03_p) {
+TEST(nntrainer_Tensor, sum_04_p) {
   int status = ML_ERROR_NONE;
   int batch = 3;
   int channel = 2;