[Tensor] Add tensor cat

This patch add tensor cat method and it's test

**Self evaluation:**
1. Build test: [X]Passed [ ]Failed [ ]Skipped
2. Run test: [X]Passed [ ]Failed [ ]Skipped

Signed-off-by: Jihoon Lee <jhoon.it.lee@samsung.com>

diff --git a/nntrainer/tensor/tensor.cpp b/nntrainer/tensor/tensor.cpp
index 8b48893..fa30fab 100644 (file)
--- a/nntrainer/tensor/tensor.cpp
+++ b/nntrainer/tensor/tensor.cpp
@@ -21,6 +21,7 @@
  *
  */
 
+#include <algorithm>
 #include <assert.h>
 #include <cmath>
 #include <cstring>
@@ -28,6 +29,7 @@
 #include <iomanip>
 #include <iostream>
 #include <iterator>
+#include <numeric>
 #include <random>
 #include <regex>
 #include <sstream>
@@ -753,9 +755,14 @@ Tensor Tensor::getSharedDataTensor(const TensorDim dim_, unsigned int offset,
 }
 
 std::vector<Tensor> Tensor::split(unsigned num_size, int axis) {
+
+  NNTR_THROW_IF(num_size == 0, std::invalid_argument)
+    << "num size cannot be zero";
+
   if (axis == -1) {
     axis = 3;
   }
+
   NNTR_THROW_IF(!(0 <= axis && axis < 4), std::invalid_argument)
     << "cannot split axis of axis: " << axis;
 
@@ -767,8 +774,8 @@ std::vector<Tensor> Tensor::split(unsigned num_size, int axis) {
   auto new_dim = dim.getTensorDim(axis) / num_size;
   ret_dim.setTensorDim(axis, new_dim);
 
-  auto iter_value = [this, &ret_dim](std::array<unsigned, 4> &loc) {
-    auto value = getValue(loc[0], loc[1], loc[2], loc[3]);
+  auto iter_value = [this, &ret_dim](std::array<unsigned, 4> &loc) -> float & {
+    auto &value = getValue(loc[0], loc[1], loc[2], loc[3]);
     for (int i = 3; i >= 0; --i) {
       loc[i]++;
       if (loc[i] % ret_dim.getTensorDim(i) == 0) {
@@ -795,6 +802,66 @@ std::vector<Tensor> Tensor::split(unsigned num_size, int axis) {
   return ret;
 }
 
+Tensor Tensor::cat(const std::vector<Tensor> &tensors, int axis) {
+
+  if (axis == -1) {
+    axis = 3;
+  }
+
+  NNTR_THROW_IF(!(0 <= axis && axis < 4), std::invalid_argument)
+    << "cannot split axis of axis: " << axis;
+
+  NNTR_THROW_IF(tensors.empty(), std::invalid_argument)
+    << "given tensor vector is empty";
+
+  auto ref_dim = tensors.front().getDim();
+  ref_dim.setTensorDim(axis, 1);
+  NNTR_THROW_IF(!std::all_of(tensors.begin(), tensors.end(),
+                             [&ref_dim, axis](const Tensor &t) {
+                               auto cur_dim = t.getDim();
+                               cur_dim.setTensorDim(axis, 1);
+                               return ref_dim == cur_dim;
+                             }),
+                std::invalid_argument)
+    << " all tensor must have the same dimension except for the axis, ref_dim: "
+    << ref_dim << " axis : " << axis;
+
+  auto axis_dim = std::accumulate(tensors.begin(), tensors.end(), 0u,
+                                  [axis](unsigned cur, const Tensor &t) {
+                                    return cur += t.getDim().getTensorDim(axis);
+                                  });
+  auto iter_value = [](std::array<unsigned, 4> &loc,
+                       std::array<unsigned, 4> &start_loc, Tensor &t,
+                       const TensorDim &ref_dim) -> float & {
+    auto &value = t.getValue(loc[0], loc[1], loc[2], loc[3]);
+    for (int i = 3; i >= 0; --i) {
+      loc[i]++;
+      if (loc[i] - start_loc[i] == ref_dim.getTensorDim(i)) {
+        loc[i] = start_loc[i];
+        continue;
+      }
+      break;
+    }
+    return value;
+  };
+
+  auto ret_dim = ref_dim;
+  ret_dim.setTensorDim(axis, axis_dim);
+
+  auto ret = Tensor(ret_dim);
+
+  std::array<unsigned, 4> loc = {0, 0, 0, 0};
+  for (auto &t : tensors) {
+    std::array<unsigned, 4> start_loc = loc;
+    for (auto i = 0u, sz = t.size(); i < sz; ++i) {
+      iter_value(loc, start_loc, ret, t.getDim()) = t.getValue(i);
+    }
+    loc[axis] += t.getDim().getTensorDim(axis);
+  }
+
+  return ret;
+}
+
 void Tensor::makeSharedDataTensor(const Tensor &src, unsigned int offset) {
   if (strides != src.strides)
     throw std::invalid_argument(

diff --git a/nntrainer/tensor/tensor.h b/nntrainer/tensor/tensor.h
index 2bec03b..dbb8776 100644 (file)
--- a/nntrainer/tensor/tensor.h
+++ b/nntrainer/tensor/tensor.h
@@ -252,16 +252,29 @@ public:
    * @param[in] h height location
    * @param[in] w width location
    */
-  float getValue(unsigned int batch, unsigned int c, unsigned int h,
-                 unsigned int w) const noexcept {
-    return getData()[getIndex(batch, c, h, w)];
+  const float &getValue(unsigned int batch, unsigned int c, unsigned int h,
+                        unsigned int w) const noexcept {
+    return getValue(getIndex(batch, c, h, w));
+  }
+
+  float &getValue(unsigned int batch, unsigned int c, unsigned int h,
+                  unsigned int w) noexcept {
+    return getValue(getIndex(batch, c, h, w));
   }
 
   /**
    * @brief     return value at specific location
    * @param[in] idx location
    */
-  float getValue(unsigned int idx) const noexcept { return getData()[idx]; }
+  const float &getValue(unsigned int idx) const noexcept {
+    return getData()[idx];
+  }
+
+  /**
+   * @brief     return value at specific location
+   * @param[in] idx location
+   */
+  float &getValue(unsigned int idx) noexcept { return getData()[idx]; }
 
   /**
    * @brief Get the Value thinking that it is padded
@@ -1067,6 +1080,15 @@ public:
   std::vector<Tensor> split(unsigned num_size, int axis = 0);
 
   /**
+   * @brief concatenate tensors along axis
+   *
+   * @param tensors tensors to be concatenated to the first tensor
+   * @param axis axis
+   * @return Tensor concatenated tensor
+   */
+  static Tensor cat(const std::vector<Tensor> &tensors, int axis = 0);
+
+  /**
    * @brief make this tensor share memory with given tensor
    *
    * @param src Source tensor whose memory is to be shared

diff --git a/test/input_gen/transLayer_v2.py b/test/input_gen/transLayer_v2.py
index 10f14ad..4f15325 100644 (file)
--- a/test/input_gen/transLayer_v2.py
+++ b/test/input_gen/transLayer_v2.py
@@ -91,14 +91,9 @@ def gru_translate(model):
     # resetgate, inputgate, newgate -> inputgate, resetgate, newgate
     def reorder_weights(params):
         reordered_weights = []
-        for param in params: # param = ("name", weight)
-            if (param[1].dim() == 2): # weight
-                hidden_size = int(param[1].shape[1] / 3)
-            else: # bias
-                hidden_size = int(param[1].shape[0] / 3)
-
-            weight = param[1].hsplit(3)
-            reordered_weights.append((param[0], torch.hstack((weight[1], weight[0], weight[2])))) # reorder
+        for (name, weight) in params: # param = ("name", weight)
+            weight = weight.hsplit(3)
+            reordered_weights.append((name, torch.hstack((weight[1], weight[0], weight[2])))) # reorder
         return reordered_weights
 
     transposed_params = [transpose_(params[0]), transpose_(params[1]), params[2], params[3]]

diff --git a/test/unittest/unittest_nntrainer_tensor.cpp b/test/unittest/unittest_nntrainer_tensor.cpp
index c5283c7..03dff03 100644 (file)
--- a/test/unittest/unittest_nntrainer_tensor.cpp
+++ b/test/unittest/unittest_nntrainer_tensor.cpp
@@ -3184,11 +3184,103 @@ TEST(nntrainer_Tensor, split_p) {
   }
 }
 
-// TEST(nntrainer_Tensor, split_n) {
-//     using namespace nntrainer;
-//     Tensor t = ranged(3, 2, 4, 5);
-//     t.split(3, 0);
-// }
+TEST(nntrainer_Tensor, cat_p) {
+  {
+    std::vector<nntrainer::Tensor> inputs;
+    inputs.reserve(2);
+    inputs.emplace_back(ranged(2, 1, 1, 2));
+    inputs.emplace_back(ranged(2, 2, 1, 2));
+    float answer_data[] = {0, 1, 0, 1, 2, 3, 2, 3, 4, 5, 6, 7};
+    nntrainer::Tensor answer(ml::train::TensorDim{2, 3, 1, 2}, answer_data);
+    EXPECT_EQ(nntrainer::Tensor::cat(inputs, 1), answer);
+  }
+  {
+    std::vector<nntrainer::Tensor> inputs;
+    inputs.reserve(2);
+    inputs.emplace_back(ranged(3, 2, 4, 5));
+    inputs.emplace_back(ranged(2, 2, 4, 5));
+    float answer_data[] = {
+      0,   1,   2,   3,   4,   5,   6,   7,   8,   9,   10,  11,  12,  13,  14,
+      15,  16,  17,  18,  19,  20,  21,  22,  23,  24,  25,  26,  27,  28,  29,
+      30,  31,  32,  33,  34,  35,  36,  37,  38,  39,  40,  41,  42,  43,  44,
+      45,  46,  47,  48,  49,  50,  51,  52,  53,  54,  55,  56,  57,  58,  59,
+      60,  61,  62,  63,  64,  65,  66,  67,  68,  69,  70,  71,  72,  73,  74,
+      75,  76,  77,  78,  79,  80,  81,  82,  83,  84,  85,  86,  87,  88,  89,
+      90,  91,  92,  93,  94,  95,  96,  97,  98,  99,  100, 101, 102, 103, 104,
+      105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
+      0,   1,   2,   3,   4,   5,   6,   7,   8,   9,   10,  11,  12,  13,  14,
+      15,  16,  17,  18,  19,  20,  21,  22,  23,  24,  25,  26,  27,  28,  29,
+      30,  31,  32,  33,  34,  35,  36,  37,  38,  39,  40,  41,  42,  43,  44,
+      45,  46,  47,  48,  49,  50,  51,  52,  53,  54,  55,  56,  57,  58,  59,
+      60,  61,  62,  63,  64,  65,  66,  67,  68,  69,  70,  71,  72,  73,  74,
+      75,  76,  77,  78,  79};
+    nntrainer::Tensor answer(ml::train::TensorDim{5, 2, 4, 5}, answer_data);
+    EXPECT_EQ(nntrainer::Tensor::cat(inputs, 0), answer);
+  }
+  {
+    std::vector<nntrainer::Tensor> inputs;
+    inputs.reserve(2);
+    inputs.emplace_back(ranged(3, 3, 4, 5));
+    inputs.emplace_back(ranged(3, 2, 4, 5));
+    float answer_data[] = {
+      0,   1,   2,   3,   4,   5,   6,   7,   8,   9,   10,  11,  12,  13,
+      14,  15,  16,  17,  18,  19,  20,  21,  22,  23,  24,  25,  26,  27,
+      28,  29,  30,  31,  32,  33,  34,  35,  36,  37,  38,  39,  40,  41,
+      42,  43,  44,  45,  46,  47,  48,  49,  50,  51,  52,  53,  54,  55,
+      56,  57,  58,  59,  0,   1,   2,   3,   4,   5,   6,   7,   8,   9,
+      10,  11,  12,  13,  14,  15,  16,  17,  18,  19,  20,  21,  22,  23,
+      24,  25,  26,  27,  28,  29,  30,  31,  32,  33,  34,  35,  36,  37,
+      38,  39,  60,  61,  62,  63,  64,  65,  66,  67,  68,  69,  70,  71,
+      72,  73,  74,  75,  76,  77,  78,  79,  80,  81,  82,  83,  84,  85,
+      86,  87,  88,  89,  90,  91,  92,  93,  94,  95,  96,  97,  98,  99,
+      100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113,
+      114, 115, 116, 117, 118, 119, 40,  41,  42,  43,  44,  45,  46,  47,
+      48,  49,  50,  51,  52,  53,  54,  55,  56,  57,  58,  59,  60,  61,
+      62,  63,  64,  65,  66,  67,  68,  69,  70,  71,  72,  73,  74,  75,
+      76,  77,  78,  79,  120, 121, 122, 123, 124, 125, 126, 127, 128, 129,
+      130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143,
+      144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157,
+      158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171,
+      172, 173, 174, 175, 176, 177, 178, 179, 80,  81,  82,  83,  84,  85,
+      86,  87,  88,  89,  90,  91,  92,  93,  94,  95,  96,  97,  98,  99,
+      100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113,
+      114, 115, 116, 117, 118, 119};
+    nntrainer::Tensor answer(ml::train::TensorDim{3, 5, 4, 5}, answer_data);
+    EXPECT_EQ(nntrainer::Tensor::cat(inputs, 1), answer);
+  }
+  {
+    std::vector<nntrainer::Tensor> inputs;
+    inputs.reserve(2);
+    inputs.emplace_back(ranged(3, 2, 1, 5));
+    inputs.emplace_back(ranged(3, 2, 2, 5));
+    float answer_data[] = {
+      0,  1,  2,  3,  4,  0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  5,  6,  7,
+      8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 10, 11, 12, 13, 14, 20,
+      21, 22, 23, 24, 25, 26, 27, 28, 29, 15, 16, 17, 18, 19, 30, 31, 32, 33,
+      34, 35, 36, 37, 38, 39, 20, 21, 22, 23, 24, 40, 41, 42, 43, 44, 45, 46,
+      47, 48, 49, 25, 26, 27, 28, 29, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59};
+    nntrainer::Tensor answer(ml::train::TensorDim{3, 2, 3, 5}, answer_data);
+    EXPECT_EQ(nntrainer::Tensor::cat(inputs, 2), answer);
+  }
+  {
+    std::vector<nntrainer::Tensor> inputs;
+    inputs.reserve(3);
+    inputs.emplace_back(ranged(3, 2, 4, 1));
+    inputs.emplace_back(ranged(3, 2, 4, 3));
+    inputs.emplace_back(ranged(3, 2, 4, 2));
+    float answer_data[] = {
+      0,  0,  1,  2,  0,  1,  1,  3,  4,  5,  2,  3,  2,  6,  7,  8,  4,  5,
+      3,  9,  10, 11, 6,  7,  4,  12, 13, 14, 8,  9,  5,  15, 16, 17, 10, 11,
+      6,  18, 19, 20, 12, 13, 7,  21, 22, 23, 14, 15, 8,  24, 25, 26, 16, 17,
+      9,  27, 28, 29, 18, 19, 10, 30, 31, 32, 20, 21, 11, 33, 34, 35, 22, 23,
+      12, 36, 37, 38, 24, 25, 13, 39, 40, 41, 26, 27, 14, 42, 43, 44, 28, 29,
+      15, 45, 46, 47, 30, 31, 16, 48, 49, 50, 32, 33, 17, 51, 52, 53, 34, 35,
+      18, 54, 55, 56, 36, 37, 19, 57, 58, 59, 38, 39, 20, 60, 61, 62, 40, 41,
+      21, 63, 64, 65, 42, 43, 22, 66, 67, 68, 44, 45, 23, 69, 70, 71, 46, 47};
+    nntrainer::Tensor answer(ml::train::TensorDim{3, 2, 4, 6}, answer_data);
+    EXPECT_EQ(nntrainer::Tensor::cat(inputs, 3), answer);
+  }
+}
 
 int main(int argc, char **argv) {
   int result = -1;