[tensor] Tensor constructor should not set zero

Tensor constructors should not zero to the memory by default
This hides many bugs in the code and also leads to big overheads
Removed setting zero in constructor and corresponding many bug fixes

Signed-off-by: Parichay Kapoor <pk.kapoor@samsung.com>

diff --git a/nntrainer/include/tensor.h b/nntrainer/include/tensor.h
index 36146593412862228ffea8162ab4ea070ea7e071..165d84f670468c8153d3f3a7345d394b423f0437 100644 (file)
--- a/nntrainer/include/tensor.h
+++ b/nntrainer/include/tensor.h
@@ -48,27 +48,7 @@ class LazyTensor;
  */
 class Tensor {
 public:
-  Tensor(const TensorDim &d, float *buf = nullptr) :
-    dim(d),
-    strides{{1, 2, 3}},
-    is_contiguous(true),
-    data(new float[d.getDataLen()], std::default_delete<float[]>()) {
-    // todo: initialize appropriate strides
-    if (buf == nullptr) {
-      setZero();
-    } else {
-      float *data = getData();
-      unsigned int len = length();
-
-#ifdef USE_BLAS
-      cblas_scopy(len, buf, 1, data, 1);
-#else
-      for (unsigned int i = 0; i < len; ++i) {
-        data[i] = buf[i];
-      }
-#endif
-    }
-  }
+  Tensor(const TensorDim &d, float *buf = nullptr);
 
   /**
    * @brief     Basic Constructor of Tensor

diff --git a/nntrainer/src/conv2d_layer.cpp b/nntrainer/src/conv2d_layer.cpp
index 5d40a945c0ea7686b8088f10e7a80e98cf8169ab..e94e9a7cfc8654c399f436c56cadc8a59df51af9 100644 (file)
--- a/nntrainer/src/conv2d_layer.cpp
+++ b/nntrainer/src/conv2d_layer.cpp
@@ -43,11 +43,11 @@ int Conv2DLayer::initialize(bool last) {
 
     delK.push_back(
       Tensor(input_dim.batch(), Kdim.channel(), Kdim.height(), Kdim.width()));
-    delBias.push_back(Tensor(1, 1, 1, 1));
+    delBias.push_back(Tensor(input_dim.batch(), 1, 1, 1));
     filters.push_back(Knl);
     weights.push_back(Knl);
 
-    Tensor B(input_dim.batch(), 1, 1, 1);
+    Tensor B(1, 1, 1, 1);
     if (!bias_init_zero) {
       B.apply([&](float x) { return random(); });
     }
@@ -78,6 +78,11 @@ void Conv2DLayer::save(std::ofstream &file) {
 }
 
 Tensor Conv2DLayer::forwarding(Tensor in, int &status) {
+  if (in.getDim() != input_dim) {
+    status = ML_ERROR_INVALID_PARAMETER;
+    return in;
+  }
+
   if (normalization) {
     input = in.normalization();
   } else {
@@ -88,8 +93,7 @@ Tensor Conv2DLayer::forwarding(Tensor in, int &status) {
     input = input.standardization();
   }
 
-  hidden = Tensor(in.batch(), output_dim.channel(), output_dim.height(),
-                  output_dim.width());
+  hidden = Tensor(output_dim);
 
   std::vector<float> output;
 

diff --git a/nntrainer/src/pooling2d_layer.cpp b/nntrainer/src/pooling2d_layer.cpp
index 1228df95e3ec30997566af752f9900dcd8ae97ac..7756b18a4a577d1b32c13460f9cbb0e59cd09d08 100644 (file)
--- a/nntrainer/src/pooling2d_layer.cpp
+++ b/nntrainer/src/pooling2d_layer.cpp
@@ -78,6 +78,7 @@ Tensor Pooling2DLayer::backwarding(Tensor derivative, int iteration) {
 
   unsigned int J, K;
   Tensor result = Tensor(input_dim);
+  result.setZero();
   float *out = result.getData();
   switch (pooling_type) {
   case PoolingType::max: {
@@ -290,6 +291,7 @@ Tensor Pooling2DLayer::pooling2d(unsigned int batch, Tensor in, int &status) {
     }
   } break;
   case PoolingType::global_max: {
+    output.setZero();
     for (unsigned int i = 0; i < channel; ++i) {
       unsigned int idx = batch * input_dim.getFeatureLen() + i * height * width;
       float max = std::numeric_limits<float>::min();
@@ -306,6 +308,7 @@ Tensor Pooling2DLayer::pooling2d(unsigned int batch, Tensor in, int &status) {
     }
   } break;
   case PoolingType::global_average: {
+    output.setZero();
     Tensor sum_wh = in.chain().sum(3).sum(2).run();
     for (unsigned int i = 0; i < channel; ++i) {
       output.setValue(0, i, 0, 0,

diff --git a/nntrainer/src/tensor.cpp b/nntrainer/src/tensor.cpp
index 44f697a87090230dcfb373b40e624f9568bb3217..0875a8bbb7b759581971460a39c85a2dae6ea0f0 100644 (file)
--- a/nntrainer/src/tensor.cpp
+++ b/nntrainer/src/tensor.cpp
@@ -61,22 +61,34 @@ static auto rng = [] {
   return rng;
 }();
 
-Tensor &Tensor::operator=(const Tensor &rhs) {
-  using std::swap;
+Tensor::Tensor(const TensorDim &d, float *buf) :
+  dim(d),
+  strides{{1, 2, 3}},
+  is_contiguous(true),
+  data(new float[d.getDataLen()], std::default_delete<float[]>()) {
+  // todo: initialize appropriate strides
+  if (buf != nullptr) {
+    float *data = getData();
+    unsigned int len = length();
 
+#ifdef USE_BLAS
+    cblas_scopy(len, buf, 1, data, 1);
+#else
+    for (unsigned int i = 0; i < len; ++i) {
+      data[i] = buf[i];
+    }
+#endif
+  }
+}
+
+Tensor &Tensor::operator=(const Tensor &rhs) {
   Tensor tmp(rhs);
-  swap(*this, tmp);
+  this->swap(*this, tmp);
   return *this;
 }
 
 Tensor &Tensor::operator=(Tensor &&rhs) noexcept {
-  using std::swap;
-
-  std::swap(dim, rhs.dim);
-  std::swap(data, rhs.data);
-  std::swap(strides, rhs.strides);
-  std::swap(is_contiguous, rhs.is_contiguous);
-
+  this->swap(*this, rhs);
   return *this;
 }
 
@@ -210,8 +222,8 @@ int Tensor::add_i(float const &value) {
   unsigned int len = length();
 #ifdef USE_BLAS
   Tensor tmp(dim);
-  tmp.setValue(1.0);
-  cblas_saxpy(len, value, tmp.getData(), 1, data, 1);
+  tmp.setValue(value);
+  cblas_saxpy(len, 1, tmp.getData(), 1, data, 1);
 #else
   for (unsigned int k = 0; k < len; ++k) {
     data[k] += value;
@@ -244,12 +256,15 @@ int Tensor::add_i(Tensor const &m, float const alpha) {
   unsigned int len = length();
 
 #ifdef USE_BLAS
-  unsigned int size = dim.width() * dim.height() * dim.channel();
+  unsigned int size = dim.getFeatureLen();
   if (m.dim.batch() == 1) {
     for (unsigned int k = 0; k < dim.batch(); ++k) {
       cblas_saxpy(size, alpha, mdata, 1, &(data[k * size]), 1);
     }
   } else {
+    if (dim.batch() != m.dim.batch()) {
+      return ML_ERROR_INVALID_PARAMETER;
+    }
     cblas_saxpy(len, alpha, mdata, 1, data, 1);
   }
 #else
@@ -262,6 +277,9 @@ int Tensor::add_i(Tensor const &m, float const alpha) {
       }
     }
   } else {
+    if (dim.batch() != m.dim.batch()) {
+      return ML_ERROR_INVALID_PARAMETER;
+    }
     for (k = 0; k < len; ++k) {
       data[k] += alpha * mdata[k];
     }
@@ -272,12 +290,9 @@ int Tensor::add_i(Tensor const &m, float const alpha) {
 }
 
 Tensor Tensor::add(Tensor const &m, float const alpha) const {
-  if ((dim.height() != m.dim.height()) || (dim.width() != m.dim.width())) {
-    throw std::runtime_error("Error: Dimension must be equal each other");
-  }
-
   Tensor result(*this);
-  result.add_i(m, alpha);
+  if (result.add_i(m, alpha) != ML_ERROR_NONE)
+    throw std::runtime_error("Error: Dimension must be equal each other");
 
   return result;
 }
@@ -496,6 +511,7 @@ Tensor Tensor::sum(int axis) const {
   switch (axis) {
   case 0: {
     ret = Tensor(1, dim.channel(), dim.height(), dim.width());
+    ret.setZero();
     float *rdata = ret.getData();
     for (unsigned int l = 0; l < dim.channel(); ++l) {
       unsigned int L = l * dim.width() * dim.height();
@@ -512,6 +528,7 @@ Tensor Tensor::sum(int axis) const {
   } break;
   case 1: {
     ret = Tensor(dim.batch(), 1, dim.height(), dim.width());
+    ret.setZero();
     float *rdata = ret.getData();
     for (unsigned int l = 0; l < dim.batch(); ++l) {
       unsigned int L = dim.width() * dim.height() * l;
@@ -529,6 +546,7 @@ Tensor Tensor::sum(int axis) const {
   } break;
   case 2: {
     ret = Tensor(dim.batch(), dim.channel(), 1, dim.width());
+    ret.setZero();
     float *rdata = ret.getData();
     for (unsigned int k = 0; k < dim.batch(); ++k) {
       unsigned int K = k * dim.channel() * dim.width();
@@ -547,6 +565,7 @@ Tensor Tensor::sum(int axis) const {
   } break;
   case 3: {
     ret = Tensor(dim.batch(), dim.channel(), dim.height(), 1);
+    ret.setZero();
     float *rdata = ret.getData();
     for (unsigned int k = 0; k < dim.batch(); ++k) {
       unsigned int K = k * dim.channel() * dim.height();
@@ -593,7 +612,7 @@ Tensor Tensor::dot(Tensor const &m) const {
 
 #ifdef USE_BLAS
   float alpha_dgemm = 1.0;
-  float beta_dgemm = 1.0;
+  float beta_dgemm = 0.0;
   if (m.dim.batch() == 1) {
     for (unsigned int k = 0; k < dim.batch(); k++) {
       unsigned int i = k * dim.width() * dim.height();

diff --git a/nntrainer/src/util_func.cpp b/nntrainer/src/util_func.cpp
index 50c20d16b1343986ce1336bbf043ba3827a0d6fe..19872f973cd5d8a27aec9f0c6dc6f340174c9036 100644 (file)
--- a/nntrainer/src/util_func.cpp
+++ b/nntrainer/src/util_func.cpp
@@ -174,6 +174,7 @@ Tensor zero_pad(int batch, Tensor const &in, unsigned int const *padding) {
   unsigned int width_p_h = w + padding[1];
 
   Tensor output(1, c, height_p, width_p);
+  output.setZero();
 
   for (unsigned int j = 0; j < c; ++j) {
     for (unsigned int k = 0; k < padding[0]; ++k) {
@@ -207,6 +208,8 @@ Tensor zero_pad(int batch, Tensor const &in, unsigned int const *padding) {
 Tensor strip_pad(Tensor const &in, unsigned int const *padding) {
   Tensor output(in.batch(), in.channel(), in.width() - padding[0] * 2,
                 in.width() - padding[1] * 2);
+  output.setZero();
+
   for (unsigned int i = 0; i < in.batch(); ++i) {
     for (unsigned int j = 0; j < in.channel(); ++j) {
       for (unsigned int k = 0; k < output.height(); ++k) {

diff --git a/test/unittest/unittest_nntrainer_tensor.cpp b/test/unittest/unittest_nntrainer_tensor.cpp
index ec8d030565ec67f2b39df7d1d4468c74f490c222..be56d010381e482013ebc6f7b3a5be9c8161eb26 100644 (file)
--- a/test/unittest/unittest_nntrainer_tensor.cpp
+++ b/test/unittest/unittest_nntrainer_tensor.cpp
@@ -59,6 +59,7 @@ TEST(nntrainer_TensorDim, setTensorDim_04_p) {
 TEST(nntrainer_Tensor, Tensor_01_p) {
   int status = ML_ERROR_NONE;
   nntrainer::Tensor tensor = nntrainer::Tensor(1, 2, 3);
+  tensor.setZero();
   ASSERT_NE(nullptr, tensor.getData());
   if (tensor.getValue(0, 0, 0, 0) != 0.0)
     status = ML_ERROR_INVALID_PARAMETER;