[bn] bug fix for setting optimizer for bn layer

This patch provides bugfix for setting the optiimzer for bn layer
although the weight updates are called only for the trainable params,
the optimizer is initialized with all the weights which are not trainable
and have no gradients
This results in un-necessary memory allocation for them as well as
creates issues for #517

This patch adds a count for trainable_param_size and a getTrainableParams() for the layer
and instead of all params, only trainable params of the layer are passed to the optimizer

An alternate was for the optimizer to check gradient size and ignore the ones with 0 size
but thats over-engineering at this point.

**Self evaluation:**
1. Build test: [x]Passed [ ]Failed [ ]Skipped
2. Run test: [x]Passed [ ]Failed [ ]Skipped

Signed-off-by: Parichay Kapoor <pk.kapoor@samsung.com>

diff --git a/nntrainer/include/optimizer.h b/nntrainer/include/optimizer.h
index 04f10cd..bf9047c 100644 (file)
--- a/nntrainer/include/optimizer.h
+++ b/nntrainer/include/optimizer.h
@@ -33,9 +33,10 @@ namespace nntrainer {
  * @brief UpdatableParam that could be updated thorugh optimizer
  */
 struct UpdatableParam {
-  Tensor weight;    /**<  weight to be updated and used */
-  Tensor grad;      /**<  gradient for the weight */
-  std::string name; /**< name of the parameter */
+  Tensor weight;         /**< weight to be updated and used */
+  Tensor grad;           /**< gradient for the weight */
+  std::string name;      /**< name of the parameter */
+  bool updatable = true; /**< if this param is updatable */
 };
 
 /**

diff --git a/nntrainer/src/bn_layer.cpp b/nntrainer/src/bn_layer.cpp
index 985cec1..6530717 100644 (file)
--- a/nntrainer/src/bn_layer.cpp
+++ b/nntrainer/src/bn_layer.cpp
@@ -53,8 +53,8 @@ int BatchNormalizationLayer::initialize() {
   beta.setZero();
 
   setParamSize(4);
-  paramsAt(0) = {std::move(mu), Tensor(), "BN:moving_average"};
-  paramsAt(1) = {std::move(var), Tensor(), "BN:moving_variance"};
+  paramsAt(0) = {std::move(mu), Tensor(), "BN:moving_average", false};
+  paramsAt(1) = {std::move(var), Tensor(), "BN:moving_variance", false};
   paramsAt(2) = {std::move(gamma), Tensor(gamma.getDim()), "BN:gamma"};
   paramsAt(3) = {std::move(beta), Tensor(beta.getDim()), "BN:beta"};
 
@@ -145,9 +145,7 @@ BatchNormalizationLayer::backwarding(sharedConstTensor derivative,
          .divide_i(cvar.multiply(batch))
          .run();
 
-  std::shared_ptr<UpdatableParam> grad_params(params, params.get() + 2);
-
-  opt.apply_gradients(grad_params, param_size - 2, iteration);
+  opt.apply_gradients(params, param_size, iteration);
 
   return MAKE_SHARED_TENSOR(std::move(dx));
 }

diff --git a/nntrainer/src/layer.cpp b/nntrainer/src/layer.cpp
index fc16e67..ffb4fbd 100644 (file)
--- a/nntrainer/src/layer.cpp
+++ b/nntrainer/src/layer.cpp
@@ -44,7 +44,7 @@ int Layer::setOptimizer(Optimizer &opt) {
   this->opt.setType(opt.getType());
   this->opt.setOptParam(opt.getOptParam());
 
-  return this->opt.initialize(getParams(), param_size, true);
+  return this->opt.initialize(params, param_size, true);
 }
 
 int Layer::checkValidation() {

diff --git a/nntrainer/src/optimizer.cpp b/nntrainer/src/optimizer.cpp
index 4062c69..0b4663e 100644 (file)
--- a/nntrainer/src/optimizer.cpp
+++ b/nntrainer/src/optimizer.cpp
@@ -68,6 +68,9 @@ int Optimizer::initialize(std::shared_ptr<UpdatableParam> params,
     for (unsigned int i = 0; i < param_size; ++i) {
       UpdatableParam &param = param_data[i];
 
+      if (!param.updatable)
+        continue;
+
       Tensor &weight = param.weight;
       Tensor &grad = param.grad;
       Tensor w = Tensor(weight.getDim());
@@ -105,6 +108,9 @@ void Optimizer::apply_gradients(std::shared_ptr<UpdatableParam> params,
   for (unsigned int i = 0; i < param_size; ++i) {
     UpdatableParam &param = param_data[i];
 
+    if (!param.updatable)
+      continue;
+
     Tensor &x = param.weight;
     const Tensor &x_grad = param.grad;
     switch (type) {