[Fix/Sharing] Fix gradient updation log

Gradient should be initialized at the very first (backward) access, but the
shared weight was updated at the very last (backward) access, this patch
resolves the issue by adding isFirstAccess to the var_grad.h

**Self evaluation:**
1. Build test: [X]Passed [ ]Failed [ ]Skipped
2. Run test: [X]Passed [ ]Failed [ ]Skipped

Signed-off-by: Jihoon Lee <jhoon.it.lee@samsung.com>

diff --git a/nntrainer/graph/network_graph.cpp b/nntrainer/graph/network_graph.cpp
index fd9bcf08ce35c88a0ca7405445e4c59f007be9d2..53b7abab8e87f0e346660c2f296ed6ef58bf34e2 100644 (file)
--- a/nntrainer/graph/network_graph.cpp
+++ b/nntrainer/graph/network_graph.cpp
@@ -815,6 +815,20 @@ int NetworkGraph::initialize(
     }
   }
 
+  for (unsigned int idx = 0; idx < graph.size(); ++idx) {
+    auto const &lnode = getSortedLayerNode(idx);
+    auto &rc = lnode->getRunContext();
+    auto first_grad_access = std::get<1>(lnode->getExecutionOrder());
+    for (unsigned i = 0; i < rc.getNumWeights(); ++i) {
+      if (!rc.weightHasGradient(i)) {
+        continue;
+      }
+      if (tensor_manager->isFirstAccess(rc.getWeightGrad(i).getName(),
+                                        first_grad_access)) {
+        rc.getWeightObject(i).setAsGradientFirstAccess();
+      }
+    }
+  }
   /**** identify model input / output to be set externally later ****/
   auto identify_as_model_input = [this](LayerNode *node) {
     auto num_input = node->getNumInputs();

diff --git a/nntrainer/layers/fc_layer.cpp b/nntrainer/layers/fc_layer.cpp
index 2a384920ecc2fbabafe6a965dc06d36f9894ed0e..9bbe00364fcaa88b8f79baeb231b2a2f091d7c3b 100644 (file)
--- a/nntrainer/layers/fc_layer.cpp
+++ b/nntrainer/layers/fc_layer.cpp
@@ -113,18 +113,18 @@ void FullyConnectedLayer::calcGradient(RunLayerContext &context) {
   Tensor &derivative_ = context.getIncomingDerivative(SINGLE_INOUT_IDX);
   Tensor &input_ = context.getInput(SINGLE_INOUT_IDX);
 
-  if (context.isWeightDependent(weight_idx[FCParams::bias])) {
+  if (context.isGradientFirstAccess(weight_idx[FCParams::bias])) {
+    derivative_.sum({0, 1, 2}, djdb);
+  } else {
     /// @todo optimize below by adding beta to Tensor::sum
     Tensor t = derivative_.sum({0, 1, 2});
     djdb.add_i(t);
-  } else {
-    derivative_.sum({0, 1, 2}, djdb);
   }
 
-  if (context.isWeightDependent(weight_idx[FCParams::weight])) {
-    input_.dot(derivative_, djdw, true, false, 1.0f);
-  } else {
+  if (context.isGradientFirstAccess(weight_idx[FCParams::weight])) {
     input_.dot(derivative_, djdw, true, false, 0.0f);
+  } else {
+    input_.dot(derivative_, djdw, true, false, 1.0f);
   }
 }
 

diff --git a/nntrainer/layers/layer_context.cpp b/nntrainer/layers/layer_context.cpp
index fc8fa7c0639f36686dfbd714ee7ede59bcb90ad5..f20a07ff227de7558680376dd94d6030e0790f8a 100644 (file)
--- a/nntrainer/layers/layer_context.cpp
+++ b/nntrainer/layers/layer_context.cpp
@@ -258,6 +258,10 @@ bool RunLayerContext::isWeightDependent(unsigned int idx) const {
   return weights[idx]->isDependent();
 }
 
+bool RunLayerContext::isGradientFirstAccess(unsigned int idx) const {
+  return weights[idx]->isGradientFirstAccess();
+}
+
 /**
  * @brief Get the tensor name
  *

diff --git a/nntrainer/layers/layer_context.h b/nntrainer/layers/layer_context.h
index 67ff6dc7d41e2f0660a717d0327b03fcdfceb7c5..f4dbea324f63ccc7834b5e72f43724daba6e531e 100644 (file)
--- a/nntrainer/layers/layer_context.h
+++ b/nntrainer/layers/layer_context.h
@@ -503,6 +503,14 @@ public:
    */
   bool isWeightDependent(unsigned int idx) const;
 
+  /**
+   * @brief check current graident is first access
+   *
+   * @param idx index
+   * @return bool true if last access
+   */
+  bool isGradientFirstAccess(unsigned int idx) const;
+
   /**
    * @brief Get the tensor name
    *

diff --git a/nntrainer/tensor/var_grad.h b/nntrainer/tensor/var_grad.h
index 9326f1353d4abfebe2adcf7a8e03068081b8f129..3aaf4bdcc7db18e5ec250e809944cbf042aa9f01 100644 (file)
--- a/nntrainer/tensor/var_grad.h
+++ b/nntrainer/tensor/var_grad.h
@@ -87,6 +87,7 @@ public:
   explicit Var_Grad(const Tensor &v, const Tensor &g, const std::string &n = "",
                     bool is_dependent = false) :
     is_dependent(is_dependent),
+    is_first_access_gradient(false),
     var(
       std::make_shared<Tensor>(v.getSharedDataTensor(v.getDim(), 0, false, n))),
     grad(std::make_shared<Tensor>(n + grad_suffix)) {
@@ -104,6 +105,7 @@ public:
    */
   explicit Var_Grad(Tensor *v, Tensor *g, bool is_dependent = false) :
     is_dependent(is_dependent),
+    is_first_access_gradient(false),
     var(std::shared_ptr<Tensor>(v, [](void *) {})),
     grad(std::shared_ptr<Tensor>(g, [](void *) {})) {
     if (!v)
@@ -259,11 +261,27 @@ public:
    */
   bool isDependent() const { return is_dependent; }
 
+  /**
+   * @brief Set the As First Gradient Access
+   *
+   */
+  void setAsGradientFirstAccess() { is_first_access_gradient = true; }
+
+  /**
+   * @brief check if given weight at the last execution order
+   * (first access of gradient)
+   *
+   * @return bool true if last access
+   */
+  bool isGradientFirstAccess() const { return is_first_access_gradient; }
+
   inline static const std::string grad_suffix = ":grad";
 
 protected:
   bool is_dependent; /**< check if the weight tensor is burrowed from somewhere
                         thus it is dependent */
+  bool is_first_access_gradient; /**< check if current weight tensor is last
+                                    access */
 
   std::shared_ptr<Tensor> var;  /**< variable to be updated and used */
   std::shared_ptr<Tensor> grad; /**< gradient for the variable */