[ExecOrder] Add exec order control and fix inference

This patch add exec order control and reduce memory overhead of saving
inference result

exec order cotrol has two part

1. Additional exec order. This enable graph to add some exec order.
2. Expose bit. This enables request to be visible after the end of the
execution.

**Self evaluation:**
1. Build test: [X]Passed [ ]Failed [ ]Skipped
2. Run test: [X]Passed [ ]Failed [ ]Skipped

Signed-off-by: Jihoon Lee <jhoon.it.lee@samsung.com>

diff --git a/nntrainer/graph/network_graph.cpp b/nntrainer/graph/network_graph.cpp
index 37406a8..638974e 100644 (file)
--- a/nntrainer/graph/network_graph.cpp
+++ b/nntrainer/graph/network_graph.cpp
@@ -70,6 +70,7 @@ int NetworkGraph::compile(const std::string &loss_type) {
   graph.topologicalSort();
 
   setExecutionOrder();
+  forward_iter_end = (*(cend() - 1)).get();
 
   inPlaceOptimize();
 
@@ -247,8 +248,10 @@ void NetworkGraph::markNodesForBackwarding() {
   }
 
   /** mark all the required nodes support backwarding */
-  for (auto const &node_name : must_support_backwarding)
-    LNODE(graph.getNode(node_name))->needsCalcDerivative(true);
+  for (auto const &node_name : must_support_backwarding) {
+    auto ln = LNODE(graph.getNode(node_name)).get();
+    ln->needsCalcDerivative(true);
+  }
 }
 
 void NetworkGraph::setBatchSize(unsigned int batch_size) {
@@ -733,14 +736,18 @@ NetworkGraph::finalizeContext(const std::shared_ptr<LayerNode> &lnode,
       TensorSpecV2::RequestType::PLACEHOLDER;
   }
 
-  /// @todo switch to check if model inputs instead and add a new tensor life
-  /// span to represent from here to ~ max_fwd_exec_order, also this is only
-  /// needed for the inference mode
+  /// @note below needs to be enabled only for inference mode, but need decision
+  /// if we are going to separate inference initialization from train
+  /// initialization this might not worth optimize because in general output of
+  /// a neuralnet is very small
   if (lnode->getOutputConnections().size() == 0u) {
-    std::for_each(out_specs.begin(), out_specs.end(), [](VarGradSpecV2 &spec) {
-      spec.variable_spec.ls = TensorLifespan::MAX_LIFESPAN;
-    });
+    std::for_each(out_specs.begin(), out_specs.end(),
+                  [this](VarGradSpecV2 &spec) {
+                    spec.variable_spec.additional_exec_order.push_back(
+                      std::get<0>(forward_iter_end->getExecutionOrder()));
+                  });
   }
+
   const std::vector<Var_Grad *> &outputs = tensor_manager->requestTensors(
     out_specs, Manager::TensorGroupType::OUTPUT, lnode->getExecutionOrder(),
     lnode->getName());
@@ -987,7 +994,6 @@ int NetworkGraph::initialize(const std::vector<Connection> &model_input_names,
   try {
     markNodesForBackwarding();
     backward_iter_end = computeBackwardEnd();
-    forward_iter_end = (*(cend() - 1)).get();
   } catch (std::exception &e) {
     ml_loge(
       "Backwarding required from layer which doesn't support backwarding: %s",

diff --git a/nntrainer/tensor/manager.cpp b/nntrainer/tensor/manager.cpp
index 56ba148..e9e8ed5 100644 (file)
--- a/nntrainer/tensor/manager.cpp
+++ b/nntrainer/tensor/manager.cpp
@@ -154,7 +154,7 @@ static Tensor *requestTensor_(const TensorSpecV2 &spec,
 
   auto [forward, calc_grad, calc_deriv] = exec_order;
 
-  std::vector<unsigned> order;
+  std::vector<unsigned> order = spec.additional_exec_order;
 
   const auto name = scope + ":" + spec.name;
 
@@ -190,7 +190,8 @@ static Tensor *requestTensor_(const TensorSpecV2 &spec,
 Var_Grad *Manager::requestTensor(const VarGradSpecV2 &spec,
                                  TensorGroupType identify_as,
                                  const GraphNode::ExecutionOrder &exec_order,
-                                 const std::string &scope) {
+                                 const std::string &scope, bool expose_var,
+                                 bool expose_grad) {
   NNTR_THROW_IF(identify_as == TensorGroupType::WEIGHT, std::invalid_argument)
     << "requestTensor with var grad spec cannot be identified as weights, use "
        "requestTensor with weight spec instead";
@@ -217,11 +218,13 @@ Var_Grad *Manager::requestTensor(const VarGradSpecV2 &spec,
 
 std::vector<Var_Grad *> Manager::requestTensors(
   const std::vector<VarGradSpecV2> &specs, TensorGroupType identify_as,
-  const GraphNode::ExecutionOrder &exec_order, const std::string &scope) {
+  const GraphNode::ExecutionOrder &exec_order, const std::string &scope,
+  bool expose_var, bool expose_grad) {
   std::vector<Var_Grad *> ret;
   ret.reserve(specs.size());
   for (auto &spec : specs) {
-    ret.push_back(requestTensor(spec, identify_as, exec_order, scope));
+    ret.push_back(requestTensor(spec, identify_as, exec_order, scope,
+                                expose_var, expose_grad));
   }
 
   return ret;

diff --git a/nntrainer/tensor/manager.h b/nntrainer/tensor/manager.h
index 89d1117..a93af9b 100644 (file)
--- a/nntrainer/tensor/manager.h
+++ b/nntrainer/tensor/manager.h
@@ -407,12 +407,17 @@ public:
    * @param identify_as identify as tensor as a group
    * @param exec_order execution order to refer to
    * @param scope common scope to attach in front of current specification name
+   * @param expose_var expose variable tensor out of graph, when allocation,
+   * this tensor will be valid max_exec_order when allocation happens
+   * @param expose_grad expose variable tensor out of graph, this tensor will be
+   * valid max_exec_order when allocation happens
    * @return Tensor* tensor
    */
   Var_Grad *requestTensor(const VarGradSpecV2 &spec,
                           TensorGroupType identify_as,
                           const GraphNode::ExecutionOrder &exec_order,
-                          const std::string &scope = "");
+                          const std::string &scope = "",
+                          bool expose_var = false, bool expose_grad = false);
 
   /**
    * @brief request vector of tensors with variable + gradient specification
@@ -421,11 +426,17 @@ public:
    * @param identify_as identify as tensor as a group
    * @param exec_order execution order to refer to
    * @param scope common scope to attach in front of current specification name
+   * @param expose_var expose variable tensor out of graph, when
+   * allocation, this tensor will be valid max_exec_order when allocation
+   * happens
+   * @param expose_grad expose variable tensor out of graph, this tensor will be
+   * valid max_exec_order when allocation happens
    * @return Tensor* tensor
    */
   std::vector<Var_Grad *> requestTensors(
     const std::vector<VarGradSpecV2> &specs, TensorGroupType identify_as,
-    const GraphNode::ExecutionOrder &exec_order, const std::string &scope = "");
+    const GraphNode::ExecutionOrder &exec_order, const std::string &scope = "",
+    bool expose_var = false, bool expose_grad = false);
 
 private:
   /** @todo: merge this list to one */

diff --git a/nntrainer/tensor/tensor_wrap_specs.h b/nntrainer/tensor/tensor_wrap_specs.h
index fe17264..504215e 100644 (file)
--- a/nntrainer/tensor/tensor_wrap_specs.h
+++ b/nntrainer/tensor/tensor_wrap_specs.h
@@ -125,6 +125,10 @@ struct TensorSpecV2 {
   /** ONLY USED FOR READ_ONLY_VIEW, MAYBE_MODIFYING_VIEW */
   unsigned int offset = 0u;   /**< tensor offset */
   std::string reference_name; /**< reference name */
+
+  /** ONLY FOR THE GRANULAR CONTROL OF LIFE OUTSIDE OF LAYER NODE */
+  /// @todo make this as an opaque information with PIMPL
+  std::vector<unsigned> additional_exec_order = {};
 };
 
 /**