[context] Layer context creation scaffolding

Added todo list for the layer context creation
Added minor new interfaces for the layer context
Disable in-place optimization and inference memory optimization

Signed-off-by: Parichay Kapoor <pk.kapoor@samsung.com>

diff --git a/Applications/Custom/LayerClient/jni/Android.mk b/Applications/Custom/LayerClient/jni/Android.mk
index eb7400e..6e3972e 100644 (file)
--- a/Applications/Custom/LayerClient/jni/Android.mk
+++ b/Applications/Custom/LayerClient/jni/Android.mk
@@ -17,6 +17,7 @@ NNTRAINER_INCLUDES := $(NNTRAINER_ROOT)/nntrainer \
        $(NNTRAINER_ROOT)/nntrainer/dataset \
        $(NNTRAINER_ROOT)/nntrainer/models \
        $(NNTRAINER_ROOT)/nntrainer/layers \
+       $(NNTRAINER_ROOT)/nntrainer/graph \
        $(NNTRAINER_ROOT)/nntrainer/utils \
        $(NNTRAINER_ROOT)/nntrainer/optimizers \
        $(NNTRAINER_ROOT)/nntrainer/tensor \

diff --git a/nntrainer/graph/network_graph.cpp b/nntrainer/graph/network_graph.cpp
index 954ef4a..da2c051 100644 (file)
--- a/nntrainer/graph/network_graph.cpp
+++ b/nntrainer/graph/network_graph.cpp
@@ -647,6 +647,9 @@ void NetworkGraph::addLayer(std::shared_ptr<LayerNode> layer) {
 }
 
 void NetworkGraph::inPlaceOptimize(Manager &manager) {
+  // TODO: update this after initial verification, this is deprecated for now.
+  return;
+
   for (auto iter = cbegin(); iter != cend(); iter++) {
     auto layer_node = *iter;
     auto &l = layer_node->getObject();
@@ -737,8 +740,21 @@ void NetworkGraph::inPlaceOptimize(Manager &manager) {
   }
 }
 
+void NetworkGraph::init2runContext(InitLayerContext &init_context,
+                                   RunLayerContext &run_context) {
+  // NOTE: this just create the wrappers and does not actually memory inside
+  // these wrappers
+  // TODO: create wrappers for weights - initialize is already done, so the
+  // object creation can be done
+  // TODO: create wrapper for the temporary tensors
+  // TODO: create wrappers for outputs
+  // TODO: create a new context with these new wrappers and then copy assign
+  // run_context
+}
+
 int NetworkGraph::initialize(std::shared_ptr<Manager> manager) {
   int status = ML_ERROR_NONE;
+  // TODO: don't delete adj list - use it here and make this more cleaner/faster
 
   for (unsigned int idx = 0; idx < graph.size(); ++idx) {
     bool first = idx == 0;
@@ -772,6 +788,7 @@ int NetworkGraph::initialize(std::shared_ptr<Manager> manager) {
           }
         }
 
+        // TODO: set init layer context init layer
         lptr->setInputDimension(
           in_layer_node->getObject()->getOutputDimension()[location], i);
       }
@@ -781,14 +798,18 @@ int NetworkGraph::initialize(std::shared_ptr<Manager> manager) {
      * Initialize all the layers, allocate output tensors for each layer
      * and add optimizer related weights for the layer
      */
+    // TODO: pass init context, this call will fill it
     status = lptr->initialize(*manager);
     NN_RETURN_STATUS();
 
+    // TODO: call init2runContext
     auto &in_out = manager->trackLayerOutputs(cur_type, lnode->getName(),
                                               lptr->getOutputDimension(),
                                               lptr->getInputDimension());
+    // TODO: remove this
     lptr->setOutputBuffers(in_out);
 
+    // TODO: fill runcontext of other guys as well
     /** Connect the output of the previous layers with the input of the current
      * layer */
     if (!first) {
@@ -808,6 +829,9 @@ int NetworkGraph::initialize(std::shared_ptr<Manager> manager) {
           getLayerNode(input_layers[i])->getObject()->net_hidden[location];
       }
     } else {
+      // TODO: remove this, input buffers are created by either by the dataset
+      // or given by the user in the inference. Same for the label. So, need not
+      // create these as a special case.
       auto &in_out = manager->trackLayerInputs(cur_type, lnode->getName(),
                                                lptr->getInputDimension(),
                                                lptr->getOutputDimension());

diff --git a/nntrainer/graph/network_graph.h b/nntrainer/graph/network_graph.h
index 4722174..49b392f 100644 (file)
--- a/nntrainer/graph/network_graph.h
+++ b/nntrainer/graph/network_graph.h
@@ -268,6 +268,15 @@ public:
    */
   int initialize(std::shared_ptr<Manager> manager);
 
+  /**
+   * @brief Create run layer context from the given init layer context
+   *
+   * @param init_context Init layer context to create run context
+   * @param run_context Run layer context to be created
+   */
+  void init2runContext(InitLayerContext &init_context,
+                       RunLayerContext &run_context);
+
 private:
   std::map<std::string, std::string> sub_in_out; /** This is map to identify
                    input and output layer name of subgraph */

diff --git a/nntrainer/layers/layer_context.h b/nntrainer/layers/layer_context.h
index 5e0b31f..f1f373c 100644 (file)
--- a/nntrainer/layers/layer_context.h
+++ b/nntrainer/layers/layer_context.h
@@ -72,7 +72,9 @@ public:
    *
    * @return std::vector<TensorDim>& Output dimensions
    */
-  std::vector<TensorDim> &getOutputDimensions() { return input_dim; }
+  const std::vector<TensorDim> &getOutputDimensions() const {
+    return output_dim;
+  }
 
   /**
    * @brief Set the Output Dimensions object
@@ -117,7 +119,6 @@ public:
     return tensors_spec.size() - 1;
   }
 
-private:
   /**
    * @brief Specification of the weight
    *
@@ -132,6 +133,21 @@ private:
    */
   typedef std::tuple<const TensorDim, bool, const std::string> TensorSpec;
 
+  /**
+   * @brief Get the current weights spec
+   *
+   * @return The current weights spec
+   */
+  const std::vector<WeightSpec> &getWeightsSpec() const { return weights_spec; }
+
+  /**
+   * @brief Get the current tensors spec
+   *
+   * @return The current tensors spec
+   */
+  const std::vector<TensorSpec> &getTensorsSpec() const { return tensors_spec; }
+
+private:
   std::vector<TensorDim> input_dim;  /**< Input dimensions for the layer */
   std::vector<TensorDim> output_dim; /**< Output dimensions for the layer */
 
@@ -159,6 +175,25 @@ public:
   RunLayerContext() = default;
 
   /**
+   * @brief Construct a new Run Layer Context object
+   *
+   * @param props properties of the layer
+   * @param w weights of the layer
+   * @param in inputs of the layer
+   * @param out outputs of the layer
+   * @param t extra tensors of the layer
+   */
+  RunLayerContext(std::tuple<props::Name> p, const std::vector<Weight *> &w,
+                  const std::vector<Var_Grad *> &in,
+                  const std::vector<Var_Grad *> &out,
+                  const std::vector<Var_Grad *> &t) :
+    props(p),
+    weights(w),
+    inputs(in),
+    outputs(out),
+    tensors(t) {}
+
+  /**
    * @brief Get the Weight tensor object
    *
    * @param idx Identifier of the weight
@@ -274,7 +309,8 @@ public:
   bool getTrainable() { return trainable; }
 
 private:
-  bool trainable; /**< if the layer is trainable */
+  std::tuple<props::Name> props; /**< props of the layer */
+  bool trainable;                /**< if the layer is trainable */
 
   std::vector<Weight *> weights;   /**< weights of the layer */
   std::vector<Var_Grad *> inputs;  /**< inputs of the layer */

diff --git a/nntrainer/layers/layer_node.h b/nntrainer/layers/layer_node.h
index 7fe64f1..1016077 100644 (file)
--- a/nntrainer/layers/layer_node.h
+++ b/nntrainer/layers/layer_node.h
@@ -277,7 +277,9 @@ public:
 private:
   // TODO: make this unique_ptr once getObject API is removed
   std::shared_ptr<nntrainer::LayerV1>
-    layer;      /**< The actual object in the graph node */
+    layer; /**< The actual object in the graph node */
+  // TODO: possibly remove, two identifiers for the same  node (name and index)
+  // can lead to issues later
   size_t index; /**< index of each node */
 
   /** TODO : move management of num_inputs to layer_node */
@@ -290,7 +292,10 @@ private:
 
   RunLayerContext
     run_context; /**< context required for running/execution of the layer. This
-                    will also contain the properties of the layer. */
+                    will also contain the properties of the layer. The
+                    properties will be copied upon final creation. Editing
+                    properties of the layer after init will not the properties
+                    in the context/graph unless intended. */
   InitLayerContext init_context; /**< context to be built for/while
                                     initialization of the layer. This will also
                                     contain the properties of the layer. */

diff --git a/nntrainer/tensor/manager.h b/nntrainer/tensor/manager.h
index a79f999..c230bb5 100644 (file)
--- a/nntrainer/tensor/manager.h
+++ b/nntrainer/tensor/manager.h
@@ -25,6 +25,8 @@
 #include <memory>
 #include <vector>
 
+#include <graph_node.h>
+#include <layer_context.h>
 #include <var_grad.h>
 #include <weight.h>
 
@@ -109,7 +111,7 @@ public:
    */
   Manager(bool enable_gradient_memory_opt_ = true,
           bool enable_derivative_memory_opt_ = true,
-          bool enable_activation_memory_opt_ = true,
+          bool enable_activation_memory_opt_ = false,
           bool enable_inference_inout_memory_opt_ = false);
 
   /**
@@ -157,6 +159,41 @@ public:
   void trackWeights(std::vector<Weight> &ws);
 
   /**
+   * @brief     Create weights with the given spec
+   *
+   * @param w   node Graph node to extract node identifiers/info
+   * @param w   weights_spec Specficiation for the weights
+   */
+  std::vector<Weight *>
+  requestWeights(const GraphNode &node,
+                 std::vector<InitLayerContext::WeightSpec> &weights_spec);
+
+  /**
+   * @brief     Create tensors with the given spec
+   *
+   * @param w   create tensors list
+   */
+  std::vector<Var_Grad *>
+  requestTensors(const GraphNode &node,
+                 std::vector<InitLayerContext::TensorSpec> &tensors_spec);
+
+  /**
+   * @brief     Create tensors with the given spec
+   *
+   * @param w   create tensors list
+   */
+  std::vector<Var_Grad *> requestInputs(const GraphNode &node,
+                                        std::vector<TensorDim> &inputs_spec);
+
+  /**
+   * @brief     Create tensors with the given spec
+   *
+   * @param w   create tensors list
+   */
+  std::vector<Var_Grad *> requestOutputs(const GraphNode &node,
+                                         std::vector<TensorDim> &outputs_spec);
+
+  /**
    * @brief     Get weights tracked with nntrainer
    *
    * @retval    list of weight references
@@ -186,6 +223,9 @@ public:
    * @param opt True to enable, else false
    */
   void setInPlaceActivationOptimization(bool opt) {
+    if (opt)
+      throw exception::not_supported(
+        "Inplace activation optimization is temporarily disabled");
     enable_activation_memory_opt = opt;
   }
 
@@ -194,6 +234,9 @@ public:
    * @param opt True to enable, else false
    */
   void setInferenceInOutMemoryOptimization(bool opt) {
+    if (opt)
+      throw exception::not_supported(
+        "Inference memory optimization is temporarily disabled");
     enable_inference_inout_memory_opt = opt;
   }
 
@@ -363,6 +406,16 @@ public:
 
 private:
   // TODO: ensure that names of these weights are unique
+
+  std::vector<std::vector<std::unique_ptr<Weight>>>
+    weights_v2; /**< weights for the layers */
+  std::vector<std::vector<std::unique_ptr<Weight>>>
+    inouts_v2; /**<
+              inputs/outputs for the layers */
+  /** NOTE: these tensors maybe split based on their lifespan later */
+  std::vector<std::vector<std::unique_ptr<Weight>>>
+    tensors_v2; /**< extra tensors required by the layers */
+
   /**< Weights of all the layer in the model to be managed */
   std::vector<std::vector<std::reference_wrapper<Weight>>> weights;
 

diff --git a/test/unittest/unittest_nntrainer_models.cpp b/test/unittest/unittest_nntrainer_models.cpp
index 7eadc1a..2978714 100644 (file)
--- a/test/unittest/unittest_nntrainer_models.cpp
+++ b/test/unittest/unittest_nntrainer_models.cpp
@@ -377,9 +377,10 @@ GraphWatcher::GraphWatcher(const std::string &config, const bool opt) :
   /** Disable gradient optimization as gradient is being matched for each layer
    */
   nn.setGradientMemoryOptimization(optimize);
-  nn.setDerivativeMemoryOptimization(optimize);
-  nn.setInPlaceLayerOptimization(optimize);
-  nn.setInferenceInOutMemoryOptimization(optimize);
+  // TODO: update to use optimize after #986
+  nn.setDerivativeMemoryOptimization(false);
+  nn.setInPlaceLayerOptimization(false);
+  nn.setInferenceInOutMemoryOptimization(false);
 
   if (nn.loadFromConfig(config)) {
     throw std::invalid_argument("load from config failed!");