[ Graph ] Add Graph into Network

Describe a commit content (Until 80 colums per line) in detail ASAP.

**Changes proposed in this PR:**
- Added TOC generator for README.md

Resolves:

**Self evaluation:**
1. Build test:	 [X]Passed [ ]Failed [ ]Skipped
2. Run test:	 [X]Passed [ ]Failed [ ]Skipped

Signed-off-by: jijoong.moon <jijoong.moon@samsung.com>

diff --git a/nntrainer/graph/network_graph.cpp b/nntrainer/graph/network_graph.cpp
index 46a93dfa2914576e95605424ae6234b8ccae1f27..83db22649976ee0225ed20088f7ba978ae4f9929 100644 (file)
--- a/nntrainer/graph/network_graph.cpp
+++ b/nntrainer/graph/network_graph.cpp
@@ -409,7 +409,10 @@ int NetworkGraph::setGraphNode(std::vector<std::shared_ptr<Layer>> layers,
     }
   }
 
-  addLossLayer(loss_type);
+  if (layers.back()->getType() != LayerType::LAYER_LOSS) {
+    status = addLossLayer(loss_type);
+    NN_RETURN_STATUS();
+  }
 
   // std::list<LayerNode>::iterator iter;
   // for (unsigned int i = 0; i < adj.size(); ++i) {

diff --git a/nntrainer/graph/network_graph.h b/nntrainer/graph/network_graph.h
index 9c0ddffa3b93214535b742e681a21572ac9fa16c..69418d8838f76c566bcc230b559adca8bfd0ad04 100644 (file)
--- a/nntrainer/graph/network_graph.h
+++ b/nntrainer/graph/network_graph.h
@@ -84,6 +84,8 @@ public:
 
   void backwarding(sharedConstTensors input, int iteration);
 
+  std::vector<LayerNode> getSorted() { return Sorted; }
+
 private:
   void topologicalSortUtil(unsigned int ith, bool visited[],
                            std::stack<LayerNode> &Stack);

diff --git a/nntrainer/layers/layer.cpp b/nntrainer/layers/layer.cpp
index 0f8220e7b96591ce7f232aae211c20aabca71988..40b56b3966542f8c4360ba8f94f1c395f5fa13be 100644 (file)
--- a/nntrainer/layers/layer.cpp
+++ b/nntrainer/layers/layer.cpp
@@ -61,6 +61,22 @@ void Layer::setBatch(unsigned int batch) {
     output_dim[idx].setTensorDim(0, batch);
 }
 
+std::vector<Tensor> Layer::getHidden() {
+  std::vector<Tensor> ret;
+  for (unsigned int i = 0; i < num_outputs; ++i) {
+    ret.push_back(net_hidden[i]->var);
+  }
+  return ret;
+}
+
+std::vector<Tensor> Layer::getGradient() {
+  std::vector<Tensor> ret;
+  for (unsigned int i = 0; i < num_inputs; ++i) {
+    ret.push_back(net_input[i]->grad);
+  }
+  return ret;
+}
+
 void Layer::copy(std::shared_ptr<Layer> l) {
   setNumWeights(l->num_weights);
   for (unsigned int i = 0; i < num_weights; ++i) {
@@ -84,6 +100,74 @@ void Layer::copy(std::shared_ptr<Layer> l) {
   this->num_outputs = l->num_outputs;
 }
 
+sharedConstTensors Layer::forwarding_with_val(sharedConstTensors input,
+                                   sharedConstTensors in) {
+
+  for(unsigned int i=0;i<num_inputs;++i){
+    net_input[i]->var = *input[i];
+  }
+
+  if (num_outputs == 0)
+    throw("invalid number of outputs");
+
+  if (num_outputs != net_hidden.size())
+    net_hidden.resize(num_outputs);
+
+  // for (unsigned int i = 0; i < num_outputs; ++i) {
+  //   sharedNetBuffer h_buffer = std::make_unique<nntrainer::NetBuffers>();
+  //   h_buffer->var = Tensor(getOutputDimension()[i]);
+  //   h_buffer->grad = Tensor(getOutputDimension()[i]);
+  //   net_hidden[i] = h_buffer;
+  // }
+
+  forwarding(in);
+
+  nntrainer::sharedConstTensors out;
+
+  for(unsigned int i =0; i<num_outputs;++i){
+    out.push_back(MAKE_SHARED_TENSOR(net_hidden[i]->var));
+  }
+
+  return out;
+}
+
+sharedConstTensors Layer::backwarding_with_val(int iteration,
+                                      sharedConstTensors deriv,
+                                      sharedConstTensors in) {
+
+  for (unsigned int i = 0; i < num_outputs; ++i) {
+    net_hidden[i]->grad = *deriv[i];
+  }
+
+  if (num_inputs == 0)
+    throw("invalid number of inputs");
+
+  if (num_inputs != net_input.size())
+    net_input.resize(num_inputs);
+
+  // for (unsigned int i = 0; i < num_inputs; ++i) {
+  //   sharedNetBuffer h_buffer = std::make_unique<nntrainer::NetBuffers>();
+  //   h_buffer->var = Tensor(getInputDimension()[i]);
+  //   h_buffer->grad = Tensor(getInputDimension()[i]);
+  //   net_input[i] = h_buffer;
+  // }
+
+  if(getType() == nntrainer::LayerType::LAYER_LOSS){
+    backwarding(iteration, in);    
+  } else {
+    backwarding(iteration, deriv);
+  }
+
+  nntrainer::sharedConstTensors out;
+
+  for(unsigned int i =0; i<num_inputs;++i){
+    out.push_back(MAKE_SHARED_TENSOR(net_input[i]->grad));
+  }
+
+  return out;
+}
+  
+
 void Layer::read(std::ifstream &file) {
   for (unsigned int i = 0; i < num_weights; ++i) {
     weightAt(i).getVariableRef().read(file);

diff --git a/nntrainer/layers/layer_internal.h b/nntrainer/layers/layer_internal.h
index bb53488738cb943d1c1d3e28aaf28795be831d08..9579e028a97b2cc8fe66e9897e9e60b0a07e8f0d 100644 (file)
--- a/nntrainer/layers/layer_internal.h
+++ b/nntrainer/layers/layer_internal.h
@@ -40,6 +40,9 @@ struct NetBuffers {
   Tensor grad;
 };
 
+typedef std::shared_ptr<nntrainer::NetBuffers> sharedNetBuffer;
+typedef std::vector<sharedNetBuffer> sharedNetBuffers;
+
 /**
  * @brief     Enumeration of activation function type
  */
@@ -110,6 +113,9 @@ public:
    */
   virtual void forwarding(sharedConstTensors in = {}) = 0;
 
+  virtual sharedConstTensors forwarding_with_val(sharedConstTensors input,
+                                      sharedConstTensors in = {});
+
   /**
    * @brief     Back Propagation of a layer
    * @param[in] in List of Derivative Tensor from the next layer
@@ -118,6 +124,10 @@ public:
    */
   virtual void backwarding(int iteration, sharedConstTensors in = {}) = 0;
 
+  virtual sharedConstTensors backwarding_with_val(int iteration, 
+                                         sharedConstTensors deriv,
+                                         sharedConstTensors in = {});
+
   /**
    * @brief     read layer Weight & Bias data from file
    * @note      derived class can call this to get/save weights
@@ -301,6 +311,29 @@ public:
     output_dim.resize(num_outputs);
   }
 
+  std::vector<Tensor> getHidden();
+
+  std::vector<Tensor> getGradient();
+
+  void resizeNetInput(unsigned int size){net_input.resize(size);}
+  
+  void resizeNetOutput(unsigned int size){net_hidden.resize(size);}
+
+  unsigned int getNumInputs(){return num_inputs;}
+  unsigned int getNumOutputs(){return num_outputs;}
+
+  void setInputBuffer(unsigned int i, std::shared_ptr<NetBuffers> n_buffer) {
+    if (i >= net_input.size())
+      throw std::invalid_argument("Error: exceed num_input size");
+    net_input[i] = n_buffer;
+  }
+
+  void setOutputBuffer(unsigned int i, std::shared_ptr<NetBuffers> n_buffer) {
+    if (i >= net_hidden.size())
+      throw std::invalid_argument("Error: exceed num_input size");
+    net_hidden[i] = n_buffer;
+  }
+
 protected:
   /**
    * @brief   Print Options when printing layer info

diff --git a/nntrainer/layers/loss_layer.cpp b/nntrainer/layers/loss_layer.cpp
index c5188444ac1cc5324c74082734f7ece31d4d3d40..d1ff8c1430818666c4a3ce4b77f6db63409213da 100644 (file)
--- a/nntrainer/layers/loss_layer.cpp
+++ b/nntrainer/layers/loss_layer.cpp
@@ -42,8 +42,33 @@ int LossLayer::initialize() {
   return status;
 }
 
+sharedConstTensors LossLayer::forwarding_with_val(sharedConstTensors in,
+                                         sharedConstTensors label) {
+
+  for(unsigned int i=0;i<num_inputs;++i){
+    net_input[i]->var = *in[i];
+  }
+
+  if (num_outputs == 0)
+    throw("invalid number of outputs");
+
+  if (num_outputs != net_hidden.size())
+    net_hidden.resize(num_outputs);
+  
+  forwarding(in, label);
+
+  nntrainer::sharedConstTensors out;
+
+  for(unsigned int i =0; i<num_outputs;++i){
+    out.push_back(MAKE_SHARED_TENSOR(net_hidden[i]->var));
+  }
+
+  return out;  
+}  
+
 sharedConstTensors LossLayer::forwarding(sharedConstTensors in,
                                          sharedConstTensors label) {
+  net_input[0]->var = *in[0];
   Tensor y2 = *label[0];
   Tensor &y = net_hidden[0]->var;
   y = net_input[0]->var;

diff --git a/nntrainer/layers/loss_layer.h b/nntrainer/layers/loss_layer.h
index 3c9ef96a59ab16d4bbf47407c498d99e46d0df83..db7cc1dab8b6d69910b4b357e3329cc05dd256c9 100644 (file)
--- a/nntrainer/layers/loss_layer.h
+++ b/nntrainer/layers/loss_layer.h
@@ -66,6 +66,9 @@ public:
   sharedConstTensors forwarding(sharedConstTensors in,
                                 sharedConstTensors label);
 
+  sharedConstTensors forwarding_with_val(sharedConstTensors in,
+                                                 sharedConstTensors label);  
+
   /**
    * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
    */

diff --git a/nntrainer/models/neuralnet.cpp b/nntrainer/models/neuralnet.cpp
index e6edf3286afe1262d1d67801398ba30a0f590f6c..b1e1929cb71c8cc9695399ceffa064b0196e5d13 100644 (file)
--- a/nntrainer/models/neuralnet.cpp
+++ b/nntrainer/models/neuralnet.cpp
@@ -199,6 +199,7 @@ int NeuralNetwork::compile() {
   NN_RETURN_STATUS();
 
   model_graph.topologicalSort();
+  setBatchSize(batch_size);
 
   return status;
 }
@@ -253,6 +254,12 @@ int NeuralNetwork::initialize() {
         model_graph.getSortedLayerNode(l.input_layers[i])
           .layer->net_hidden[location] = n_buffer;
       }
+    } else {
+      for (unsigned int i = 0; i < l.input_layers.size(); ++i) {
+        std::cout << "      " << l.input_layers[i];
+        std::shared_ptr<NetBuffers> n_buffer = std::make_unique<NetBuffers>();
+        l.net_input[i] = n_buffer;
+      }
     }
 
     std::cout << std::endl;
@@ -277,7 +284,7 @@ int NeuralNetwork::initialize() {
       Tensor(model_graph.Sorted.back().layer->getOutputDimension()[i]);
     model_graph.Sorted.back().layer->net_hidden[i] = last_hidden_buffer;
   }
-
+  initialized = true;
   return status;
 }
 
@@ -552,7 +559,7 @@ int NeuralNetwork::train(std::vector<std::string> values) {
   NN_RETURN_STATUS();
 
   /** set batch size just before training */
-  setBatchSize(batch_size);
+  // setBatchSize(batch_size);
 
   /** Setup data buffer properties */
   status = data_buffer->setClassNum(getOutputDimension()[0].width());

diff --git a/nntrainer/models/neuralnet.h b/nntrainer/models/neuralnet.h
index 80541d53eac2201097f0d3a56485ae4b52242d2d..8ae964d6dfc87be7d00b0df7ef87a93159f24b25 100644 (file)
--- a/nntrainer/models/neuralnet.h
+++ b/nntrainer/models/neuralnet.h
@@ -75,6 +75,7 @@ public:
   using GraphType = std::vector<NodeType>; /** actual graph type */
   using FlatGraphType =
     std::vector<NodeType>; /** topological sorted, iterable 1-D list of nodes */
+  using NetworkGraphType = nntrainer::NetworkGraph;
 
   /**
    * @brief     Constructor of NeuralNetwork Class
@@ -309,6 +310,8 @@ public:
    */
   FlatGraphType getFlatGraph() { return layers; }
 
+  NetworkGraphType getNetworkGraph() { return model_graph; }
+
   /**
    * @brief get current graph from the model
    * @note graph contains pointer to the actual nodes, which is not deeply

diff --git a/nntrainer/tensor/tensor.cpp b/nntrainer/tensor/tensor.cpp
index 14424122fc4aa15bd19ed0502c96a3f6621ad054..3481ab00137aa7a9aabfa4d3cba631e07cea27c8 100644 (file)
--- a/nntrainer/tensor/tensor.cpp
+++ b/nntrainer/tensor/tensor.cpp
@@ -636,7 +636,7 @@ void Tensor::print(std::ostream &out) const {
 
   out << dim;
 
-  if (len > 100) {
+  if (len > 10000000) {
     out << '[' << data[0] << ' ' << data[1] << ' ' << data[2] << " ... "
         << data[len - 3] << ' ' << data[len - 2] << ' ' << data[len - 1] << ']'
         << std::endl;

diff --git a/test/unittest/unittest_nntrainer_graph.cpp b/test/unittest/unittest_nntrainer_graph.cpp
index 72a2c31a3b08bd90fde60628b8c0605543d3f2e2..644337545ccd0b75083715be3058a1e8d7b41703 100644 (file)
--- a/test/unittest/unittest_nntrainer_graph.cpp
+++ b/test/unittest/unittest_nntrainer_graph.cpp
@@ -62,7 +62,7 @@ TEST_P(nntrainerGraphTest, loadConfig) {
   std::cout << std::get<0>(GetParam()) << std::endl;
   int status = NN.loadFromConfig(getIniName());
 
-  int batch = 32;
+  int batch = 16;
   int channel = 3;
   int height = 32;
   int width = 32;
@@ -94,16 +94,18 @@ TEST_P(nntrainerGraphTest, loadConfig) {
 
   NN.forwarding({MAKE_SHARED_TENSOR(input)});
 
-  nntrainer::Tensor output(10);
+  nntrainer::Tensor output(batch, 1, 1, 10);
 
   output.setZero();
-  output.setValue(0, 0, 0, 3, 1.0);
+
+  for (int i = 0; i < batch; ++i)
+    output.setValue(i, 0, 0, 3, 1.0);
 
   NN.backwarding({MAKE_SHARED_TENSOR(input)}, {MAKE_SHARED_TENSOR(output)}, 1);
 }
 
 static IniSection nw_base("model", "Type = NeuralNetwork | "
-                                   "batch_size = 32 | "
+                                   "batch_size = 16 | "
                                    "epsilon = 1e-7 | "
                                    "loss = cross");
 

diff --git a/test/unittest/unittest_nntrainer_layers.cpp b/test/unittest/unittest_nntrainer_layers.cpp
index 745d77cc07287fabe614235d90b1087f9429bc9e..bcc9496fae8ff82628742c75d3d76e40853b38f0 100644 (file)
--- a/test/unittest/unittest_nntrainer_layers.cpp
+++ b/test/unittest/unittest_nntrainer_layers.cpp
@@ -55,8 +55,29 @@ protected:
   virtual int reinitialize() {
     int status = layer.initialize();
     EXPECT_EQ(status, ML_ERROR_NONE);
+    
     in = nntrainer::Tensor(layer.getInputDimension()[0]);
     out = nntrainer::Tensor(layer.getOutputDimension()[0]);
+
+    layer.resizeNetInput(layer.getNumInputs());
+    layer.resizeNetOutput(layer.getNumOutputs());
+
+    for (unsigned int i = 0; i < layer.getNumInputs(); ++i) {
+      std::shared_ptr<nntrainer::NetBuffers> n_buffer =
+        std::make_unique<nntrainer::NetBuffers>();
+      n_buffer->var = nntrainer::Tensor(layer.getInputDimension()[i]);
+      n_buffer->grad = nntrainer::Tensor(layer.getInputDimension()[i]);
+      layer.setInputBuffer(i,n_buffer);
+    }
+
+    for (unsigned int i = 0; i < layer.getNumOutputs(); ++i) {
+      std::shared_ptr<nntrainer::NetBuffers> n_buffer =
+        std::make_unique<nntrainer::NetBuffers>();
+      n_buffer->var = nntrainer::Tensor(layer.getOutputDimension()[i]);
+      n_buffer->grad = nntrainer::Tensor(layer.getOutputDimension()[i]);
+      layer.setOutputBuffer(i,n_buffer);      
+    }
+
     return status;
   }
 
@@ -125,7 +146,7 @@ protected:
   }
 
   // setting optimizer property separated by "|"
-  int setOptimizer(const std::string &type, const std::string &str = "") {
+  int setOptimizer(nntrainer::OptType type, const std::string &str = "") {
     std::vector<std::string> input_str;
     std::regex words_regex("[^|]+");
     auto words_begin =
@@ -282,10 +303,10 @@ TEST_F(nntrainer_InputLayer, set_property_05_p) {
  * @brief Input Layer
  */
 TEST_F(nntrainer_InputLayer, setOptimizer_01_p) {
-  status = setOptimizer("adam", "learning_rate=0.001 |"
-                                "beta1=0.9 |"
-                                "beta2=0.9999 |"
-                                "epsilon=1e-7");
+  status = setOptimizer(nntrainer::OptType::ADAM, "learning_rate=0.001 |"
+                                                  "beta1=0.9 |"
+                                                  "beta2=0.9999 |"
+                                                  "epsilon=1e-7");
 
   EXPECT_EQ(status, ML_ERROR_NONE);
 }
@@ -397,10 +418,10 @@ TEST(nntrainer_FullyConnectedLayer_init_name, initialize_05_n) {
  * @brief Fully Connected Layer
  */
 TEST_F(nntrainer_FullyConnectedLayer, setOptimizer_01_p) {
-  status = setOptimizer("adam", "learning_rate=0.001 |"
-                                "beta1=0.9 |"
-                                "beta2=0.9999 |"
-                                "epsilon=1e-7");
+  status = setOptimizer(nntrainer::OptType::ADAM, "learning_rate=0.001 |"
+                                                  "beta1=0.9 |"
+                                                  "beta2=0.9999 |"
+                                                  "epsilon=1e-7");
   EXPECT_EQ(status, ML_ERROR_NONE);
 }
 
@@ -408,10 +429,9 @@ TEST_F(nntrainer_FullyConnectedLayer, setOptimizer_01_p) {
  * @brief FullyConnected Layer
  */
 TEST_F(nntrainer_FullyConnectedLayer, setOptimizer_02_p) {
-  status = setOptimizer("sgd", "learning_rate=0.1");
+  status = setOptimizer(nntrainer::OptType::SGD, "learning_rate=0.1");
   EXPECT_EQ(status, ML_ERROR_NONE);
 }
-
 /**
  * @brief Fully Connected Layer
  */
@@ -471,8 +491,30 @@ protected:
       {"input_shape=" + getDimensionString(layer.getOutputDimension()[0])});
     EXPECT_EQ(status, ML_ERROR_NONE);
 
+    act_layer->setBatch(layer.getOutputDimension()[0].batch());
+
     status = act_layer->initialize();
     EXPECT_EQ(status, ML_ERROR_NONE);
+
+    act_layer->resizeNetInput(act_layer->getNumInputs());
+    act_layer->resizeNetOutput(act_layer->getNumOutputs());
+
+    for (unsigned int i = 0; i < act_layer->getNumInputs(); ++i) {
+      std::shared_ptr<nntrainer::NetBuffers> n_buffer =
+        std::make_unique<nntrainer::NetBuffers>();
+      n_buffer->var = nntrainer::Tensor(act_layer->getInputDimension()[i]);
+      n_buffer->grad = nntrainer::Tensor(act_layer->getInputDimension()[i]);
+      act_layer->setInputBuffer(i, n_buffer);
+    }
+
+    for (unsigned int i = 0; i < act_layer->getNumOutputs(); ++i) {
+      std::shared_ptr<nntrainer::NetBuffers> n_buffer =
+        std::make_unique<nntrainer::NetBuffers>();
+      n_buffer->var = nntrainer::Tensor(act_layer->getOutputDimension()[i]);
+      n_buffer->grad = nntrainer::Tensor(act_layer->getOutputDimension()[i]);
+      act_layer->setOutputBuffer(i, n_buffer);      
+    }
+    
     layers.push_back(act_layer);
   }
 
@@ -484,10 +526,32 @@ protected:
       {"input_shape=" + getDimensionString(layer.getOutputDimension()[0])});
     EXPECT_EQ(status, ML_ERROR_NONE);
 
+    loss_layer->setBatch(layer.getOutputDimension()[0].batch());    
+
     status = loss_layer->initialize();
     EXPECT_EQ(status, ML_ERROR_NONE);
     status = loss_layer->setLoss(type);
     EXPECT_EQ(status, ML_ERROR_NONE);
+
+    loss_layer->resizeNetInput(loss_layer->getNumInputs());
+    loss_layer->resizeNetOutput(loss_layer->getNumOutputs());
+
+    for (unsigned int i = 0; i < loss_layer->getNumInputs(); ++i) {
+      std::shared_ptr<nntrainer::NetBuffers> n_buffer =
+        std::make_unique<nntrainer::NetBuffers>();
+      n_buffer->var = nntrainer::Tensor(loss_layer->getInputDimension()[i]);
+      n_buffer->grad = nntrainer::Tensor(loss_layer->getInputDimension()[i]);
+      loss_layer->setInputBuffer(i, n_buffer);
+    }
+
+    for (unsigned int i = 0; i < loss_layer->getNumOutputs(); ++i) {
+      std::shared_ptr<nntrainer::NetBuffers> n_buffer =
+        std::make_unique<nntrainer::NetBuffers>();
+      n_buffer->var = nntrainer::Tensor(loss_layer->getOutputDimension()[i]);
+      n_buffer->grad = nntrainer::Tensor(loss_layer->getOutputDimension()[i]);
+      loss_layer->setOutputBuffer(i, n_buffer);      
+    }
+
     layers.push_back(loss_layer);
 
     if (type == nntrainer::LossType::LOSS_ENTROPY_SOFTMAX) {
@@ -500,20 +564,19 @@ protected:
 
   void matchForwarding(const char *file) {
     sharedConstTensor out;
-    EXPECT_NO_THROW(out = layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+    EXPECT_NO_THROW(out = layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
 
     if (layers.size() > 0) {
       for (unsigned int idx = 0; idx < layers.size() - 1; idx++) {
-        EXPECT_NO_THROW(out = layers[idx]->forwarding({out})[0]);
+        EXPECT_NO_THROW(out = layers[idx]->forwarding_with_val({out})[0]);
       }
 
-      if (nntrainer::istrequal(layers.back()->getType(),
-                               nntrainer::LossLayer::type)) {
+      if (layers.back()->getType() == nntrainer::LayerType::LAYER_LOSS) {
         std::shared_ptr<nntrainer::LossLayer> loss_layer =
           std::static_pointer_cast<nntrainer::LossLayer>(layers.back());
         EXPECT_NO_THROW(out = loss_layer->forwarding({out}, {label})[0]);
       } else {
-        EXPECT_NO_THROW(out = layers.back()->forwarding({out})[0]);
+        EXPECT_NO_THROW(out = layers.back()->forwarding_with_val({out})[0]);
       }
       EXPECT_EQ(status, ML_ERROR_NONE);
     }
@@ -534,10 +597,11 @@ protected:
       MAKE_SHARED_TENSOR(constant(1.0, 3, 1, 1, 15));
     sharedConstTensor back_out;
 
-    if (layers.size() && nntrainer::istrequal(layers.back()->getType(),
-                                              nntrainer::LossLayer::type)) {
+    if (layers.size() &&
+        layers.back()->getType() == nntrainer::LayerType::LAYER_LOSS) {
       if (with_loss) {
-        EXPECT_NO_THROW(back_out = layers.back()->backwarding({label}, 1)[0]);
+        EXPECT_NO_THROW(layers.back()->backwarding(1, {label}));
+       back_out=MAKE_SHARED_TENSOR(layers.back()->getGradient()[0]);
       } else {
         back_out = def_derivative;
       }
@@ -547,9 +611,9 @@ protected:
     }
 
     for (; idx >= 0; --idx)
-      EXPECT_NO_THROW(back_out = layers[idx]->backwarding({back_out}, 1)[0]);
+      EXPECT_NO_THROW(back_out = layers[idx]->backwarding_with_val( 1, {back_out})[0]);
 
-    EXPECT_NO_THROW(back_out = layer.backwarding({back_out}, 1)[0]);
+    EXPECT_NO_THROW(back_out = layer.backwarding_with_val(1, {back_out})[0]);
     matchOutput(*back_out.get(), file_dx);
 
     loadUpdatedWeightsGradients(file_uw, file_g);
@@ -599,11 +663,11 @@ TEST_F(nntrainer_FullyConnectedLayer_TFmatch,
   std::vector<float> weight_data;
   std::vector<float> bias_data;
 
-  setOptimizer("adam", "learning_rate=1.0");
+  setOptimizer(nntrainer::OptType::ADAM, "learning_rate=1.0");
 
   sharedConstTensor out;
 
-  EXPECT_NO_THROW(out = layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+  EXPECT_NO_THROW(out = layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
 
   nntrainer::Tensor derivatives(3, 1, 1, 15);
 
@@ -613,7 +677,7 @@ TEST_F(nntrainer_FullyConnectedLayer_TFmatch,
 
   nntrainer::Tensor result;
   EXPECT_NO_THROW(
-    result = *layer.backwarding({MAKE_SHARED_TENSOR(derivatives)}, 1)[0]);
+    result = *layer.backwarding_with_val(1, {MAKE_SHARED_TENSOR(derivatives)})[0]);
 
   matchOutput(result, "tc_fc_1_goldenFCGradientAdam.out");
 
@@ -633,7 +697,7 @@ TEST_F(nntrainer_FullyConnectedLayer_TFmatch,
  */
 TEST_F(nntrainer_FullyConnectedLayer_TFmatch,
        forwarding_backwarding_loss_00_p) {
-  setOptimizer("adam", "learning_rate=0.0001");
+  setOptimizer(nntrainer::OptType::ADAM, "learning_rate=0.0001");
   addLoss(nntrainer::LossType::LOSS_ENTROPY_SOFTMAX);
 
   matchForwarding("tc_fc_1_goldenFCResultSoftmaxCrossAdam.out");
@@ -648,7 +712,7 @@ TEST_F(nntrainer_FullyConnectedLayer_TFmatch,
  */
 TEST_F(nntrainer_FullyConnectedLayer_TFmatch, forwarding_backwarding_01_p) {
 
-  setOptimizer("sgd", "learning_rate=1.0");
+  setOptimizer(nntrainer::OptType::SGD, "learning_rate=1.0");
 
   /** Verify forwarding and backwarding without loss */
   matchForwarding("tc_fc_1_goldenFCResultActNone.out");
@@ -666,7 +730,7 @@ TEST_F(nntrainer_FullyConnectedLayer_TFmatch, forwarding_backwarding_02_p) {
 
   addActivation(nntrainer::ActivationType::ACT_SIGMOID);
   addLoss(nntrainer::LossType::LOSS_MSE);
-  setOptimizer("sgd", "learning_rate=1.0");
+  setOptimizer(nntrainer::OptType::SGD, "learning_rate=1.0");
 
   /** Verify forwarding value */
   matchForwarding("tc_fc_1_goldenFCResultSigmoidMse.out");
@@ -687,7 +751,7 @@ TEST_F(nntrainer_FullyConnectedLayer_TFmatch, forwarding_backwarding_03_p) {
 
   addActivation(nntrainer::ActivationType::ACT_SOFTMAX);
   addLoss(nntrainer::LossType::LOSS_MSE);
-  setOptimizer("sgd", "learning_rate=1.0");
+  setOptimizer(nntrainer::OptType::SGD, "learning_rate=1.0");
 
   /** Verify forwarding value */
   matchForwarding("tc_fc_1_goldenFCResultSoftmaxMse.out");
@@ -707,7 +771,7 @@ TEST_F(nntrainer_FullyConnectedLayer_TFmatch, forwarding_backwarding_03_p) {
 TEST_F(nntrainer_FullyConnectedLayer_TFmatch, forwarding_backwarding_04_p) {
 
   addLoss(nntrainer::LossType::LOSS_MSE);
-  setOptimizer("sgd", "learning_rate=1.0");
+  setOptimizer(nntrainer::OptType::SGD, "learning_rate=1.0");
 
   /** Verify forwarding value */
   matchForwarding("tc_fc_1_goldenFCResultActNone.out");
@@ -740,7 +804,7 @@ TEST_F(nntrainer_FullyConnectedLayer_TFmatch, forwarding_backwarding_05_p) {
 
   addActivation(nntrainer::ActivationType::ACT_SIGMOID);
   addLoss(nntrainer::LossType::LOSS_MSE);
-  setOptimizer("sgd", "learning_rate=1.0");
+  setOptimizer(nntrainer::OptType::SGD, "learning_rate=1.0");
 
   /** Verify forwarding value */
   matchForwarding("tc_fc_1_goldenFCResultSigmoidMse.out");
@@ -761,7 +825,7 @@ TEST_F(nntrainer_FullyConnectedLayer_TFmatch, forwarding_backwarding_06_p) {
 
   addActivation(nntrainer::ActivationType::ACT_SOFTMAX);
   addLoss(nntrainer::LossType::LOSS_MSE);
-  setOptimizer("sgd", "learning_rate=1.0");
+  setOptimizer(nntrainer::OptType::SGD, "learning_rate=1.0");
 
   /** Verify forwarding value */
   matchForwarding("tc_fc_1_goldenFCResultSoftmaxMse.out");
@@ -782,7 +846,7 @@ TEST_F(nntrainer_FullyConnectedLayer_TFmatch, forwarding_backwarding_06_p) {
 TEST_F(nntrainer_FullyConnectedLayer_TFmatch, forwarding_backwarding_07_p) {
 
   addLoss(nntrainer::LossType::LOSS_ENTROPY_SIGMOID);
-  setOptimizer("sgd", "learning_rate=1.0");
+  setOptimizer(nntrainer::OptType::SGD, "learning_rate=1.0");
 
   /** Verify forwarding value */
   matchForwarding("tc_fc_1_goldenFCResultSigmoidCross.out");
@@ -803,7 +867,7 @@ TEST_F(nntrainer_FullyConnectedLayer_TFmatch, forwarding_backwarding_07_p) {
 TEST_F(nntrainer_FullyConnectedLayer_TFmatch, forwarding_backwarding_08_p) {
 
   addLoss(nntrainer::LossType::LOSS_ENTROPY_SOFTMAX);
-  setOptimizer("sgd", "learning_rate=1.0");
+  setOptimizer(nntrainer::OptType::SGD, "learning_rate=1.0");
 
   /** Verify forwarding value */
   matchForwarding("tc_fc_1_goldenFCResultSoftmaxCross.out");
@@ -839,7 +903,7 @@ protected:
   virtual void prepareLayer() {
     setProperty("input_shape=1:1:12 | epsilon=0.001 | momentum=0.90");
     setBatch(3);
-    setOptimizer("sgd", "learning_rate=1");
+    setOptimizer(nntrainer::OptType::SGD, "learning_rate=1");
   }
 };
 
@@ -856,7 +920,7 @@ TEST_F(nntrainer_BatchNormalizationLayer, initialize_01_p) {
  */
 TEST_F(nntrainer_BatchNormalizationLayer, setOptimizer_01_p) {
   status = setOptimizer(
-    "adam", "learning_rate=0.001 | beta1=0.9 | beta2=0.9999 | epsilon=1e-7");
+    nntrainer::OptType::ADAM, "learning_rate=0.001 | beta1=0.9 | beta2=0.9999 | epsilon=1e-7");
   EXPECT_EQ(status, ML_ERROR_NONE);
 }
 
@@ -892,14 +956,14 @@ TEST_F(nntrainer_BatchNormalizationLayer, forward_backward_training_01_p) {
   sharedConstTensor forward_result;
 
   EXPECT_NO_THROW(forward_result =
-                    layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+                 layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
   matchOutput(*forward_result, "tc_bn_fc_1_goldenBNResultForward.out");
 
   nntrainer::Tensor backward_in(layer.getOutputDimension()[0]);
   loadFile("tc_bn_fc_1_goldenBNLayerBackwardDxIn.out", backward_in);
 
   nntrainer::Tensor backward_result =
-    *layer.backwarding({MAKE_SHARED_TENSOR(backward_in)}, 1)[0];
+    *layer.backwarding_with_val(1, {MAKE_SHARED_TENSOR(backward_in)})[0];
 
   matchOutput(backward_result, "tc_bn_fc_1_goldenBNLayerBackwardDx.out");
 }
@@ -923,7 +987,7 @@ protected:
   virtual void prepareLayer() {
     setProperty("input_shape=2:4:5 | epsilon=0.001 | momentum=0.90");
     setBatch(3);
-    setOptimizer("sgd", "learning_rate=1");
+    setOptimizer(nntrainer::OptType::SGD, "learning_rate=1");
   }
 };
 
@@ -931,14 +995,14 @@ TEST_F(nntrainer_BatchNormalizationLayer_Conv, forward_backward_training_01_p) {
   layer.setTrainable(true);
   sharedConstTensor forward_result;
 
-  forward_result = layer.forwarding({MAKE_SHARED_TENSOR(in)})[0];
+  forward_result = layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0];
   matchOutput(*forward_result, "tc_bn_conv_1_goldenBNResultForward.out");
 
   nntrainer::Tensor backward_in(layer.getOutputDimension()[0]);
   loadFile("tc_bn_conv_1_goldenBNLayerBackwardDxIn.out", backward_in);
 
   nntrainer::Tensor backward_result =
-    *layer.backwarding({MAKE_SHARED_TENSOR(backward_in)}, 1)[0];
+    *layer.backwarding_with_val( 1, {MAKE_SHARED_TENSOR(backward_in)})[0];
 
   matchOutput(backward_result, "tc_bn_conv_1_goldenBNLayerBackwardDx.out");
 }
@@ -964,7 +1028,7 @@ protected:
   virtual void prepareLayer() {
     setProperty("input_shape=2:4:5 | epsilon=0.001 | momentum=0.90");
     setBatch(1);
-    setOptimizer("sgd", "learning_rate=1");
+    setOptimizer(nntrainer::OptType::SGD, "learning_rate=1");
   }
 };
 
@@ -973,14 +1037,14 @@ TEST_F(nntrainer_BatchNormalizationLayer_Conv2,
   layer.setTrainable(true);
   sharedConstTensor forward_result;
 
-  forward_result = layer.forwarding({MAKE_SHARED_TENSOR(in)})[0];
+  forward_result = layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0];
   matchOutput(*forward_result, "tc_bn_conv_2_goldenBNResultForward.out");
 
   nntrainer::Tensor backward_in(layer.getOutputDimension()[0]);
   loadFile("tc_bn_conv_2_goldenBNLayerBackwardDxIn.out", backward_in);
 
   nntrainer::Tensor backward_result =
-    *layer.backwarding({MAKE_SHARED_TENSOR(backward_in)}, 1)[0];
+    *layer.backwarding_with_val( 1, {MAKE_SHARED_TENSOR(backward_in)})[0];
 
   matchOutput(backward_result, "tc_bn_conv_2_goldenBNLayerBackwardDx.out");
 }
@@ -1067,7 +1131,7 @@ TEST_F(nntrainer_Conv2DLayer, forwarding_01_p) {
   loadFile("tc_conv2d_1_conv2DLayer.in", in);
   loadFile("tc_conv2d_1_conv2DKernel.in", layer);
 
-  EXPECT_NO_THROW(out = *layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+  EXPECT_NO_THROW(out = *layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
   matchOutput(out, "tc_conv2d_1_goldenConv2DResult.out");
 }
 
@@ -1089,7 +1153,7 @@ TEST_F(nntrainer_Conv2DLayer, forwarding_02_p) {
   loadFile("tc_conv2d_2_conv2DLayer.in", in);
   loadFile("tc_conv2d_2_conv2DKernel.in", layer);
 
-  EXPECT_NO_THROW(out = *layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+  EXPECT_NO_THROW(out = *layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
   matchOutput(out, "tc_conv2d_2_goldenConv2DResult.out");
 }
 
@@ -1112,15 +1176,16 @@ TEST_F(nntrainer_Conv2DLayer, backwarding_01_p) {
 
   loadFile("tc_conv2d_1_conv2DLayer.in", in);
   loadFile("tc_conv2d_1_conv2DKernel.in", layer);
-  setOptimizer("sgd", "learning_rate=1.0");
+  setOptimizer(nntrainer::OptType::SGD, "learning_rate=1.0");
 
-  EXPECT_NO_THROW(out = *layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+  EXPECT_NO_THROW(out = *layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
 
   for (unsigned int i = 0; i < derivatives.getDim().getDataLen(); ++i) {
     derivatives.getData()[i] = 1.0;
   }
+  
   EXPECT_NO_THROW(
-    result = *layer.backwarding({MAKE_SHARED_TENSOR(derivatives)}, 1)[0]);
+                 result = *layer.backwarding_with_val(1, {MAKE_SHARED_TENSOR(derivatives)})[0]);
 
   nntrainer::Weight *param_data = layer.getWeights().get();
 
@@ -1168,15 +1233,15 @@ TEST_F(nntrainer_Conv2DLayer, backwarding_02_p) {
   loadFile("tc_conv2d_2_conv2DLayer.in", in);
   loadFile("tc_conv2d_2_conv2DKernel.in", layer);
 
-  setOptimizer("sgd", "learning_rate=1.0");
+  setOptimizer(nntrainer::OptType::SGD, "learning_rate=1.0");
 
-  EXPECT_NO_THROW(out = *layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+  EXPECT_NO_THROW(out = *layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
 
   for (unsigned int i = 0; i < derivatives.getDim().getDataLen(); ++i) {
     derivatives.getData()[i] = 1.0;
   }
   EXPECT_NO_THROW(
-    result = *layer.backwarding({MAKE_SHARED_TENSOR(derivatives)}, 1)[0]);
+                 result = *layer.backwarding_with_val(1, {MAKE_SHARED_TENSOR(derivatives)})[0]);
   param_data = layer.getWeights().get();
 
   for (unsigned int i = 0; i < filter_size * 2; ++i) {
@@ -1201,9 +1266,9 @@ TEST_F(nntrainer_Conv2DLayer, backwarding_02_p) {
   matchOutput(bias_grad, "tc_conv2d_2_goldenBiasGrad.out");
 
   for (int i = 0; i < 4; i++) {
-    EXPECT_NO_THROW(out = *layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+    EXPECT_NO_THROW(out = *layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
     EXPECT_NO_THROW(
-      result = *layer.backwarding({MAKE_SHARED_TENSOR(derivatives)}, 1)[0]);
+                   result = *layer.backwarding_with_val(1, {MAKE_SHARED_TENSOR(derivatives)})[0]);
   }
 
   param_data = layer.getWeights().get();
@@ -1258,7 +1323,7 @@ TEST_F(nntrainer_Conv2DLayer, backwarding_03_p) {
   loadFile("tc_conv2d_int_conv2DKernel.in", layer1);
 
   std::shared_ptr<nntrainer::Optimizer> op;
-  EXPECT_NO_THROW(op = nntrainer::createOptimizer("sgd"));
+  EXPECT_NO_THROW(op = nntrainer::createOptimizer(nntrainer::OptType::SGD));
   status = op->setProperty({"learning_rate=1.0"});
   EXPECT_EQ(status, ML_ERROR_NONE);
   status = layer1.setOptimizer(op);
@@ -1278,13 +1343,13 @@ TEST_F(nntrainer_Conv2DLayer, backwarding_03_p) {
 
   loadFile("tc_conv2d_int_conv2DKernel2.in", layer2);
   std::shared_ptr<nntrainer::Optimizer> op2;
-  EXPECT_NO_THROW(op2 = nntrainer::createOptimizer("sgd"));
+  EXPECT_NO_THROW(op2 = nntrainer::createOptimizer(nntrainer::OptType::SGD));
   status = op2->setProperty({"learning_rate=1.0"});
   EXPECT_EQ(status, ML_ERROR_NONE);
   status = layer2.setOptimizer(op2);
   EXPECT_EQ(status, ML_ERROR_NONE);
 
-  setOptimizer("sgd", "learning_rate=1.0");
+  setOptimizer(nntrainer::OptType::SGD, "learning_rate=1.0");
 
   unsigned int filter_size;
   std::vector<float> grad_data;
@@ -1296,11 +1361,11 @@ TEST_F(nntrainer_Conv2DLayer, backwarding_03_p) {
   nntrainer::Tensor derivatives(1, 12, 24, 24);
 
   nntrainer::Tensor out1;
-  EXPECT_NO_THROW(out1 = *layer1.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+  EXPECT_NO_THROW(out1 = *layer1.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
 
   nntrainer::Tensor out2;
 
-  EXPECT_NO_THROW(out2 = *layer2.forwarding({MAKE_SHARED_TENSOR(out1)})[0]);
+  EXPECT_NO_THROW(out2 = *layer2.forwarding_with_val({MAKE_SHARED_TENSOR(out1)})[0]);
 
   matchOutput(out1, "tc_conv2d_int_goldenConv2DResult.out");
   matchOutput(out2, "tc_conv2d_int_goldenConv2DResult2.out");
@@ -1311,10 +1376,10 @@ TEST_F(nntrainer_Conv2DLayer, backwarding_03_p) {
 
   nntrainer::Tensor result2;
   EXPECT_NO_THROW(
-    result2 = *layer2.backwarding({MAKE_SHARED_TENSOR(derivatives)}, 1)[0]);
+                 result2 = *layer2.backwarding_with_val(1, {MAKE_SHARED_TENSOR(derivatives)})[0]);
 
   EXPECT_NO_THROW(result =
-                    *layer1.backwarding({MAKE_SHARED_TENSOR(result2)}, 1)[0]);
+                 *layer1.backwarding_with_val(1, {MAKE_SHARED_TENSOR(result2)})[0]);
 
   /** Compare second conv */
   param_data = layer2.getWeights().get();
@@ -1388,14 +1453,14 @@ TEST_F(nntrainer_Conv2DLayer, backwarding_04_p) {
   loadFile("tc_conv2d_3_conv2DLayer.in", in);
   loadFile("tc_conv2d_3_conv2DKernel.in", layer);
 
-  setOptimizer("sgd", "learning_rate=1.0");
-  EXPECT_NO_THROW(out = *layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+  setOptimizer(nntrainer::OptType::SGD, "learning_rate=1.0");
+  EXPECT_NO_THROW(out = *layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
 
   for (unsigned int i = 0; i < derivatives.getDim().getDataLen(); ++i) {
     derivatives.getData()[i] = 1.0;
   }
   EXPECT_NO_THROW(
-    result = *layer.backwarding({MAKE_SHARED_TENSOR(derivatives)}, 1)[0]);
+                 result = *layer.backwarding_with_val( 1, {MAKE_SHARED_TENSOR(derivatives)})[0]);
 
   nntrainer::Weight *param_data = layer.getWeights().get();
 
@@ -1460,7 +1525,7 @@ TEST_F(nntrainer_Pooling2DLayer, forwarding_01_p) {
 
   loadFile("tc_pooling2d_1.in", in);
 
-  EXPECT_NO_THROW(out = *layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+  EXPECT_NO_THROW(out = *layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
 
   matchOutput(out, "tc_pooling2d_1_goldenPooling2Dmax.out");
 }
@@ -1473,7 +1538,7 @@ TEST_F(nntrainer_Pooling2DLayer, forwarding_02_p) {
 
   loadFile("tc_pooling2d_1.in", in);
 
-  EXPECT_NO_THROW(out = *layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+  EXPECT_NO_THROW(out = *layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
 
   matchOutput(out, "tc_pooling2d_1_goldenPooling2Daverage.out");
 }
@@ -1486,7 +1551,7 @@ TEST_F(nntrainer_Pooling2DLayer, forwarding_03_p) {
 
   loadFile("tc_pooling2d_1.in", in);
 
-  EXPECT_NO_THROW(out = *layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+  EXPECT_NO_THROW(out = *layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
 
   matchOutput(out, "tc_pooling2d_1_goldenPooling2Dglobal_max.out");
 }
@@ -1499,7 +1564,7 @@ TEST_F(nntrainer_Pooling2DLayer, forwarding_04_p) {
 
   loadFile("tc_pooling2d_1.in", in);
 
-  EXPECT_NO_THROW(out = *layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+  EXPECT_NO_THROW(out = *layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
 
   matchOutput(out, "tc_pooling2d_1_goldenPooling2Dglobal_average.out");
 }
@@ -1512,7 +1577,7 @@ TEST_F(nntrainer_Pooling2DLayer, forwarding_05_p) {
   reinitialize();
 
   loadFile("tc_pooling2d_2.in", in);
-  EXPECT_NO_THROW(out = *layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+  EXPECT_NO_THROW(out = *layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
   matchOutput(out, "tc_pooling2d_2_goldenPooling2Dglobal_max.out");
 }
 
@@ -1525,7 +1590,7 @@ TEST_F(nntrainer_Pooling2DLayer, forwarding_06_p) {
 
   loadFile("tc_pooling2d_2.in", in);
 
-  EXPECT_NO_THROW(out = *layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+  EXPECT_NO_THROW(out = *layer.forwarding_with_val( {MAKE_SHARED_TENSOR(in)})[0]);
   matchOutput(out, "tc_pooling2d_2_goldenPooling2Dglobal_average.out");
 }
 
@@ -1537,7 +1602,7 @@ TEST_F(nntrainer_Pooling2DLayer, backwarding_01_p) {
   reinitialize();
   loadFile("tc_pooling2d_1.in", in);
 
-  EXPECT_NO_THROW(out = *layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+  EXPECT_NO_THROW(out = *layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
 
   nntrainer::Tensor grad(out.getDim());
 
@@ -1545,7 +1610,7 @@ TEST_F(nntrainer_Pooling2DLayer, backwarding_01_p) {
     grad.getData()[i] = 1.0;
   }
 
-  EXPECT_NO_THROW(in = *layer.backwarding({MAKE_SHARED_TENSOR(grad)}, 0)[0]);
+  EXPECT_NO_THROW(in = *layer.backwarding_with_val(1, {MAKE_SHARED_TENSOR(grad)})[0]);
 
   matchOutput(in, "tc_pooling2d_1_goldenPooling2DmaxGrad.out");
 }
@@ -1557,7 +1622,7 @@ TEST_F(nntrainer_Pooling2DLayer, backwarding_02_p) {
   reinitialize();
   loadFile("tc_pooling2d_1.in", in);
 
-  EXPECT_NO_THROW(out = *layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+  EXPECT_NO_THROW(out = *layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
 
   sharedTensor grad = MAKE_SHARED_TENSOR(out.getDim());
 
@@ -1565,7 +1630,7 @@ TEST_F(nntrainer_Pooling2DLayer, backwarding_02_p) {
     grad->getData()[i] = 1.0;
   }
 
-  EXPECT_NO_THROW(in = *layer.backwarding({grad}, 0)[0]);
+  EXPECT_NO_THROW(in = *layer.backwarding_with_val(1, {grad})[0]);
 
   matchOutput(in, "tc_pooling2d_1_goldenPooling2DaverageGrad.out");
 }
@@ -1578,7 +1643,7 @@ TEST_F(nntrainer_Pooling2DLayer, backwarding_03_p) {
 
   loadFile("tc_pooling2d_1.in", in);
 
-  EXPECT_NO_THROW(out = *layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+  EXPECT_NO_THROW(out = *layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
 
   nntrainer::Tensor grad(out.getDim());
 
@@ -1586,7 +1651,7 @@ TEST_F(nntrainer_Pooling2DLayer, backwarding_03_p) {
     grad.getData()[i] = 1.0;
   }
 
-  EXPECT_NO_THROW(in = *layer.backwarding({MAKE_SHARED_TENSOR(grad)}, 0)[0]);
+  EXPECT_NO_THROW(in = *layer.backwarding_with_val(1, {MAKE_SHARED_TENSOR(grad)})[0]);
 
   matchOutput(in, "tc_pooling2d_1_goldenPooling2Dglobal_maxGrad.out");
 }
@@ -1598,7 +1663,7 @@ TEST_F(nntrainer_Pooling2DLayer, backwarding_04_p) {
   reinitialize();
   loadFile("tc_pooling2d_1.in", in);
 
-  EXPECT_NO_THROW(out = *layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+  EXPECT_NO_THROW(out = *layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
 
   nntrainer::Tensor grad(out.getDim());
 
@@ -1606,7 +1671,7 @@ TEST_F(nntrainer_Pooling2DLayer, backwarding_04_p) {
     grad.getData()[i] = 1.0;
   }
 
-  EXPECT_NO_THROW(in = *layer.backwarding({MAKE_SHARED_TENSOR(grad)}, 0)[0]);
+  EXPECT_NO_THROW(in = *layer.backwarding_with_val( 1, {MAKE_SHARED_TENSOR(grad)})[0]);
 
   matchOutput(in, "tc_pooling2d_1_goldenPooling2Dglobal_averageGrad.out");
 }
@@ -1630,7 +1695,7 @@ TEST_F(nntrainer_FlattenLayer, forwarding_01_p) {
 
   loadFile("tc_pooling2d_1_goldenPooling2Dmax.out", in);
 
-  EXPECT_NO_THROW(out = *layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+  EXPECT_NO_THROW(out = *layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
 
   matchOutput(out, "tc_pooling2d_1_goldenPooling2Dmax.out");
 }
@@ -1647,7 +1712,7 @@ TEST_F(nntrainer_FlattenLayer, forwarding_02_p) {
 
   loadFile("tc_pooling2d_2_goldenPooling2Dmax.out", in);
 
-  EXPECT_NO_THROW(out = *layer.forwarding({MAKE_SHARED_TENSOR(in)})[0]);
+  EXPECT_NO_THROW(out = *layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
 
   matchOutput(out, "tc_pooling2d_2_goldenPooling2Dmax.out");
 }
@@ -1662,7 +1727,7 @@ TEST_F(nntrainer_FlattenLayer, backwarding_01_p) {
 
   loadFile("tc_pooling2d_1_goldenPooling2Dmax.out", out);
 
-  EXPECT_NO_THROW(in = *layer.backwarding({MAKE_SHARED_TENSOR(out)}, 0)[0]);
+  EXPECT_NO_THROW(in = *layer.backwarding_with_val( 1, {MAKE_SHARED_TENSOR(out)})[0]);
   EXPECT_EQ(in.getDim(), nntrainer::TensorDim(1, 2, 4, 4));
 
   matchOutput(in, "tc_pooling2d_1_goldenPooling2Dmax.out");
@@ -1680,7 +1745,7 @@ TEST_F(nntrainer_FlattenLayer, backwarding_02_p) {
 
   loadFile("tc_pooling2d_2_goldenPooling2Dmax.out", out);
 
-  EXPECT_NO_THROW(in = *layer.backwarding({MAKE_SHARED_TENSOR(out)}, 0)[0]);
+  EXPECT_NO_THROW(in = *layer.backwarding_with_val( 1, {MAKE_SHARED_TENSOR(out)})[0]);
   EXPECT_EQ(in.getDim(), nntrainer::TensorDim(2, 2, 4, 4));
 
   matchOutput(in, "tc_pooling2d_2_goldenPooling2Dmax.out");
@@ -1717,14 +1782,14 @@ TEST(nntrainer_LossLayer, forward_loss_unknown_n) {
   nntrainer::Tensor a = constant(1.0, 1, 1, 1, 1);
   nntrainer::Tensor b = constant(1.0, 1, 1, 1, 1);
   EXPECT_THROW(
-    layer.forwarding({MAKE_SHARED_TENSOR(a)}, {MAKE_SHARED_TENSOR(b)}),
+    layer.forwarding( {MAKE_SHARED_TENSOR(a)}, {MAKE_SHARED_TENSOR(b)}),
     std::runtime_error);
 }
 
 TEST(nntrainer_LossLayer, backward_loss_unknown_n) {
   nntrainer::LossLayer layer;
   nntrainer::Tensor a = constant(1.0, 1, 1, 1, 1);
-  EXPECT_THROW(layer.backwarding({MAKE_SHARED_TENSOR(a)}, 1),
+  EXPECT_THROW(layer.backwarding( 1, {MAKE_SHARED_TENSOR(a)}),
                std::runtime_error);
 }
 
@@ -1742,7 +1807,7 @@ TEST(nntrainer_LossLayer, backward_loss_backward_entropy_n) {
   nntrainer::LossLayer layer;
   layer.setLoss(nntrainer::LossType::LOSS_ENTROPY);
   nntrainer::Tensor a = constant(1.0, 1, 1, 1, 1);
-  EXPECT_THROW(layer.backwarding({MAKE_SHARED_TENSOR(a)}, 1),
+  EXPECT_THROW(layer.backwarding( 1, {MAKE_SHARED_TENSOR(a)}),
                std::runtime_error);
 }
 
@@ -1823,12 +1888,12 @@ TEST(nntrainer_ActivationLayer, forward_backward_01_p) {
   GEN_TEST_INPUT(expected,
                  nntrainer::ActivationLayer::relu((l - 4) * 0.1 * (i + 1)));
   nntrainer::Tensor result;
-  EXPECT_NO_THROW(result = *layer.forwarding({MAKE_SHARED_TENSOR(input)})[0]);
+  EXPECT_NO_THROW(result = *layer.forwarding_with_val({MAKE_SHARED_TENSOR(input)})[0]);
   EXPECT_TRUE(result == expected);
 
   expected.copy(input);
-  EXPECT_NO_THROW(result = *layer.backwarding(
-                    {MAKE_SHARED_TENSOR(constant(1.0, 3, 1, 1, 10))}, 1)[0]);
+  EXPECT_NO_THROW(result = *layer.backwarding_with_val(1, 
+                    {MAKE_SHARED_TENSOR(constant(1.0, 3, 1, 1, 10))})[0]);
   GEN_TEST_INPUT(expected,
                  nntrainer::ActivationLayer::reluPrime(
                    nntrainer::ActivationLayer::relu((l - 4) * 0.1 * (i + 1))));
@@ -1885,7 +1950,7 @@ TEST_F(nntrainer_AdditionLayer, forwarding_01_n) {
 
   in = nntrainer::Tensor();
 
-  EXPECT_THROW(layer.forwarding({input}), std::runtime_error);
+  EXPECT_THROW(layer.forwarding_with_val({input}), std::runtime_error);
 }
 
 /*
@@ -1901,7 +1966,7 @@ TEST_F(nntrainer_AdditionLayer, DISABLED_forwarding_02_n) {
 
   in = nntrainer::Tensor(layer.getInputDimension()[0]);
 
-  EXPECT_THROW(layer.forwarding({input}), std::runtime_error);
+  EXPECT_THROW(layer.forwarding_with_val({input}), std::runtime_error);
 }
 
 TEST_F(nntrainer_AdditionLayer, DISABLED_forwarding_03_p) {
@@ -1914,7 +1979,7 @@ TEST_F(nntrainer_AdditionLayer, DISABLED_forwarding_03_p) {
 
   input.get()[1] = *input;
 
-  EXPECT_NO_THROW(layer.forwarding({input}));
+  EXPECT_NO_THROW(layer.forwarding_with_val({input}));
 }
 
 /**

diff --git a/test/unittest/unittest_nntrainer_models.cpp b/test/unittest/unittest_nntrainer_models.cpp
index 2417a0cdeb472cc5ce13eb3df4cdad01cca3d2da..e032b2a8a19fc3ae8243a0a2d05382c85b55cbe1 100644 (file)
--- a/test/unittest/unittest_nntrainer_models.cpp
+++ b/test/unittest/unittest_nntrainer_models.cpp
@@ -27,8 +27,9 @@
  * Watcher Classes                                      *
  ********************************************************/
 
-using NodeType = nntrainer::NeuralNetwork::NodeType;
+using NodeType = nntrainer::LayerNode;
 using FlatGraphType = nntrainer::NeuralNetwork::FlatGraphType;
+using NetworkGraphType = nntrainer::NetworkGraph;
 
 /**
  * @brief verify tensor to the reference and throw if not match to stop
@@ -81,16 +82,16 @@ public:
    * @param node node to watch.
    */
   NodeWatcher(const NodeType &node) : node(node) {
-    unsigned int num_weights = node->getNumWeights();
-    node->setTrainable(true);
+    unsigned int num_weights = node.layer->getNumWeights();
+    node.layer->setTrainable(true);
 
     for (unsigned int i = 0; i < num_weights; ++i) {
-      const nntrainer::Weight &w = node->weightAt(i);
+      const nntrainer::Weight &w = node.layer->weightAt(i);
       expected_weights.push_back(w);
     }
 
-    expected_output = nntrainer::Tensor(node->getOutputDimension()[0]);
-    expected_dx = nntrainer::Tensor(node->getInputDimension()[0]);
+    expected_output = nntrainer::Tensor(node.layer->getOutputDimension()[0]);
+    expected_dx = nntrainer::Tensor(node.layer->getInputDimension()[0]);
   }
 
   /**
@@ -98,9 +99,9 @@ public:
    *
    */
   void readLayerWeight(std::ifstream &f) {
-    for (unsigned int i = 0; i < node->getNumWeights(); ++i) {
+    for (unsigned int i = 0; i < node.layer->getNumWeights(); ++i) {
       /// @note below is harrasing the fact the tensor shares same base memory
-      node->weightAt(i).getVariable().read(f);
+      node.layer->weightAt(i).getVariable().read(f);
     }
   }
 
@@ -111,8 +112,7 @@ public:
    * @param iteration iteration
    * @return nntrainer::sharedConstTensor
    */
-  nntrainer::sharedConstTensors forward(nntrainer::sharedConstTensors in,
-                                        int iteration);
+  void forward(int iteration);
 
   /**
    * @brief forward loss node with verifying inputs/weights/outputs
@@ -134,9 +134,7 @@ public:
    * @param should_verify should verify the inputs/gradients/outputs
    * @return nntrainer::sharedConstTensor
    */
-  nntrainer::sharedConstTensors backward(nntrainer::sharedConstTensors deriv,
-                                         int iteration,
-                                         bool should_verify = true);
+  void backward(int iteration, bool should_verify = true);
 
   /**
    * @brief verify weights of the current node
@@ -157,7 +155,7 @@ public:
    *
    * @return float loss
    */
-  float getLoss() { return node->getLoss(); }
+  float getLoss() { return node.layer->getLoss(); }
 
   /**
    * @brief read Node
@@ -212,14 +210,15 @@ void NodeWatcher::read(std::ifstream &in) {
 
 void NodeWatcher::verifyWeight(const std::string &error_msg) {
   for (unsigned int i = 0; i < expected_weights.size(); ++i) {
-    verify(node->weightAt(i).getVariable(), expected_weights[i].getVariable(),
-           error_msg + " " + node->weightAt(i).getName() + " weight");
+    verify(node.layer->weightAt(i).getVariable(),
+           expected_weights[i].getVariable(),
+           error_msg + " " + node.layer->weightAt(i).getName() + " weight");
   }
 }
 
 void NodeWatcher::verifyGrad(const std::string &error_msg) {
   for (unsigned int i = 0; i < expected_weights.size(); ++i) {
-    auto weight = node->weightAt(i);
+    auto weight = node.layer->weightAt(i);
     if (weight.getTrainable()) {
       verify(weight.getGradient(), expected_weights[i].getGradient(),
              error_msg + " " + weight.getName() + " grad");
@@ -227,48 +226,45 @@ void NodeWatcher::verifyGrad(const std::string &error_msg) {
   }
 }
 
-nntrainer::sharedConstTensors
-NodeWatcher::forward(nntrainer::sharedConstTensors in, int iteration) {
+void NodeWatcher::forward(int iteration) {
   std::stringstream ss;
-  ss << "forward failed at " << node->getName() << " at iteration "
+  ss << "forward failed at " << node.layer->getName() << " at iteration "
      << iteration;
   std::string err_msg = ss.str();
 
-  nntrainer::sharedConstTensors out = node->forwarding(in);
-  verify(*out[0], expected_output, err_msg + " at output");
-  return out;
+  std::vector<nntrainer::Tensor> out = node.layer->getHidden();
+
+  verify(out[0], expected_output, err_msg + " at output");
 }
 
 nntrainer::sharedConstTensors
 NodeWatcher::lossForward(nntrainer::sharedConstTensors pred,
                          nntrainer::sharedConstTensors answer, int iteration) {
   std::stringstream ss;
-  ss << "loss failed at " << node->getName() << " at iteration " << iteration;
+  ss << "loss failed at " << node.layer->getName() << " at iteration "
+     << iteration;
   std::string err_msg = ss.str();
 
   nntrainer::sharedConstTensors out =
-    std::static_pointer_cast<nntrainer::LossLayer>(node)->forwarding(pred,
-                                                                     answer);
+    std::static_pointer_cast<nntrainer::LossLayer>(node.layer)
+      ->forwarding(pred, answer);
 
   return out;
 }
 
-nntrainer::sharedConstTensors
-NodeWatcher::backward(nntrainer::sharedConstTensors deriv, int iteration,
-                      bool should_verify) {
+void NodeWatcher::backward(int iteration, bool should_verify) {
   std::stringstream ss;
-  ss << "backward failed at " << node->getName() << " at iteration "
+  ss << "backward failed at " << node.layer->getName() << " at iteration "
      << iteration;
   std::string err_msg = ss.str();
 
-  nntrainer::sharedConstTensors out = node->backwarding(deriv, iteration);
+  std::vector<nntrainer::Tensor> out = node.layer->getGradient();
+
   if (should_verify) {
+    verify(out[0], expected_dx, err_msg);
     verifyGrad(err_msg);
-    verify(*out[0], expected_dx, err_msg);
     verifyWeight(err_msg);
   }
-
-  return out;
 }
 
 GraphWatcher::GraphWatcher(const std::string &config) {
@@ -277,11 +273,17 @@ GraphWatcher::GraphWatcher(const std::string &config) {
     throw std::invalid_argument("load from config failed!");
   };
 
-  if (nn.init()) {
+  if (nn.compile()) {
+    throw std::invalid_argument("initiation failed");
+  };
+
+  if (nn.initialize()) {
     throw std::invalid_argument("initiation failed");
   };
 
-  FlatGraphType graph = nn.getFlatGraph();
+  NetworkGraphType model_graph = nn.getNetworkGraph();
+
+  std::vector<NodeType> graph = model_graph.getSorted();
 
   for (auto it = graph.begin(); it != graph.end() - 1; ++it) {
     nodes.push_back(NodeWatcher(*it));
@@ -311,18 +313,17 @@ void GraphWatcher::compareFor(const std::string &reference,
 
     readIteration(ref);
 
-    /// forward pass
-    for (auto &i : nodes)
-      input = i.forward(input, iteration);
-
-    loss_node.lossForward(input, label, iteration);
+    nn.forwarding(input, label);
     EXPECT_NEAR(expected_loss, loss_node.getLoss(), nntrainer::Tensor::epsilon);
 
-    /// backward pass and update weights
-    nntrainer::sharedConstTensors output =
-      loss_node.backward(label, iteration, false);
-    for (auto it = nodes.rbegin(); it != nodes.rend(); it++)
-      output = it->backward(output, iteration);
+    for (auto it = nodes.begin(); it != nodes.end() - 1; ++it) {
+      it->forward(iteration);
+    }
+
+    nn.getNetworkGraph().backwarding(label, iteration);
+
+    for (auto it = nodes.rbegin(); it != nodes.rend() - 1; it++)
+      it->backward(iteration);
   }
 }
 
@@ -475,43 +476,34 @@ using INI = IniTestWrapper;
 // clang-format off
 INI fc_sigmoid_mse(
   "fc_sigmoid_mse",
-  {
-    nn_base + "learning_rate=1 | optimizer=sgd | loss=mse | batch_size = 3",
-    I("input") + input_base + "input_shape = 1:1:3",
-    I("dense") + fc_base + "unit = 5",
-    I("act") + sigmoid_base,
-    I("dense_1") + fc_base + "unit = 10",
-    I("act_1") + softmax_base
-  }
-);
+  {nn_base + "learning_rate=1 | optimizer=sgd | loss=mse | batch_size = 3",
+   I("input") + input_base + "input_shape = 1:1:3",
+   I("dense") + fc_base + "unit = 5" + "input_layers=input",
+   I("act") + sigmoid + "input_layers=dense",
+   I("dense_1") + fc_base + "unit = 10" + "input_layers=act",
+   I("act_1") + softmax + "input_layers=dense_1"});
 
 INI fc_sigmoid_cross =
   INI("fc_sigmoid_cross") + fc_sigmoid_mse + "model/loss=cross";
 
 INI fc_relu_mse(
   "fc_relu_mse",
-  {
-    nn_base + "Learning_rate=0.1 | Optimizer=sgd | Loss=mse | batch_size = 3",
-    I("input") + input_base + "input_shape = 1:1:3",
-    I("dense") + fc_base + "unit = 10",
-    I("act") + relu_base,
-    I("dense_1") + fc_base + "unit = 2",
-    I("act_1") + sigmoid_base
-  }
-);
+  {nn_base + "Learning_rate=0.1 | Optimizer=sgd | Loss=mse | batch_size = 3",
+   I("input") + input_base + "input_shape = 1:1:3",
+   I("dense") + fc_base + "unit = 10" + "input_layers=input",
+   I("act") + relu + "input_layers=dense",
+   I("dense_1") + fc_base + "unit = 2" + "input_layers=act",
+   I("act_1") + sigmoid + "input_layers=dense" + "input_layers=dense_1"});
 
 INI fc_bn_sigmoid_cross(
   "fc_bn_sigmoid_cross",
-  {
-    nn_base + "learning_rate=1 | optimizer=sgd | loss=cross | batch_size = 3",
-    I("input") + input_base + "input_shape = 1:1:3",
-    I("dense") + fc_base + "unit = 10",
-    I("bn") + bn_base,
-    I("act") + sigmoid_base,
-    I("dense_2") + fc_base + "unit = 10",
-    I("act_3") + softmax_base
-  }
-);
+  {nn_base + "learning_rate=1 | optimizer=sgd | loss=cross | batch_size = 3",
+   I("input") + input_base + "input_shape = 1:1:3",
+   I("dense") + fc_base + "unit = 10" + "input_layers=input",
+   I("bn") + bn_base + "input_layers=dense",
+   I("act") + sigmoid + "input_layers=bn",
+   I("dense_2") + fc_base + "unit = 10" + "input_layers=act",
+   I("act_3") + softmax + "input_layers=dense_2"});
 
 INI fc_bn_sigmoid_mse =
   INI("fc_bn_sigmoid_mse") + fc_bn_sigmoid_cross + "model/loss=mse";
@@ -524,12 +516,12 @@ INI mnist_conv_cross(
   {
     nn_base + "learning_rate=0.1 | optimizer=sgd | loss=cross | batch_size=3",
     I("input") + input_base + "input_shape=2:4:5",
-    I("conv2d_c1_layer") + conv_base + "kernel_size=3,4 | filters=2",
-    I("act_1") + sigmoid_base,
-    I("pool_1") + mnist_pooling,
-    I("flatten", "type=flatten"),
-    I("outputlayer") + fc_base + "unit = 10",
-    I("act_3") + softmax_base
+    I("conv2d_c1_layer") + conv_base + "kernel_size=3,4 | filters=2" +"input_layers=input",
+    I("act_1") + sigmoid_base +"input_layers=conv2d_c1_layer",
+    I("pool_1") + mnist_pooling+"input_layers=act_1",
+    I("flatten", "type=flatten")+"input_layers=pool_1" ,
+    I("outputlayer") + fc_base + "unit = 10" +"input_layers=flatten",
+    I("act_3") + softmax_base +"input_layers=outputlayer"
   }
 );