[layer] Update dropout layer to V2

Update dropout layer to V2 design.

Signed-off-by: Parichay Kapoor <kparichay@gmail.com>

diff --git a/api/ccapi/include/layer.h b/api/ccapi/include/layer.h
index 3fadda7..67a099b 100644 (file)
--- a/api/ccapi/include/layer.h
+++ b/api/ccapi/include/layer.h
@@ -55,7 +55,7 @@ enum LayerType {
   LAYER_GRU,                        /** GRU Layer type */
   LAYER_TIME_DIST,                  /** Time Distributed Layer type */
   LAYER_PERMUTE,                    /** Permute layer */
-  LAYER_DROPOUT,              /** DropOut Layer type */
+  LAYER_DROPOUT,                    /** DropOut Layer type */
   LAYER_LOSS_MSE = 500,             /** Mean Squared Error Loss Layer type */
   LAYER_LOSS_CROSS_ENTROPY,         /** Cross Entropy Loss Layer type */
   LAYER_LOSS_CROSS_ENTROPY_SIGMOID, /** Cross Entropy with Sigmoid Loss Layer

diff --git a/nntrainer/app_context.cpp b/nntrainer/app_context.cpp
index f8fd1d7..f8d6d22 100644 (file)
--- a/nntrainer/app_context.cpp
+++ b/nntrainer/app_context.cpp
@@ -33,10 +33,10 @@
 #include <bn_layer.h>
 #include <concat_layer.h>
 #include <conv2d_layer.h>
-#include <dropout.h>
 #include <cross_entropy_loss_layer.h>
 #include <cross_entropy_sigmoid_loss_layer.h>
 #include <cross_entropy_softmax_loss_layer.h>
+#include <dropout.h>
 #include <embedding.h>
 #include <fc_layer.h>
 #include <flatten_layer.h>

diff --git a/nntrainer/layers/dropout.cpp b/nntrainer/layers/dropout.cpp
index 2496a3e..bee29b6 100644 (file)
--- a/nntrainer/layers/dropout.cpp
+++ b/nntrainer/layers/dropout.cpp
@@ -12,7 +12,6 @@
  */
 
 #include <dropout.h>
-#include <layer_internal.h>
 #include <nntrainer_error.h>
 #include <nntrainer_log.h>
 #include <parse_util.h>
@@ -20,52 +19,65 @@
 
 namespace nntrainer {
 
-int DropOutLayer::initialize(Manager &manager) {
-  output_dim = input_dim;
-  // TODO : asking Tensor to manager
-  for (auto &t : input_dim) {
-    mask.push_back(std::make_shared<Tensor>(t, true));
-  }
+static constexpr size_t SINGLE_INOUT_IDX = 0;
+
+void DropOutLayer::finalize(InitLayerContext &context) {
+  auto const &input_dims = context.getInputDimensions();
+  context.setOutputDimensions(input_dims);
 
-  return ML_ERROR_NONE;
+  mask_idx.reserve(input_dims.size());
+  for (auto &t : input_dims) {
+    mask_idx.push_back(
+      context.requestTensor(t, "DropoutMask", false, ITERATION_LIFESPAN));
+  }
 }
 
-void DropOutLayer::forwarding(bool training) {
+void DropOutLayer::forwarding(RunLayerContext &context, bool training) {
   auto &rate_ = std::get<props::DropOutSpec>(dropout_rate).get();
+
   // Assume it is in-place calculation. It means input and output share mem
   // buffer. So if the training is false, the output is the same with input. In
-  // orther words, there is nothing happen during inference.
+  // other words, there is nothing happen during inference.
 
   if (training && rate_ > 0.0) {
-    for (unsigned int i = 0; i < input_dim.size(); ++i) {
-      Tensor &input = net_input[i]->getVariableRef();
-      Tensor &mask_ = *mask[i].get();
+    for (unsigned int i = 0; i < context.getNumInputs(); ++i) {
+      Tensor &input_ = context.getInput(i);
+      Tensor &output_ = context.getOutput(i);
+      Tensor &mask_ = context.getTensor(mask_idx[i]);
+
+      mask_ = input_.dropout_mask(rate_);
+      input_.multiply_i(mask_);
 
-      mask_ = input.dropout_mask(rate_);
-      input.multiply_i(mask_);
+      /** @todo: remove below once in_place support is ready from manager */
+      output_.fill(input_);
     }
   }
 }
 
-void DropOutLayer::calcDerivative() {
+void DropOutLayer::calcDerivative(RunLayerContext &context) {
   // Assume it is in-place calculation
   auto &rate_ = std::get<props::DropOutSpec>(dropout_rate).get();
   if (rate_ > 0.0) {
-    for (unsigned int i = 0; i < input_dim.size(); ++i) {
-      Tensor deriv = net_hidden[i]->getGradientRef();
-      deriv.multiply_i(*mask[i].get());
+    for (unsigned int i = 0; i < context.getNumInputs(); ++i) {
+      Tensor &derivative_ = context.getIncomingDerivative(i);
+      Tensor &ret_ = context.getOutgoingDerivative(SINGLE_INOUT_IDX);
+      Tensor &mask_ = context.getTensor(mask_idx[i]);
+
+      derivative_.multiply_i(mask_);
+
+      /** @todo: remove below once in_place support is ready from manager */
+      ret_.fill(derivative_);
     }
   }
 }
 
-int DropOutLayer::setProperty(std::vector<std::string> values) {
-  try {
-    values = loadProperties(values, dropout_rate);
-  } catch (std::invalid_argument &e) {
-    ml_loge("parsing property failed, reason: %s", e.what());
-    return ML_ERROR_INVALID_PARAMETER;
+void DropOutLayer::setProperty(const std::vector<std::string> &values) {
+  auto remain_props = loadProperties(values, dropout_rate);
+  if (!remain_props.empty()) {
+    std::string msg = "[FlattenLayer] Unknown Layer Properties count " +
+                      std::to_string(values.size());
+    throw exception::not_supported(msg);
   }
-
-  return LayerV1::setProperty(values);
 }
+
 } /* namespace nntrainer */

diff --git a/nntrainer/layers/dropout.h b/nntrainer/layers/dropout.h
index 27aacce..d4281d1 100644 (file)
--- a/nntrainer/layers/dropout.h
+++ b/nntrainer/layers/dropout.h
@@ -16,9 +16,8 @@
 #ifdef __cplusplus
 
 #include <common_properties.h>
-#include <layer_internal.h>
+#include <layer_devel.h>
 #include <node_exporter.h>
-#include <tensor.h>
 
 namespace nntrainer {
 
@@ -26,17 +25,14 @@ namespace nntrainer {
  * @class   DropOut Layer
  * @brief   DropOut Layer
  */
-class DropOutLayer : public LayerV1 {
+class DropOutLayer : public Layer {
 public:
   /**
    * @brief     Constructor of DropOut Layer
    */
-  template <typename... Args>
-  DropOutLayer(float dropout = 0.0, Args... args) :
-    LayerV1(args...),
-    dropout_rate(props::DropOutSpec(dropout)) {
-    setTrainable(false);
-  }
+  DropOutLayer(float dropout = 0.0) :
+    Layer(),
+    dropout_rate(props::DropOutSpec(dropout)) {}
 
   /**
    * @brief     Destructor of DropOut Layer
@@ -56,61 +52,54 @@ public:
   DropOutLayer &operator=(DropOutLayer &&rhs) = default;
 
   /**
-   * @brief     initialize layer
-   * @retval #ML_ERROR_NONE Successful.
-   * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+   * @copydoc Layer::finalize(InitLayerContext &context)
    */
-  int initialize(Manager &manager) override;
+  void finalize(InitLayerContext &context) override;
 
   /**
-   * @brief     Read Weight & Bias Data from file
-   * @param[in] file input stream file
+   * @copydoc Layer::forwarding(RunLayerContext &context, bool training)
    */
-  void read(std::ifstream &file) override{};
+  void forwarding(RunLayerContext &context, bool training) override;
 
   /**
-   * @brief     Save Weight & Bias Data to file
-   * @param[in] file output stream file
+   * @copydoc Layer::calcDerivative(RunLayerContext &context)
    */
-  void save(std::ofstream &file) override{};
+  void calcDerivative(RunLayerContext &context) override;
 
   /**
-   * @copydoc Layer::forwarding(bool training)
+   * @copydoc Layer::exportTo(Exporter &exporter, ExportMethods method)
    */
-  void forwarding(bool training = true) override;
+  void exportTo(Exporter &exporter,
+                const ExportMethods &method) const override {}
 
   /**
-   * @copydoc Layer::calcDerivative()
-   */
-  void calcDerivative() override;
-
-  /**
-   * @copydoc Layer::supportInPlace()
+   * @copydoc Layer::getType()
    */
-  bool supportInPlace() const override { return true; }
+  const std::string getType() const override { return DropOutLayer::type; };
 
   /**
-   * @copydoc Layer::setProperty(std::vector<std::string> values)
+   * @copydoc Layer::supportBackwarding()
    */
-  int setProperty(std::vector<std::string> values) override;
+  bool supportBackwarding() const { return true; }
 
   /**
-   * @copydoc Layer::export_to(Exporter &exporter, ExportMethods method)
+   * @copydoc Layer::setProperty(const PropertyType type, const std::string
+   * &value)
    */
-  void export_to(
-    Exporter &exporter,
-    ExportMethods method = ExportMethods::METHOD_STRINGVECTOR) const override{};
+  void setProperty(const std::vector<std::string> &values) override;
 
   /**
-   * @copydoc Layer::getType()
+   * @copydoc Layer::supportInPlace()
+   *
+   * @todo Enable in-place support once supported by manager
    */
-  const std::string getType() const override { return DropOutLayer::type; };
+  bool supportInPlace() const override { return false; }
 
   inline static const std::string type = "dropout";
 
 private:
   std::tuple<props::DropOutSpec> dropout_rate;
-  std::vector<std::shared_ptr<Tensor>> mask;
+  std::vector<unsigned int> mask_idx;
 };
 
 } // namespace nntrainer

diff --git a/nntrainer/layers/rnn.h b/nntrainer/layers/rnn.h
index f3c701c..ccc3df5 100644 (file)
--- a/nntrainer/layers/rnn.h
+++ b/nntrainer/layers/rnn.h
@@ -39,7 +39,7 @@ public:
     hidden_state_activation_type(hidden_state_activation_type_),
     acti_func(hidden_state_activation_type, true),
     return_sequences(ret_sequence),
-    dropout_rate(dropout){}
+    dropout_rate(dropout) {}
 
   /**
    * @brief     Destructor of RNNLayer