[Property] Add loss scale property

It add loss scale property as model common property.

Signed-off-by: Jiho Chu <jiho.chu@samsung.com>

diff --git a/nntrainer/models/model_common_properties.cpp b/nntrainer/models/model_common_properties.cpp
index a1f560c49ae9dc79b0b04918ee9dc7408155a5d4..984cad662a6bda3ea5574b38a7d6fad8434fadd4 100644 (file)
--- a/nntrainer/models/model_common_properties.cpp
+++ b/nntrainer/models/model_common_properties.cpp
@@ -39,4 +39,6 @@ MemorySwapLookahead::MemorySwapLookahead(const unsigned int &value) {
 ModelTensorDataType::ModelTensorDataType(ModelTensorDataTypeInfo::Enum value) {
   set(value);
 }
+LossScale::LossScale(float value) { set(value); }
+
 } // namespace nntrainer::props

diff --git a/nntrainer/models/model_common_properties.h b/nntrainer/models/model_common_properties.h
index 791f9ed5d321d4d57af3e00b27f6f6b58639b4ee..3776afefca5f6c18553b39b902fd743769c007c6 100644 (file)
--- a/nntrainer/models/model_common_properties.h
+++ b/nntrainer/models/model_common_properties.h
@@ -211,6 +211,17 @@ public:
                         ModelTensorDataTypeInfo::Enum::W32A32);
 };
 
+/**
+ * @brief LossScale property, loss is scaled by this value
+ *
+ */
+class LossScale : public Property<float> {
+public:
+  LossScale(float value = 0.0f);
+  static constexpr const char *key = "loss_scale"; /**< unique key to access */
+  using prop_tag = float_prop_tag;                 /**< property type */
+};
+
 } // namespace nntrainer::props
 
 #endif

diff --git a/nntrainer/models/neuralnet.cpp b/nntrainer/models/neuralnet.cpp
index af719237da8e8944554c0d3e33a66bc35680c80b..9b69ecea9875ea9c3eda52086c3d2d20679c7a77 100644 (file)
--- a/nntrainer/models/neuralnet.cpp
+++ b/nntrainer/models/neuralnet.cpp
@@ -70,7 +70,7 @@ NeuralNetwork::NeuralNetwork() :
     props::Epochs(), props::TrainingBatchSize(), props::SavePath(),
     props::ContinueTrain(), props::SaveBestPath(), props::MemoryOptimization(),
     props::MemorySwap(), props::MemorySwapPath(), props::MemorySwapLookahead(),
-    props::TensorFormat(), props::ModelTensorDataType()),
+    props::TensorFormat(), props::ModelTensorDataType(), props::LossScale()),
   load_path(std::string()),
   epoch_idx(0),
   iter(0),
@@ -88,7 +88,7 @@ NeuralNetwork::NeuralNetwork(AppContext app_context_) :
     props::Epochs(), props::TrainingBatchSize(), props::SavePath(),
     props::ContinueTrain(), props::SaveBestPath(), props::MemoryOptimization(),
     props::MemorySwap(), props::MemorySwapPath(), props::MemorySwapLookahead(),
-    props::TensorFormat(), props::ModelTensorDataType()),
+    props::TensorFormat(), props::ModelTensorDataType(), props::LossScale()),
   load_path(std::string()),
   epoch_idx(0),
   iter(0),
@@ -179,8 +179,9 @@ int NeuralNetwork::compile() {
   const std::string tensor_type =
     to_string(std::get<props::ModelTensorDataType>(model_flex_props));
 
+  const float loss_scale = std::get<props::LossScale>(model_flex_props);
   model_graph = NetworkGraph(memory_swap, memory_swap_path, lookahead,
-                             tensor_format, tensor_type);
+                             tensor_format, tensor_type, loss_scale);
 
   model_graph.setMemoryOptimizations(
     std::get<props::MemoryOptimization>(model_flex_props));
@@ -1018,6 +1019,7 @@ int NeuralNetwork::train_run(
 
   auto train_for_iteration =
     [this, stop_cb, stop_user_data](RunStats &stat, DataBuffer &buffer) {
+      ml_loge("train for iteration");
       forwarding(true, stop_cb, stop_user_data);
       backwarding(iter++, stop_cb, stop_user_data);
 

diff --git a/nntrainer/models/neuralnet.h b/nntrainer/models/neuralnet.h
index 457b7d1e97e7d301ca0be6e9ba35ff3ab6190beb..a2923ae8a7a9714603885b56def7e5b283bac19e 100644 (file)
--- a/nntrainer/models/neuralnet.h
+++ b/nntrainer/models/neuralnet.h
@@ -221,10 +221,11 @@ public:
   /**
    * @brief     Forward Propagation of the neural network
    */
-  sharedConstTensors forwarding(bool training = true,
-                                std::function<bool(void *userdata)> stop_cb =
-                                  [](void *user_data) { return false; },
-                                void *user_data = nullptr);
+  sharedConstTensors forwarding(
+    bool training = true,
+    std::function<bool(void *userdata)> stop_cb =
+      [](void *user_data) { return false; },
+    void *user_data = nullptr);
 
   /**
    * @brief     Forward Propagation of the neural network
@@ -239,12 +240,11 @@ public:
   /**
    * @brief     Incremental forward Propagation of the neural network
    */
-  sharedConstTensors
-  incremental_forwarding(unsigned int from, unsigned int to,
-                         bool training = true,
-                         std::function<bool(void *userdata)> stop_cb =
-                           [](void *user_data) { return false; },
-                         void *user_data = nullptr);
+  sharedConstTensors incremental_forwarding(
+    unsigned int from, unsigned int to, bool training = true,
+    std::function<bool(void *userdata)> stop_cb =
+      [](void *user_data) { return false; },
+    void *user_data = nullptr);
 
   /**
    * @brief     Incremental forward Propagation of the neural network
@@ -261,10 +261,11 @@ public:
    * @brief     Backward Propagation of the neural network
    * @param[in] iteration Iteration Number for the optimizer
    */
-  void backwarding(int iteration,
-                   std::function<bool(void *userdata)> stop_cb =
-                     [](void *user_data) { return false; },
-                   void *user_data = nullptr);
+  void backwarding(
+    int iteration,
+    std::function<bool(void *userdata)> stop_cb =
+      [](void *user_data) { return false; },
+    void *user_data = nullptr);
 
   /**
    * @copydoc Model::save(const std::string &file_path, ml::train::ModelFormat
@@ -329,13 +330,14 @@ public:
    * @retval #ML_ERROR_NONE Successful.
    * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
    */
-  int train(const std::vector<std::string> &values = {},
-            std::function<bool(void *)> stop_cb =
-              [](void *stop_user_data) { return false; },
-            void *stop_user_data = nullptr,
-            std::function<void(void *)> epoch_complete_cb =
-              [](void *epoch_user_data) { return false; },
-            void *epoch_user_data = nullptr) override;
+  int train(
+    const std::vector<std::string> &values = {},
+    std::function<bool(void *)> stop_cb =
+      [](void *stop_user_data) { return false; },
+    void *stop_user_data = nullptr,
+    std::function<void(void *)> epoch_complete_cb =
+      [](void *epoch_user_data) { return false; },
+    void *epoch_user_data = nullptr) override;
 
   /**
    * @brief     Run NeuralNetwork inference
@@ -622,12 +624,11 @@ s   * @retval shared_ptr<const Tensor>
                const std::string file_path) override;
 
 private:
-  using FlexiblePropTypes =
-    std::tuple<props::Epochs, props::TrainingBatchSize, props::SavePath,
-               props::ContinueTrain, props::SaveBestPath,
-               props::MemoryOptimization, props::MemorySwap,
-               props::MemorySwapPath, props::MemorySwapLookahead,
-               props::TensorFormat, props::ModelTensorDataType>;
+  using FlexiblePropTypes = std::tuple<
+    props::Epochs, props::TrainingBatchSize, props::SavePath,
+    props::ContinueTrain, props::SaveBestPath, props::MemoryOptimization,
+    props::MemorySwap, props::MemorySwapPath, props::MemorySwapLookahead,
+    props::TensorFormat, props::ModelTensorDataType, props::LossScale>;
   using RigidPropTypes =
     std::tuple<props::LossType, std::vector<props::InputConnection>,
                std::vector<props::LabelLayer>, props::ClipGradByGlobalNorm>;
@@ -709,12 +710,12 @@ private:
    * @retval #ML_ERROR_NONE Successful.
    * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
    */
-  int train_run(std::function<bool(void *)> stop_cb =
-                  [](void *) { return false; },
-                void *user_data = nullptr,
-                std::function<void(void *)> epoch_complete_cb =
-                  [](void *) { return false; },
-                void *data = nullptr);
+  int train_run(
+    std::function<bool(void *)> stop_cb = [](void *) { return false; },
+    void *user_data = nullptr,
+    std::function<void(void *)> epoch_complete_cb =
+      [](void *) { return false; },
+    void *data = nullptr);
 
   /**
    * @brief     Swap function for the class