From d88d8d78ceada194fbfd1e306658ccde99cb7ad2 Mon Sep 17 00:00:00 2001 From: Parichay Kapoor Date: Fri, 30 Jul 2021 14:26:25 +0900 Subject: [PATCH] [optimizer] Cleanup optimizer and remove dependency on weight Cleanup optimizer interface where some extra interfaces have been removed, and arguments interface has been updated. Further, the dependency on weight header has been removed, and added an optimizer context interface which provides an interface to weight related tensors required by the optimizer. Signed-off-by: Parichay Kapoor --- api/capi/src/nntrainer.cpp | 5 +- api/ccapi/include/optimizer.h | 8 +- nntrainer/models/dynamic_training_optimization.cpp | 1 + nntrainer/models/model_loader.cpp | 11 ++- nntrainer/models/neuralnet.cpp | 5 +- nntrainer/optimizers/adam.cpp | 10 +-- nntrainer/optimizers/adam.h | 16 ++-- nntrainer/optimizers/meson.build | 3 +- nntrainer/optimizers/optimizer_context.cpp | 45 +++++++++++ nntrainer/optimizers/optimizer_context.h | 90 ++++++++++++++++++++++ nntrainer/optimizers/optimizer_devel.cpp | 53 ++----------- nntrainer/optimizers/optimizer_devel.h | 33 ++------ nntrainer/optimizers/optimizer_impl.cpp | 25 +++++- nntrainer/optimizers/optimizer_impl.h | 30 ++++---- nntrainer/optimizers/plugged_optimizer.h | 32 ++------ nntrainer/optimizers/sgd.cpp | 4 +- nntrainer/optimizers/sgd.h | 4 +- test/unittest/unittest_nntrainer_appcontext.cpp | 14 +--- 18 files changed, 230 insertions(+), 159 deletions(-) create mode 100644 nntrainer/optimizers/optimizer_context.cpp create mode 100644 nntrainer/optimizers/optimizer_context.h diff --git a/api/capi/src/nntrainer.cpp b/api/capi/src/nntrainer.cpp index c2f2479..853ea1b 100644 --- a/api/capi/src/nntrainer.cpp +++ b/api/capi/src/nntrainer.cpp @@ -792,7 +792,10 @@ int ml_train_optimizer_set_property(ml_train_optimizer_h optimizer, ...) { opt = nnopt->optimizer; } - returnable f = [&]() { return opt->setProperty(arg_list); }; + returnable f = [&]() { + opt->setProperty(arg_list); + return ML_ERROR_NONE; + }; status = nntrainer_exception_boundary(f); diff --git a/api/ccapi/include/optimizer.h b/api/ccapi/include/optimizer.h index 72efac1..bb56e73 100644 --- a/api/ccapi/include/optimizer.h +++ b/api/ccapi/include/optimizer.h @@ -71,12 +71,10 @@ public: /** * @brief set Optimizer Parameters * @param[in] values Optimizer Parameter list - * @retval #ML_ERROR_NONE Successful. - * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter. * @details This function accepts vector of properties in the format - * { std::string property_name, void * property_val, ...} */ - virtual int setProperty(std::vector values) = 0; + virtual void setProperty(const std::vector &values) = 0; }; /** @@ -105,9 +103,7 @@ std::unique_ptr createOptimizer(const std::vector &props = {}) { std::unique_ptr ptr = std::make_unique(); - if (ptr->setProperty(props) != ML_ERROR_NONE) { - throw std::invalid_argument("Set properties failed for optimizer"); - } + ptr->setProperty(props); return ptr; } diff --git a/nntrainer/models/dynamic_training_optimization.cpp b/nntrainer/models/dynamic_training_optimization.cpp index b47279f..8eed273 100644 --- a/nntrainer/models/dynamic_training_optimization.cpp +++ b/nntrainer/models/dynamic_training_optimization.cpp @@ -17,6 +17,7 @@ #include #include #include +#include namespace nntrainer { DynamicTrainingOptimization::DynamicTrainingOptimization(int threshold_, diff --git a/nntrainer/models/model_loader.cpp b/nntrainer/models/model_loader.cpp index dd42670..6301344 100644 --- a/nntrainer/models/model_loader.cpp +++ b/nntrainer/models/model_loader.cpp @@ -180,8 +180,15 @@ int ModelLoader::loadModelConfigIni(dictionary *ini, NeuralNetwork &model) { } } - status = model.opt->setProperty(optimizer_prop); - NN_RETURN_STATUS(); + try { + model.opt->setProperty(optimizer_prop); + } catch (std::exception &e) { + ml_loge("%s %s", typeid(e).name(), e.what()); + return ML_ERROR_INVALID_PARAMETER; + } catch (...) { + ml_loge("Settings properties to optimizer failed."); + return ML_ERROR_INVALID_PARAMETER; + } return status; } diff --git a/nntrainer/models/neuralnet.cpp b/nntrainer/models/neuralnet.cpp index c32c077..b7b165b 100644 --- a/nntrainer/models/neuralnet.cpp +++ b/nntrainer/models/neuralnet.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -295,11 +296,13 @@ void NeuralNetwork::backwarding(std::shared_ptr node, int iteration, if (apply_gradient && node->getTrainable()) { // TODO: ask network_graph for weights of node and then remove // getWeightObject() interface from layer_context + RunOptimizerContext opt_context; for (unsigned int idx = 0; idx < node->getNumWeights(); idx++) { auto &weight = node->getWeightObject(idx); if (weight.hasGradient()) { weight.calcRegularizationGradient(); - opt->applyGradient(weight, iteration); + opt_context = RunOptimizerContext(&weight, iteration); + opt->applyGradient(opt_context); } } } diff --git a/nntrainer/optimizers/adam.cpp b/nntrainer/optimizers/adam.cpp index 0884a35..47040f5 100644 --- a/nntrainer/optimizers/adam.cpp +++ b/nntrainer/optimizers/adam.cpp @@ -40,9 +40,9 @@ double Adam::getLearningRate(size_t iteration) const { return ll; } -void Adam::applyGradient(Weight &weight, int iteration) { +void Adam::applyGradient(RunOptimizerContext &context) { - Tensor &x_grad = weight.getGradientRef(); + Tensor &x_grad = context.getGradient(); // This is implementation of adam from original paper. // This is not deleted intentionally. @@ -66,8 +66,8 @@ void Adam::applyGradient(Weight &weight, int iteration) { return 1 / (sqrtDouble(f) + this->epsilon); }; - Tensor &wm = weight.getOptimizerVariableRef(AdamParams::wm); - Tensor &wv = weight.getOptimizerVariableRef(AdamParams::wv); + Tensor &wm = context.getOptimizerVariable(AdamParams::wm); + Tensor &wv = context.getOptimizerVariable(AdamParams::wv); wm.multiply_i(beta1); wm.add_i(x_grad, 1.0f - beta1); @@ -77,7 +77,7 @@ void Adam::applyGradient(Weight &weight, int iteration) { x_grad = wv.apply(sqrtEps, x_grad); x_grad.multiply_i(wm); - weight.applyGradient(getLearningRate(iteration)); + context.applyGradient(getLearningRate(context.getIteration())); } void Adam::setProperty(const std::string &key, const std::string &value) { diff --git a/nntrainer/optimizers/adam.h b/nntrainer/optimizers/adam.h index b5d2a6f..66c879e 100644 --- a/nntrainer/optimizers/adam.h +++ b/nntrainer/optimizers/adam.h @@ -36,9 +36,9 @@ public: epsilon(ep) {} /** - * @copydoc applyGradient(Weight &weight, int iteration) + * @copydoc applyGradient(RunOptimizerContext &context) */ - void applyGradient(Weight &weight, int iteration); + void applyGradient(RunOptimizerContext &context); /** * @copydoc Optimizer::getType() @@ -51,12 +51,6 @@ public: double getLearningRate(size_t iteration) const; /** - * @copydoc setProperty(const std::string &key, - const std::string &value) - */ - void setProperty(const std::string &key, const std::string &value); - - /** * @copydoc Optimizer::getOptimizerVariableDim(const TensorDim &dim) */ std::vector getOptimizerVariableDim(const TensorDim &dim) override; @@ -82,6 +76,12 @@ private: double beta1; /** momentum for grad */ double beta2; /** momentum for grad**2 */ double epsilon; /** epsilon to protect overflow */ + + /** + * @copydoc LayerImpl::setProperty(const std::string &key, + const std::string &value) + */ + void setProperty(const std::string &key, const std::string &value); }; } /* namespace nntrainer */ diff --git a/nntrainer/optimizers/meson.build b/nntrainer/optimizers/meson.build index a59ecb0..866b07b 100644 --- a/nntrainer/optimizers/meson.build +++ b/nntrainer/optimizers/meson.build @@ -2,7 +2,8 @@ optimizer_sources = [ 'adam.cpp', 'optimizer_devel.cpp', 'optimizer_impl.cpp', - 'sgd.cpp' + 'sgd.cpp', + 'optimizer_context.cpp' ] optimizer_headers = [ diff --git a/nntrainer/optimizers/optimizer_context.cpp b/nntrainer/optimizers/optimizer_context.cpp new file mode 100644 index 0000000..da4cd1f --- /dev/null +++ b/nntrainer/optimizers/optimizer_context.cpp @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 +/** + * Copyright (C) 2021 Parichay Kapoor + * + * @file optimizer_context.h + * @date 30 July 2021 + * @see https://github.com/nnstreamer/nntrainer + * @author Parichay Kapoor + * @bug No known bugs except for NYI items + * @brief This is the layer context for each layer + */ + +#include +#include + +namespace nntrainer { + +/** + * @brief Get the Weight tensor object + */ +Tensor &RunOptimizerContext::getWeight() const { + return weight->getVariableRef(); +} + +/** + * @brief Get the Weight Gradient tensor object + */ +Tensor &RunOptimizerContext::getGradient() const { + return weight->getGradientRef(); +} + +/** + * @brief Get the optimizer variable associated to this weight + */ +Tensor &RunOptimizerContext::getOptimizerVariable(unsigned int idx) const { + return weight->getOptimizerVariableRef(idx); +} + +/** + * @brief Apply the gradient with the given learning rate + */ +void RunOptimizerContext::applyGradient(double lr) const { + weight->applyGradient(lr); +} +} // namespace nntrainer diff --git a/nntrainer/optimizers/optimizer_context.h b/nntrainer/optimizers/optimizer_context.h new file mode 100644 index 0000000..5dc7b3f --- /dev/null +++ b/nntrainer/optimizers/optimizer_context.h @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: Apache-2.0 +/** + * Copyright (C) 2021 Parichay Kapoor + * + * @file optimizer_context.h + * @date 30 July 2021 + * @see https://github.com/nnstreamer/nntrainer + * @author Parichay Kapoor + * @bug No known bugs except for NYI items + * @brief This is the optimizer context for each optimizer + */ + +#ifndef __OPTIMIZER_CONTEXT_H__ +#define __OPTIMIZER_CONTEXT_H__ + +#include +#include + +#include + +namespace nntrainer { + +class Weight; + +/** + * @class Op Context class for all optimizers + * @brief Class for Optimizer context + * + * @details This provides for the optimizer execution. + */ +class RunOptimizerContext { +public: + /** + * @brief Construct a new Run Optimizer Context object + * + */ + RunOptimizerContext(Weight *w = nullptr, size_t iter = 0) : + weight(w), + iteration(iter) {} + + /** + * @brief Get the Weight tensor object + * + * @return Tensor& Reference to the weight tensor + */ + Tensor &getWeight() const; + + /** + * @brief Get the Weight Gradient tensor object + * + * @return Tensor& Reference to the weight grad tensor + */ + Tensor &getGradient() const; + + /** + * @brief Get the optimizer variable associated to this weight + * + * @param idx Identifier of the associated weight + * @return Tensor& Reference to the optimizer variable + */ + Tensor &getOptimizerVariable(unsigned int idx) const; + + /** + * @brief Check if run context is set and is ready to use + * + * @return true if ready, else false + */ + bool readyToUse() const { return weight != nullptr; } + + /** + * @brief Apply the gradient with the given learning rate + * + * @param lr learning rate + */ + void applyGradient(double lr) const; + + /** + * @brief Get the current iteration value + * + * @return iteration value + */ + size_t getIteration() const { return iteration; } + +private: + Weight *weight; /**< weights for the optimizer */ + size_t iteration; /**< iteration number */ +}; + +} // namespace nntrainer +#endif // __OPTIMIZER_CONTEXT_H__ diff --git a/nntrainer/optimizers/optimizer_devel.cpp b/nntrainer/optimizers/optimizer_devel.cpp index dd88bb2..8718c45 100644 --- a/nntrainer/optimizers/optimizer_devel.cpp +++ b/nntrainer/optimizers/optimizer_devel.cpp @@ -15,60 +15,19 @@ #include #include -#include +#include #include #include #include namespace nntrainer { -int Optimizer::setProperty(std::vector values) { - int status = ML_ERROR_NONE; - - for (unsigned int i = 0; i < values.size(); ++i) { - std::string key; - std::string value; - - status = getKeyValue(values[i], key, value); - NN_RETURN_STATUS(); - - if (value.empty()) { - return ML_ERROR_INVALID_PARAMETER; - } - - try { - /// @note this calls derived setProperty if available - setProperty(key, value); - } catch (...) { - return ML_ERROR_INVALID_PARAMETER; - } +void Optimizer::setProperty(const std::vector &values) { + if (!values.empty()) { + std::string msg = "[OptimizerDevel] Unknown properties count " + + std::to_string(values.size()); + throw exception::not_supported(msg); } - - try { - checkValidation(); - } catch (...) { - return ML_ERROR_INVALID_PARAMETER; - } - return status; -} - -void Optimizer::checkValidation() const { - if (getLearningRate() <= 0.0f) - throw std::invalid_argument("Learning rate must be positive"); -} - -void Optimizer::setProperty(const std::string &key, const std::string &value) { - int status = ML_ERROR_NONE; - unsigned int type = parseOptProperty(key); - - switch (type) { - default: - ml_loge("Error: Unknown Optimizer Property Key"); - status = ML_ERROR_INVALID_PARAMETER; - break; - } - - throw_status(status); } void Optimizer::read(std::ifstream &file) { diff --git a/nntrainer/optimizers/optimizer_devel.h b/nntrainer/optimizers/optimizer_devel.h index be41621..fa1ec28 100644 --- a/nntrainer/optimizers/optimizer_devel.h +++ b/nntrainer/optimizers/optimizer_devel.h @@ -19,8 +19,8 @@ #include #include +#include #include -#include namespace nntrainer { @@ -48,18 +48,15 @@ public: /** * @brief apply gradient to weight - * @param[in] params Weight - * @param[in] iteration nth epoch number + * @param[in] context Optimizer context */ - virtual void applyGradient(Weight ¶m, int iteration) = 0; + virtual void applyGradient(RunOptimizerContext &context) = 0; /** * @brief set Optimizer Parameters * @param[in] values Optimizer Parameter list - * @retval #ML_ERROR_NONE Successful. - * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter. */ - virtual int setProperty(std::vector values); + virtual void setProperty(const std::vector &values); /** * @brief Default allowed properties @@ -89,24 +86,9 @@ public: }; /** - * @brief setProperty individually - * @note By passing empty string, this can validate if @a type is valid - * @param[in] key key to be passed as string - * @param[in] value value to be passed, if empty string is passed, do nothing - * but throws error when @a type is invalid - * @exception exception::not_supported when string type is not valid for - * the particular layer - * @exception std::invalid_argument invalid argument - */ - virtual void setProperty(const std::string &key, - const std::string &value) = 0; - - /** * @brief initialize optimizer. - * @retval #ML_ERROR_NONE Successful. - * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter. */ - virtual int initialize() = 0; + virtual void initialize(){}; /** * @brief Read Training optimizer paramters from file @@ -121,11 +103,6 @@ public: virtual void save(std::ofstream &file); /** - * @brief validate the optimizer - */ - virtual void checkValidation() const; - - /** * @brief Get dimension of extra variables if the optimizer needs any. * @param dim Dimension of tensor to be added as a optimizer variable * @return Vector of dimensions diff --git a/nntrainer/optimizers/optimizer_impl.cpp b/nntrainer/optimizers/optimizer_impl.cpp index 48ab683..d929cb7 100644 --- a/nntrainer/optimizers/optimizer_impl.cpp +++ b/nntrainer/optimizers/optimizer_impl.cpp @@ -24,7 +24,26 @@ namespace nntrainer { -int OptimizerImpl::initialize() { return ML_ERROR_NONE; } +void OptimizerImpl::setProperty(const std::vector &values) { + /// @todo: deprecate this in favor of loadProperties + for (unsigned int i = 0; i < values.size(); ++i) { + std::string key; + std::string value; + std::stringstream ss; + + if (getKeyValue(values[i], key, value) != ML_ERROR_NONE) { + throw std::invalid_argument("Error parsing the property: " + values[i]); + } + + if (value.empty()) { + ss << "value is empty: key: " << key << ", value: " << value; + throw std::invalid_argument(ss.str()); + } + + /// @note this calls derived setProperty if available + setProperty(key, value); + } +} void OptimizerImpl::setProperty(const std::string &key, const std::string &value) { @@ -45,8 +64,7 @@ void OptimizerImpl::setProperty(const std::string &key, status = setBoolean(continue_train, value); break; default: - Optimizer::setProperty(key, value); - status = ML_ERROR_NONE; + status = ML_ERROR_INVALID_PARAMETER; break; } @@ -62,4 +80,5 @@ double OptimizerImpl::getLearningRate(size_t iteration) const { return ll; } + } // namespace nntrainer diff --git a/nntrainer/optimizers/optimizer_impl.h b/nntrainer/optimizers/optimizer_impl.h index 90331c5..96bc2cb 100644 --- a/nntrainer/optimizers/optimizer_impl.h +++ b/nntrainer/optimizers/optimizer_impl.h @@ -88,23 +88,9 @@ public: double getLearningRate(size_t iteration) const; /** - * @brief setProperty individually - * @note By passing empty string, this can validate if @a type is valid - * @param[in] key key to be passed as string - * @param[in] value value to be passed, if empty string is passed, do nothing - * but throws error when @a type is invalid - * @exception exception::not_supported when string type is not valid for - * the particular layer - * @exception std::invalid_argument invalid argument - */ - virtual void setProperty(const std::string &key, const std::string &value); - - /** - * @brief initialize optimizer. - * @retval #ML_ERROR_NONE Successful. - * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter. + * @copydoc Layer::setProperty(const std::vector &values) */ - virtual int initialize(); + virtual void setProperty(const std::vector &values); /** * @brief Get dimension of extra variables if the optimizer needs any. @@ -121,6 +107,18 @@ protected: float decay_rate; /** decay rate for learning rate */ unsigned int decay_steps; /** decay steps for learning rate */ bool continue_train; /** Continue training with previous tensors for adam */ + + /** + * @brief setProperty individually + * @note By passing empty string, this can validate if @a type is valid + * @param[in] key key to be passed as string + * @param[in] value value to be passed, if empty string is passed, do nothing + * but throws error when @a type is invalid + * @exception exception::not_supported when string type is not valid for + * the particular layer + * @exception std::invalid_argument invalid argument + */ + virtual void setProperty(const std::string &key, const std::string &value); }; } /* namespace nntrainer */ diff --git a/nntrainer/optimizers/plugged_optimizer.h b/nntrainer/optimizers/plugged_optimizer.h index 32a6af4..28eaba5 100644 --- a/nntrainer/optimizers/plugged_optimizer.h +++ b/nntrainer/optimizers/plugged_optimizer.h @@ -83,11 +83,10 @@ public: /** * @brief apply gradient to weight - * @param[in] params Weight - * @param[in] iteration nth epoch number + * @param[in] context Optimizer context */ - void applyGradient(Weight ¶m, int iteration) override { - optimizer_devel->applyGradient(param, iteration); + void applyGradient(RunOptimizerContext &context) override { + optimizer_devel->applyGradient(context); } /** @@ -96,22 +95,8 @@ public: * @retval #ML_ERROR_NONE Successful. * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter. */ - int setProperty(std::vector values) override { - return optimizer_devel->setProperty(values); - } - - /** - * @brief setProperty individually - * @note By passing empty string, this can validate if @a type is valid - * @param[in] key key to be passed as string - * @param[in] value value to be passed, if empty string is passed, do nothing - * but throws error when @a type is invalid - * @exception exception::not_supported when string type is not valid for - * the particular layer - * @exception std::invalid_argument invalid argument - */ - void setProperty(const std::string &key, const std::string &value) override { - optimizer_devel->setProperty(key, value); + void setProperty(const std::vector &values) override { + optimizer_devel->setProperty(values); } /** @@ -119,7 +104,7 @@ public: * @retval #ML_ERROR_NONE Successful. * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter. */ - int initialize() override { return optimizer_devel->initialize(); } + void initialize() override { optimizer_devel->initialize(); } /** * @brief Read Training optimizer paramters from file @@ -134,11 +119,6 @@ public: void save(std::ofstream &file) override { optimizer_devel->save(file); } /** - * @brief validate the optimizer - */ - void checkValidation() const override { optimizer_devel->checkValidation(); } - - /** * @brief Get dimension of extra variables if the optimizer needs any. * @param dim Dimension of tensor to be added as a optimizer variable * @return Vector of dimensions diff --git a/nntrainer/optimizers/sgd.cpp b/nntrainer/optimizers/sgd.cpp index 7d145b1..e03dc96 100644 --- a/nntrainer/optimizers/sgd.cpp +++ b/nntrainer/optimizers/sgd.cpp @@ -15,8 +15,8 @@ namespace nntrainer { -void SGD::applyGradient(Weight &weight, int iteration) { - weight.applyGradient(getLearningRate(iteration)); +void SGD::applyGradient(RunOptimizerContext &context) { + context.applyGradient(getLearningRate(context.getIteration())); } } // namespace nntrainer diff --git a/nntrainer/optimizers/sgd.h b/nntrainer/optimizers/sgd.h index d8df7f1..155c78c 100644 --- a/nntrainer/optimizers/sgd.h +++ b/nntrainer/optimizers/sgd.h @@ -31,9 +31,9 @@ public: SGD(float lr = 0.0001f, Args... args) : OptimizerImpl(lr, args...) {} /** - * @copydoc applyGradient(Weight &weight, int iteration) + * @copydoc applyGradient(RunOptimizerContext &context) */ - void applyGradient(Weight &weight, int iteration); + void applyGradient(RunOptimizerContext &context); /** * @copydoc Optimizer::getType() diff --git a/test/unittest/unittest_nntrainer_appcontext.cpp b/test/unittest/unittest_nntrainer_appcontext.cpp index 2e2bc9f..de436bc 100644 --- a/test/unittest/unittest_nntrainer_appcontext.cpp +++ b/test/unittest/unittest_nntrainer_appcontext.cpp @@ -106,20 +106,14 @@ public: double getLearningRate(size_t iteration) const override { return 1.0f; } - int setProperty(std::vector values) override { return 1; } - - int initialize() override { return 0; } + void setProperty(const std::vector &values) override {} std::vector getOptimizerVariableDim(const nntrainer::TensorDim &dim) override { return std::vector(); } - void setProperty(const std::string &key, const std::string &value) override {} - - void checkValidation() const override {} - - void applyGradient(nntrainer::Weight &weight, int iteration) override {} + void applyGradient(nntrainer::RunOptimizerContext &context) override {} }; /** @@ -131,8 +125,6 @@ public: /** Minimal custom optimizer example which define only necessary functions */ const std::string getType() const override { return "identity_optimizer"; } - int initialize() override { return 0; } - double getLearningRate(size_t iteration) const override { return 1.0f; } std::vector @@ -140,7 +132,7 @@ public: return std::vector(); } - void applyGradient(nntrainer::Weight &weight, int iteration) override {} + void applyGradient(nntrainer::RunOptimizerContext &context) override {} }; /** -- 2.7.4