opt = nnopt->optimizer;
}
- returnable f = [&]() { return opt->setProperty(arg_list); };
+ returnable f = [&]() {
+ opt->setProperty(arg_list);
+ return ML_ERROR_NONE;
+ };
status = nntrainer_exception_boundary(f);
/**
* @brief set Optimizer Parameters
* @param[in] values Optimizer Parameter list
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
* @details This function accepts vector of properties in the format -
* { std::string property_name, void * property_val, ...}
*/
- virtual int setProperty(std::vector<std::string> values) = 0;
+ virtual void setProperty(const std::vector<std::string> &values) = 0;
};
/**
createOptimizer(const std::vector<std::string> &props = {}) {
std::unique_ptr<Optimizer> ptr = std::make_unique<T>();
- if (ptr->setProperty(props) != ML_ERROR_NONE) {
- throw std::invalid_argument("Set properties failed for optimizer");
- }
+ ptr->setProperty(props);
return ptr;
}
#include <dynamic_training_optimization.h>
#include <tensor.h>
#include <util_func.h>
+#include <weight.h>
namespace nntrainer {
DynamicTrainingOptimization::DynamicTrainingOptimization(int threshold_,
}
}
- status = model.opt->setProperty(optimizer_prop);
- NN_RETURN_STATUS();
+ try {
+ model.opt->setProperty(optimizer_prop);
+ } catch (std::exception &e) {
+ ml_loge("%s %s", typeid(e).name(), e.what());
+ return ML_ERROR_INVALID_PARAMETER;
+ } catch (...) {
+ ml_loge("Settings properties to optimizer failed.");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
return status;
}
#include <neuralnet.h>
#include <nntrainer_error.h>
#include <nntrainer_log.h>
+#include <optimizer_context.h>
#include <parse_util.h>
#include <profiler.h>
#include <time_dist.h>
if (apply_gradient && node->getTrainable()) {
// TODO: ask network_graph for weights of node and then remove
// getWeightObject() interface from layer_context
+ RunOptimizerContext opt_context;
for (unsigned int idx = 0; idx < node->getNumWeights(); idx++) {
auto &weight = node->getWeightObject(idx);
if (weight.hasGradient()) {
weight.calcRegularizationGradient();
- opt->applyGradient(weight, iteration);
+ opt_context = RunOptimizerContext(&weight, iteration);
+ opt->applyGradient(opt_context);
}
}
}
return ll;
}
-void Adam::applyGradient(Weight &weight, int iteration) {
+void Adam::applyGradient(RunOptimizerContext &context) {
- Tensor &x_grad = weight.getGradientRef();
+ Tensor &x_grad = context.getGradient();
// This is implementation of adam from original paper.
// This is not deleted intentionally.
return 1 / (sqrtDouble(f) + this->epsilon);
};
- Tensor &wm = weight.getOptimizerVariableRef(AdamParams::wm);
- Tensor &wv = weight.getOptimizerVariableRef(AdamParams::wv);
+ Tensor &wm = context.getOptimizerVariable(AdamParams::wm);
+ Tensor &wv = context.getOptimizerVariable(AdamParams::wv);
wm.multiply_i(beta1);
wm.add_i(x_grad, 1.0f - beta1);
x_grad = wv.apply(sqrtEps, x_grad);
x_grad.multiply_i(wm);
- weight.applyGradient(getLearningRate(iteration));
+ context.applyGradient(getLearningRate(context.getIteration()));
}
void Adam::setProperty(const std::string &key, const std::string &value) {
epsilon(ep) {}
/**
- * @copydoc applyGradient(Weight &weight, int iteration)
+ * @copydoc applyGradient(RunOptimizerContext &context)
*/
- void applyGradient(Weight &weight, int iteration);
+ void applyGradient(RunOptimizerContext &context);
/**
* @copydoc Optimizer::getType()
double getLearningRate(size_t iteration) const;
/**
- * @copydoc setProperty(const std::string &key,
- const std::string &value)
- */
- void setProperty(const std::string &key, const std::string &value);
-
- /**
* @copydoc Optimizer::getOptimizerVariableDim(const TensorDim &dim)
*/
std::vector<TensorDim> getOptimizerVariableDim(const TensorDim &dim) override;
double beta1; /** momentum for grad */
double beta2; /** momentum for grad**2 */
double epsilon; /** epsilon to protect overflow */
+
+ /**
+ * @copydoc LayerImpl::setProperty(const std::string &key,
+ const std::string &value)
+ */
+ void setProperty(const std::string &key, const std::string &value);
};
} /* namespace nntrainer */
'adam.cpp',
'optimizer_devel.cpp',
'optimizer_impl.cpp',
- 'sgd.cpp'
+ 'sgd.cpp',
+ 'optimizer_context.cpp'
]
optimizer_headers = [
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2021 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file optimizer_context.h
+ * @date 30 July 2021
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is the layer context for each layer
+ */
+
+#include <optimizer_context.h>
+#include <weight.h>
+
+namespace nntrainer {
+
+/**
+ * @brief Get the Weight tensor object
+ */
+Tensor &RunOptimizerContext::getWeight() const {
+ return weight->getVariableRef();
+}
+
+/**
+ * @brief Get the Weight Gradient tensor object
+ */
+Tensor &RunOptimizerContext::getGradient() const {
+ return weight->getGradientRef();
+}
+
+/**
+ * @brief Get the optimizer variable associated to this weight
+ */
+Tensor &RunOptimizerContext::getOptimizerVariable(unsigned int idx) const {
+ return weight->getOptimizerVariableRef(idx);
+}
+
+/**
+ * @brief Apply the gradient with the given learning rate
+ */
+void RunOptimizerContext::applyGradient(double lr) const {
+ weight->applyGradient(lr);
+}
+} // namespace nntrainer
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2021 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file optimizer_context.h
+ * @date 30 July 2021
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is the optimizer context for each optimizer
+ */
+
+#ifndef __OPTIMIZER_CONTEXT_H__
+#define __OPTIMIZER_CONTEXT_H__
+
+#include <memory>
+#include <vector>
+
+#include <tensor.h>
+
+namespace nntrainer {
+
+class Weight;
+
+/**
+ * @class Op Context class for all optimizers
+ * @brief Class for Optimizer context
+ *
+ * @details This provides for the optimizer execution.
+ */
+class RunOptimizerContext {
+public:
+ /**
+ * @brief Construct a new Run Optimizer Context object
+ *
+ */
+ RunOptimizerContext(Weight *w = nullptr, size_t iter = 0) :
+ weight(w),
+ iteration(iter) {}
+
+ /**
+ * @brief Get the Weight tensor object
+ *
+ * @return Tensor& Reference to the weight tensor
+ */
+ Tensor &getWeight() const;
+
+ /**
+ * @brief Get the Weight Gradient tensor object
+ *
+ * @return Tensor& Reference to the weight grad tensor
+ */
+ Tensor &getGradient() const;
+
+ /**
+ * @brief Get the optimizer variable associated to this weight
+ *
+ * @param idx Identifier of the associated weight
+ * @return Tensor& Reference to the optimizer variable
+ */
+ Tensor &getOptimizerVariable(unsigned int idx) const;
+
+ /**
+ * @brief Check if run context is set and is ready to use
+ *
+ * @return true if ready, else false
+ */
+ bool readyToUse() const { return weight != nullptr; }
+
+ /**
+ * @brief Apply the gradient with the given learning rate
+ *
+ * @param lr learning rate
+ */
+ void applyGradient(double lr) const;
+
+ /**
+ * @brief Get the current iteration value
+ *
+ * @return iteration value
+ */
+ size_t getIteration() const { return iteration; }
+
+private:
+ Weight *weight; /**< weights for the optimizer */
+ size_t iteration; /**< iteration number */
+};
+
+} // namespace nntrainer
+#endif // __OPTIMIZER_CONTEXT_H__
#include <fstream>
#include <iostream>
-#include <nntrainer_log.h>
+#include <nntrainer_error.h>
#include <optimizer_devel.h>
#include <parse_util.h>
#include <util_func.h>
namespace nntrainer {
-int Optimizer::setProperty(std::vector<std::string> values) {
- int status = ML_ERROR_NONE;
-
- for (unsigned int i = 0; i < values.size(); ++i) {
- std::string key;
- std::string value;
-
- status = getKeyValue(values[i], key, value);
- NN_RETURN_STATUS();
-
- if (value.empty()) {
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- try {
- /// @note this calls derived setProperty if available
- setProperty(key, value);
- } catch (...) {
- return ML_ERROR_INVALID_PARAMETER;
- }
+void Optimizer::setProperty(const std::vector<std::string> &values) {
+ if (!values.empty()) {
+ std::string msg = "[OptimizerDevel] Unknown properties count " +
+ std::to_string(values.size());
+ throw exception::not_supported(msg);
}
-
- try {
- checkValidation();
- } catch (...) {
- return ML_ERROR_INVALID_PARAMETER;
- }
- return status;
-}
-
-void Optimizer::checkValidation() const {
- if (getLearningRate() <= 0.0f)
- throw std::invalid_argument("Learning rate must be positive");
-}
-
-void Optimizer::setProperty(const std::string &key, const std::string &value) {
- int status = ML_ERROR_NONE;
- unsigned int type = parseOptProperty(key);
-
- switch (type) {
- default:
- ml_loge("Error: Unknown Optimizer Property Key");
- status = ML_ERROR_INVALID_PARAMETER;
- break;
- }
-
- throw_status(status);
}
void Optimizer::read(std::ifstream &file) {
#include <memory>
#include <optimizer.h>
+#include <optimizer_context.h>
#include <tensor.h>
-#include <weight.h>
namespace nntrainer {
/**
* @brief apply gradient to weight
- * @param[in] params Weight
- * @param[in] iteration nth epoch number
+ * @param[in] context Optimizer context
*/
- virtual void applyGradient(Weight ¶m, int iteration) = 0;
+ virtual void applyGradient(RunOptimizerContext &context) = 0;
/**
* @brief set Optimizer Parameters
* @param[in] values Optimizer Parameter list
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
*/
- virtual int setProperty(std::vector<std::string> values);
+ virtual void setProperty(const std::vector<std::string> &values);
/**
* @brief Default allowed properties
};
/**
- * @brief setProperty individually
- * @note By passing empty string, this can validate if @a type is valid
- * @param[in] key key to be passed as string
- * @param[in] value value to be passed, if empty string is passed, do nothing
- * but throws error when @a type is invalid
- * @exception exception::not_supported when string type is not valid for
- * the particular layer
- * @exception std::invalid_argument invalid argument
- */
- virtual void setProperty(const std::string &key,
- const std::string &value) = 0;
-
- /**
* @brief initialize optimizer.
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
*/
- virtual int initialize() = 0;
+ virtual void initialize(){};
/**
* @brief Read Training optimizer paramters from file
virtual void save(std::ofstream &file);
/**
- * @brief validate the optimizer
- */
- virtual void checkValidation() const;
-
- /**
* @brief Get dimension of extra variables if the optimizer needs any.
* @param dim Dimension of tensor to be added as a optimizer variable
* @return Vector of dimensions
namespace nntrainer {
-int OptimizerImpl::initialize() { return ML_ERROR_NONE; }
+void OptimizerImpl::setProperty(const std::vector<std::string> &values) {
+ /// @todo: deprecate this in favor of loadProperties
+ for (unsigned int i = 0; i < values.size(); ++i) {
+ std::string key;
+ std::string value;
+ std::stringstream ss;
+
+ if (getKeyValue(values[i], key, value) != ML_ERROR_NONE) {
+ throw std::invalid_argument("Error parsing the property: " + values[i]);
+ }
+
+ if (value.empty()) {
+ ss << "value is empty: key: " << key << ", value: " << value;
+ throw std::invalid_argument(ss.str());
+ }
+
+ /// @note this calls derived setProperty if available
+ setProperty(key, value);
+ }
+}
void OptimizerImpl::setProperty(const std::string &key,
const std::string &value) {
status = setBoolean(continue_train, value);
break;
default:
- Optimizer::setProperty(key, value);
- status = ML_ERROR_NONE;
+ status = ML_ERROR_INVALID_PARAMETER;
break;
}
return ll;
}
+
} // namespace nntrainer
double getLearningRate(size_t iteration) const;
/**
- * @brief setProperty individually
- * @note By passing empty string, this can validate if @a type is valid
- * @param[in] key key to be passed as string
- * @param[in] value value to be passed, if empty string is passed, do nothing
- * but throws error when @a type is invalid
- * @exception exception::not_supported when string type is not valid for
- * the particular layer
- * @exception std::invalid_argument invalid argument
- */
- virtual void setProperty(const std::string &key, const std::string &value);
-
- /**
- * @brief initialize optimizer.
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ * @copydoc Layer::setProperty(const std::vector<std::string> &values)
*/
- virtual int initialize();
+ virtual void setProperty(const std::vector<std::string> &values);
/**
* @brief Get dimension of extra variables if the optimizer needs any.
float decay_rate; /** decay rate for learning rate */
unsigned int decay_steps; /** decay steps for learning rate */
bool continue_train; /** Continue training with previous tensors for adam */
+
+ /**
+ * @brief setProperty individually
+ * @note By passing empty string, this can validate if @a type is valid
+ * @param[in] key key to be passed as string
+ * @param[in] value value to be passed, if empty string is passed, do nothing
+ * but throws error when @a type is invalid
+ * @exception exception::not_supported when string type is not valid for
+ * the particular layer
+ * @exception std::invalid_argument invalid argument
+ */
+ virtual void setProperty(const std::string &key, const std::string &value);
};
} /* namespace nntrainer */
/**
* @brief apply gradient to weight
- * @param[in] params Weight
- * @param[in] iteration nth epoch number
+ * @param[in] context Optimizer context
*/
- void applyGradient(Weight ¶m, int iteration) override {
- optimizer_devel->applyGradient(param, iteration);
+ void applyGradient(RunOptimizerContext &context) override {
+ optimizer_devel->applyGradient(context);
}
/**
* @retval #ML_ERROR_NONE Successful.
* @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
*/
- int setProperty(std::vector<std::string> values) override {
- return optimizer_devel->setProperty(values);
- }
-
- /**
- * @brief setProperty individually
- * @note By passing empty string, this can validate if @a type is valid
- * @param[in] key key to be passed as string
- * @param[in] value value to be passed, if empty string is passed, do nothing
- * but throws error when @a type is invalid
- * @exception exception::not_supported when string type is not valid for
- * the particular layer
- * @exception std::invalid_argument invalid argument
- */
- void setProperty(const std::string &key, const std::string &value) override {
- optimizer_devel->setProperty(key, value);
+ void setProperty(const std::vector<std::string> &values) override {
+ optimizer_devel->setProperty(values);
}
/**
* @retval #ML_ERROR_NONE Successful.
* @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
*/
- int initialize() override { return optimizer_devel->initialize(); }
+ void initialize() override { optimizer_devel->initialize(); }
/**
* @brief Read Training optimizer paramters from file
void save(std::ofstream &file) override { optimizer_devel->save(file); }
/**
- * @brief validate the optimizer
- */
- void checkValidation() const override { optimizer_devel->checkValidation(); }
-
- /**
* @brief Get dimension of extra variables if the optimizer needs any.
* @param dim Dimension of tensor to be added as a optimizer variable
* @return Vector of dimensions
namespace nntrainer {
-void SGD::applyGradient(Weight &weight, int iteration) {
- weight.applyGradient(getLearningRate(iteration));
+void SGD::applyGradient(RunOptimizerContext &context) {
+ context.applyGradient(getLearningRate(context.getIteration()));
}
} // namespace nntrainer
SGD(float lr = 0.0001f, Args... args) : OptimizerImpl(lr, args...) {}
/**
- * @copydoc applyGradient(Weight &weight, int iteration)
+ * @copydoc applyGradient(RunOptimizerContext &context)
*/
- void applyGradient(Weight &weight, int iteration);
+ void applyGradient(RunOptimizerContext &context);
/**
* @copydoc Optimizer::getType()
double getLearningRate(size_t iteration) const override { return 1.0f; }
- int setProperty(std::vector<std::string> values) override { return 1; }
-
- int initialize() override { return 0; }
+ void setProperty(const std::vector<std::string> &values) override {}
std::vector<nntrainer::TensorDim>
getOptimizerVariableDim(const nntrainer::TensorDim &dim) override {
return std::vector<nntrainer::TensorDim>();
}
- void setProperty(const std::string &key, const std::string &value) override {}
-
- void checkValidation() const override {}
-
- void applyGradient(nntrainer::Weight &weight, int iteration) override {}
+ void applyGradient(nntrainer::RunOptimizerContext &context) override {}
};
/**
/** Minimal custom optimizer example which define only necessary functions */
const std::string getType() const override { return "identity_optimizer"; }
- int initialize() override { return 0; }
-
double getLearningRate(size_t iteration) const override { return 1.0f; }
std::vector<nntrainer::TensorDim>
return std::vector<nntrainer::TensorDim>();
}
- void applyGradient(nntrainer::Weight &weight, int iteration) override {}
+ void applyGradient(nntrainer::RunOptimizerContext &context) override {}
};
/**
projects / platform / core / ml / nntrainer.git / commitdiff