}
std::vector<std::string> optimizer_prop = {};
- optimizer_prop.push_back(
- {"learning_rate=" +
- std::string(iniparser_getstring(
- ini, "Model:Learning_rate",
- std::to_string(model.opt->getLearningRate()).c_str()))});
-
- if (model.opt->getType() == SGD::type || model.opt->getType() == Adam::type) {
- std::shared_ptr<OptimizerImpl> opt_impl =
- std::static_pointer_cast<OptimizerImpl>(model.opt);
-
- optimizer_prop.push_back(
- {"decay_steps=" + std::string(iniparser_getstring(
- ini, "Model:Decay_steps",
- std::to_string(opt_impl->getDecaySteps()).c_str()))});
- optimizer_prop.push_back(
- {"decay_rate=" + std::string(iniparser_getstring(
- ini, "Model:Decay_rate",
- std::to_string(opt_impl->getDecayRate()).c_str()))});
-
- if (opt_impl->getType() == "adam") {
- std::shared_ptr<Adam> opt_adam = std::static_pointer_cast<Adam>(opt_impl);
-
- optimizer_prop.push_back(
- {"beta1=" +
- std::string(iniparser_getstring(
- ini, "Model:Beta1", std::to_string(opt_adam->getBeta1()).c_str()))});
- optimizer_prop.push_back(
- {"beta2=" +
- std::string(iniparser_getstring(
- ini, "Model:Beta2", std::to_string(opt_adam->getBeta2()).c_str()))});
- optimizer_prop.push_back(
- {"epsilon=" + std::string(iniparser_getstring(
- ini, "Model:Epsilon",
- std::to_string(opt_adam->getEpsilon()).c_str()))});
+
+ /** push only if ini_key exist as prop_key=ini_value */
+ auto maybe_push = [ini](std::vector<std::string> &prop_vector,
+ const std::string &ini_key,
+ const std::string &prop_key) {
+ constexpr const char *LOCAL_UNKNOWN = "unknown";
+ std::string ini_value =
+ iniparser_getstring(ini, ini_key.c_str(), LOCAL_UNKNOWN);
+ if (!istrequal(ini_value, LOCAL_UNKNOWN)) {
+ prop_vector.push_back(prop_key + "=" + ini_value);
}
+ };
+
+ const std::vector<std::string> deprecated_optimizer_keys = {
+ "learning_rate", "decay_rate", "decay_steps", "beta1", "beta2", "epsilon"};
+ for (const auto &key : deprecated_optimizer_keys) {
+ maybe_push(optimizer_prop, "Model:" + key, key);
}
try {
#include <adam.h>
#include <nntrainer_error.h>
#include <nntrainer_log.h>
-#include <parse_util.h>
+#include <node_exporter.h>
#include <util_func.h>
namespace nntrainer {
+Adam::Adam() : adam_props(PropsB1(), PropsB2(), PropsEpsilon()) {
+ /** default properties */
+ setProperty({"learning_rate=0.001"});
+ auto &[b1, b2, eps] = adam_props;
+ b1.set(0.9f);
+ b2.set(0.999f);
+ eps.set(1.0e-7f);
+}
+
+Adam::~Adam() {}
+
enum AdamParams { wm, wv };
std::vector<TensorDim> Adam::getOptimizerVariableDim(const TensorDim &dim) {
return {dim, dim};
}
+void Adam::exportTo(Exporter &exporter, const ExportMethods &method) const {
+ exporter.saveResult(adam_props, method, this);
+ OptimizerImpl::exportTo(exporter, method);
+}
+
+void Adam::setProperty(const std::vector<std::string> &values) {
+ auto left = loadProperties(values, adam_props);
+ OptimizerImpl::setProperty(left);
+}
+
double Adam::getLearningRate(size_t iteration) const {
+ auto &beta1 = std::get<PropsB1>(adam_props).get();
+ auto &beta2 = std::get<PropsB2>(adam_props).get();
double ll = OptimizerImpl::getLearningRate(iteration);
std::function<float(double)> biasCorrection = [&](float f) {
}
void Adam::applyGradient(RunOptimizerContext &context) {
-
Tensor &x_grad = context.getGradient();
+ auto &beta1 = std::get<PropsB1>(adam_props).get();
+ auto &beta2 = std::get<PropsB2>(adam_props).get();
+ auto &epsilon = std::get<PropsEpsilon>(adam_props).get();
+
// This is implementation of adam from original paper.
// This is not deleted intentionally.
// float biasCorrection1 = 1 - pow(beta1, iteration + 1);
// .add(epsilon);
// x.add_i(wm.divide(denom), -ll / biasCorrection1);
- std::function<double(double)> sqrtEps = [&](double f) {
- return 1 / (sqrtDouble(f) + this->epsilon);
+ std::function<double(double)> sqrtEps = [epsilon](double f) {
+ return 1 / (sqrtDouble(f) + epsilon);
};
Tensor &wm = context.getOptimizerVariable(AdamParams::wm);
context.applyGradient(getLearningRate(context.getIteration()));
}
-void Adam::setProperty(const std::string &key, const std::string &value) {
- int status = ML_ERROR_NONE;
- PropertyType type = static_cast<PropertyType>(parseOptProperty(key));
-
- switch (type) {
- case PropertyType::beta1:
- status = setDouble(beta1, value);
- break;
- case PropertyType::beta2:
- status = setDouble(beta2, value);
- break;
- case PropertyType::epsilon:
- status = setDouble(epsilon, value);
- break;
- default:
- OptimizerImpl::setProperty(key, value);
- status = ML_ERROR_NONE;
- break;
- }
-
- throw_status(status);
-}
-
} // namespace nntrainer
#define __ADAM_H__
#ifdef __cplusplus
+#include <tuple>
+
+#include <base_properties.h>
#include <optimizer_impl.h>
namespace nntrainer {
+/**
+ * @brief Beta 1 props
+ *
+ */
+class PropsB1 : public Property<double> {
+public:
+ static constexpr const char *key = "beta1"; /**< unique key to access */
+ using prop_tag = double_prop_tag; /**< property type */
+};
+
+/**
+ * @brief Beta 2 props
+ *
+ */
+class PropsB2 : public Property<double> {
+public:
+ static constexpr const char *key = "beta2"; /**< unique key to access */
+ using prop_tag = double_prop_tag; /**< property type */
+};
+
+/**
+ * @brief epsilon props
+ * @todo move this to common props
+ *
+ */
+class PropsEpsilon : public Property<double> {
+public:
+ static constexpr const char *key = "epsilon"; /**< unique key to access */
+ using prop_tag = double_prop_tag; /**< property type */
+};
+
/**
* @class Adam optimizer class
* @brief Adam optimizer
class Adam : public OptimizerImpl {
public:
/**
- * @brief Constructor of Optimizer Class
+ * @brief Construct a new Adam object
+ *
*/
- template <typename... Args>
- Adam(float lr = 0.001f, double b1 = 0.9f, double b2 = 0.999f,
- double ep = 1.0e-7f, Args... args) :
- OptimizerImpl(lr, args...),
- beta1(b1),
- beta2(b2),
- epsilon(ep) {}
+ Adam();
+
+ /**
+ * @brief Destroy the Adam object
+ *
+ */
+ ~Adam();
/**
* @copydoc applyGradient(RunOptimizerContext &context)
std::vector<TensorDim> getOptimizerVariableDim(const TensorDim &dim) override;
/**
- * @brief get beta1
+ * @copydoc Optimizer::exportTo(Exporter &exporter, const ExportMethods&
+ * method)
*/
- double getBeta1() { return beta1; };
+ void exportTo(Exporter &exporter, const ExportMethods &method) const override;
- /**
- * @brief get beta2
- */
- double getBeta2() { return beta2; };
+ inline static const std::string type = "adam";
/**
- * @brief get epsilon
+ * @copydoc Optimizer::setProperty(const std::vector<std::string> &values)
*/
- double getEpsilon() { return epsilon; }
-
- inline static const std::string type = "adam";
+ virtual void setProperty(const std::vector<std::string> &values) override;
private:
- double beta1; /** momentum for grad */
- double beta2; /** momentum for grad**2 */
- double epsilon; /** epsilon to protect overflow */
-
- /**
- * @copydoc LayerImpl::setProperty(const std::string &key,
- const std::string &value)
- */
- void setProperty(const std::string &key, const std::string &value);
+ std::tuple<PropsB1, PropsB2, PropsEpsilon> adam_props;
};
} /* namespace nntrainer */
namespace nntrainer {
+class Exporter;
+enum class ExportMethods;
+
/**
* @class Optimizer Base class for optimizers
* @brief Base class for all optimizers
virtual void setProperty(const std::vector<std::string> &values);
/**
- * @brief Default allowed properties
- * Available for all optimizers
- * - learning_rate : float
- *
- * Available for SGD and Adam optimizers
- * - decay_rate : float,
- * - decay_steps : float,
- *
- * Available for Adam optimizer
- * - beta1 : float,
- * - beta2 : float,
- * - epsilon : float,
+ * @brief this function helps exporting the optimizer in a predefined format,
+ * while workarounding issue caused by templated function type eraser
*
- * @todo: convert to string
+ * @param exporter exporter that conatins exporting logic
+ * @param method enum value to identify how it should be exported to
*/
- enum class PropertyType {
- learning_rate = 0,
- decay_rate = 1,
- decay_steps = 2,
- beta1 = 3,
- beta2 = 4,
- epsilon = 5,
- continue_train = 6,
- unknown = 7,
- };
+ virtual void exportTo(Exporter &exporter, const ExportMethods &method) const {
+ }
/**
* @brief finalize optimizer.
#include <cmath>
#include <nntrainer_error.h>
#include <nntrainer_log.h>
+#include <node_exporter.h>
#include <optimizer_impl.h>
-#include <parse_util.h>
#include <util_func.h>
namespace nntrainer {
-void OptimizerImpl::setProperty(const std::vector<std::string> &values) {
- /// @todo: deprecate this in favor of loadProperties
- for (unsigned int i = 0; i < values.size(); ++i) {
- std::string key;
- std::string value;
- std::stringstream ss;
-
- if (getKeyValue(values[i], key, value) != ML_ERROR_NONE) {
- throw std::invalid_argument("Error parsing the property: " + values[i]);
- }
-
- if (value.empty()) {
- ss << "value is empty: key: " << key << ", value: " << value;
- throw std::invalid_argument(ss.str());
- }
+OptimizerImpl::OptimizerImpl() :
+ optimizer_impl_props(PropsLR(), PropsDecayRate(), PropsDecaySteps()) {}
- /// @note this calls derived setProperty if available
- setProperty(key, value);
- }
+void OptimizerImpl::setProperty(const std::vector<std::string> &values) {
+ auto left = loadProperties(values, optimizer_impl_props);
+ NNTR_THROW_IF(left.size(), std::invalid_argument)
+ << "[OptimizerImpl] There are unparsed properties";
}
-void OptimizerImpl::setProperty(const std::string &key,
- const std::string &value) {
- int status = ML_ERROR_NONE;
- PropertyType type = static_cast<PropertyType>(parseOptProperty(key));
-
- switch (type) {
- case PropertyType::learning_rate:
- status = setFloat(learning_rate, value);
- break;
- case PropertyType::decay_steps:
- status = setUint(decay_steps, value);
- break;
- case PropertyType::decay_rate:
- status = setFloat(decay_rate, value);
- break;
- case PropertyType::continue_train:
- status = setBoolean(continue_train, value);
- break;
- default:
- status = ML_ERROR_INVALID_PARAMETER;
- break;
- }
-
- throw_status(status);
+void OptimizerImpl::exportTo(Exporter &exporter,
+ const ExportMethods &method) const {
+ exporter.saveResult(optimizer_impl_props, method, this);
}
double OptimizerImpl::getLearningRate(size_t iteration) const {
- double ll = learning_rate;
- if (decay_steps != 0) {
+ auto &[float_lr, decay_rate, decay_steps] = optimizer_impl_props;
+ double ll = float_lr;
+
+ if (!decay_steps.empty() && !decay_rate.empty()) {
ll = ll * pow(decay_rate, (iteration / (float)decay_steps));
}
#define __OPTIMIZER_IMPL_H__
#ifdef __cplusplus
+#include <tuple>
+
+#include <base_properties.h>
#include <optimizer_devel.h>
namespace nntrainer {
+/**
+ * @brief Learning Rate props
+ *
+ */
+class PropsLR : public Property<float> {
+public:
+ static constexpr const char *key =
+ "learning_rate"; /**< unique key to access */
+ using prop_tag = float_prop_tag; /**< property type */
+};
+
+/**
+ * @brief Decay rate property
+ *
+ */
+class PropsDecayRate : public Property<float> {
+public:
+ static constexpr const char *key = "decay_rate"; /**< unique key to access */
+ using prop_tag = float_prop_tag; /**< property type */
+};
+
+/**
+ * @brief decay steps property
+ *
+ */
+class PropsDecaySteps : public PositiveIntegerProperty {
+public:
+ static constexpr const char *key = "decay_steps"; /**< unique key to access */
+ using prop_tag = uint_prop_tag; /**< property type */
+};
+
/**
* @class Optimizer Base class for optimizers
* @brief Basic implementation class for nntrainer supported optimizers
public:
/**
- * @brief Default Constructor of Optimizer Class
+ * @brief Construct a new Optimizer Impl object
+ *
*/
- OptimizerImpl(float lr, float decay_rate = 1.0f, unsigned int decay_steps = 0,
- float continue_train = false) :
- Optimizer(),
- learning_rate(lr),
- decay_rate(decay_rate),
- decay_steps(decay_steps),
- continue_train(continue_train) {}
+ OptimizerImpl();
/**
* @brief copy constructor
* @brief Move assignment operator.
* @parma[in] rhs OptimizerImpl to be moved.
*/
- OptimizerImpl &operator=(OptimizerImpl &&rhs) = default;
-
- /**
- * @brief get Learning Rate
- * @retval Learning rate in float
- */
- float getLearningRate() const { return learning_rate; };
-
- /**
- * @brief get Decay Rate for learning rate decay
- * @retval decay rate
- */
- float getDecayRate() const { return decay_rate; };
-
- /**
- * @brief get Decay Steps for learning rate decay
- * @retval decay steps
- */
- float getDecaySteps() const { return decay_steps; };
+ OptimizerImpl &operator=(OptimizerImpl &&rhs) noexcept = default;
/**
* @brief get Learning Rate for the given iteration
double getLearningRate(size_t iteration) const;
/**
- * @copydoc Layer::setProperty(const std::vector<std::string> &values)
+ * @copydoc Optimizer::setProperty(const std::vector<std::string> &values)
*/
virtual void setProperty(const std::vector<std::string> &values);
+ /**
+ * @copydoc Optimizer::exportTo(Exporter &exporter, const ExportMethods&
+ * method)
+ */
+ void exportTo(Exporter &exporter, const ExportMethods &method) const override;
+
/**
* @brief Get dimension of extra variables if the optimizer needs any.
* @param dim Dimension of tensor to be added as a optimizer variable
}
protected:
- float learning_rate; /**< learning rate */
- float decay_rate; /** decay rate for learning rate */
- unsigned int decay_steps; /** decay steps for learning rate */
- bool continue_train; /** Continue training with previous tensors for adam */
-
- /**
- * @brief setProperty individually
- * @note By passing empty string, this can validate if @a type is valid
- * @param[in] key key to be passed as string
- * @param[in] value value to be passed, if empty string is passed, do nothing
- * but throws error when @a type is invalid
- * @exception exception::not_supported when string type is not valid for
- * the particular layer
- * @exception std::invalid_argument invalid argument
- */
- virtual void setProperty(const std::string &key, const std::string &value);
+ std::tuple<PropsLR, PropsDecayRate, PropsDecaySteps> optimizer_impl_props;
};
} /* namespace nntrainer */
namespace nntrainer {
+SGD::SGD() { setProperty({"learning_rate=0.0001"}); }
+
void SGD::applyGradient(RunOptimizerContext &context) {
context.applyGradient(getLearningRate(context.getIteration()));
}
class SGD : public OptimizerImpl {
public:
/**
- * @brief Constructor of Optimizer Class
+ * @brief Construct a new SGD object
+ *
*/
- template <typename... Args>
- SGD(float lr = 0.0001f, Args... args) : OptimizerImpl(lr, args...) {}
+ SGD();
/**
* @copydoc applyGradient(RunOptimizerContext &context)
return std::stof(value);
}
+template <>
+std::string
+str_converter<double_prop_tag, double>::to_string(const double &value) {
+ return std::to_string(value);
+}
+
+template <>
+double
+str_converter<double_prop_tag, double>::from_string(const std::string &value) {
+ return std::stod(value);
+}
+
template <>
std::string str_converter<dimension_prop_tag, TensorDim>::to_string(
const TensorDim &dimension) {
*/
struct float_prop_tag {};
+/**
+ * @brief property is treated as double
+ *
+ */
+struct double_prop_tag {};
+
/**
* @brief property is treated as string
*
float str_converter<float_prop_tag, float>::from_string(
const std::string &value);
+/**
+ * @copydoc template <typename Tag, typename DataType> struct str_converter
+ */
+template <>
+std::string
+str_converter<double_prop_tag, double>::to_string(const double &value);
+
+/**
+ * @copydoc template <typename Tag, typename DataType> struct str_converter
+ */
+template <>
+double
+str_converter<double_prop_tag, double>::from_string(const std::string &value);
+
/**
* @brief convert dispatcher (to string)
*
std::string propToStr(unsigned int type) { return property_string[type]; }
-unsigned int parseOptProperty(std::string property) {
- unsigned int i;
-
- /**
- * @brief Optimizer Properties
- * learning_rate = 0,
- * decay_rate = 1,
- * decay_steps = 2
- * beta1 = 3,
- * beta2 = 4,
- * epsilon = 5,
- */
- std::array<std::string, 7> property_string = {
- "learning_rate", "decay_rate", "decay_steps", "beta1", "beta2", "epsilon"};
-
- for (i = 0; i < property_string.size(); i++) {
- unsigned int size = (property_string[i].size() > property.size())
- ? property_string[i].size()
- : property.size();
-
- if (!strncasecmp(property_string[i].c_str(), property.c_str(), size)) {
- return (i);
- }
- }
-
- return (unsigned int)Optimizer::PropertyType::unknown;
-}
-
int setUint(unsigned int &val, const std::string &str) {
int status = ML_ERROR_NONE;
try {
*/
unsigned int parseType(std::string ll, InputType t);
-/**
- * @brief Parsing Optimizer Property
- * @param[in] property string to be parsed
- * @retval int enumerated type
- */
-unsigned int parseOptProperty(std::string property);
-
} /* namespace nntrainer */
#endif /* __cplusplus */
projects / platform / core / ml / nntrainer.git / commitdiff