/* set property for fc layer */
status = ml_train_layer_set_property(
layers[1], "unit= 10", "activation=softmax", "bias_initializer=zeros",
- "weight_decay=l2norm", "weight_decay_lambda=0.005",
+ "weight_regularizer=l2norm", "weight_regularizer_constant=0.005",
"weight_initializer=xavier_uniform", NULL);
NN_RETURN_STATUS();
/* set property for fc layer */
status = ml_train_layer_set_property(
layers[1], "unit= 10", "activation=softmax", "bias_initializer=zeros",
- "weight_decay=l2norm", "weight_decay_lambda=0.005",
+ "weight_regularizer=l2norm", "weight_regularizer_constant=0.005",
"weight_initializer=xavier_uniform", NULL);
NN_RETURN_STATUS();
| relu | relu function | set as layer propery |
| softmax | softmax function | set as layer propery |
| weight_initializer | Weight Initialization | Xavier(Normal/Uniform), LeCun(Normal/Uniform), HE(Normal/Unifor) |
- | weight_decay | weight decay ( L2Norm only ) | needs set weight_decay_param & type |
+ | weight_regularizer | weight decay ( L2Norm only ) | needs set weight_regularizer_param & type |
| learnig_rate_decay | learning rate decay | need to set step |
### Tensor
| Keyward | Loss Name | Description |
|:-------:|:---:|:---|
| weight_initializer | Weight Initialization | Xavier(Normal/Uniform), LeCun(Normal/Uniform), HE(Normal/Unifor) |
- | weight_decay | weight decay ( L2Norm only ) | needs set weight_decay_param & type |
+ | weight_regularizer | weight decay ( L2Norm only ) | needs set weight_regularizer_constant & type |
| learnig_rate_decay | learning rate decay | need to set step |
### APIs
* // Many of these hyperparmeters are optional
* status = ml_train_layer_set_property(handle, "input_shape=1:1:6270",
* "unit=10", "bias_initializer=zeros", "activation=sigmoid",
- * "weight_decay=l2_norm", "weight_initializer=he_uniform", NULL);
+ * "weight_regularizer=l2_norm", "weight_initializer=he_uniform", NULL);
* if (status != ML_ERROR_NONE) {
* // Handle error case
* ml_train_layer_destroy(handle);
set weight decay
* l2norm : L2 normalization
-9. ```weight_decay_lambda = <float>```
+9. ```weight_regularizer_constant = <float>```
coefficient for weight decay
| Layer | Properties |
|:-------:|:---|
- | conv2d |<ul><li>filters</li><li>kernel_size</li><li>stride</li><li>padding</li><li>normalization</li><li>standardization</li><li>input_shape</li><li>bias_init_zero</li><li>activation</li><li>flatten</li><li>weight_decay</li><li>weight_decay_lambda</li><li>weight_initializer</li></ul>|
+ | conv2d |<ul><li>filters</li><li>kernel_size</li><li>stride</li><li>padding</li><li>normalization</li><li>standardization</li><li>input_shape</li><li>bias_init_zero</li><li>activation</li><li>flatten</li><li>weight_decay</li><li>weight_regularizer_constant</li><li>weight_initializer</li></ul>|
| pooling2d | <ul><li>pooling</li><li>pool_size</li><li>stride</li><li>padding</li></ul> |
| flatten | - |
- | fully_connected | <lu><li>unit</li><li>normalization</li><li>standardization</li><li>input_shape</li><li>bias_initializer</li><li>activation</li><li>flatten</li><li>weight_decay</li><li>weight_decay_lambda</li><li>weight_initializer</li></lu>|
+ | fully_connected | <lu><li>unit</li><li>normalization</li><li>standardization</li><li>input_shape</li><li>bias_initializer</li><li>activation</li><li>flatten</li><li>weight_decay</li><li>weight_regularizer_constant</li><li>weight_initializer</li></lu>|
| input | <lu><li>normalization </li><li>standardization</li><li>input_shape</li><li>flatten</li></lu>|
| batch_normalization | <lu><li>epsilon</li><li>flatten</li></lu> |
type(LAYER_UNKNOWN),
loss(0.0f),
activation_type(ACT_NONE),
- weight_decay(),
+ weight_regularizer(),
weight_initializer(WEIGHT_XAVIER_UNIFORM),
bias_initializer(WEIGHT_ZEROS),
flatten(false),
* 3. standardization : bool
* 4. activation : string (type)
* 5. epsilon : float
- * 6. weight_decay : string (type)
- * 7. weight_decay_lambda : float
+ * 6. weight_regularizer : string (type)
+ * 7. weight_regularizer_constant : float
* 8. unit : int
* 9. weight_initializer : string (type)
* 10. filter_size : int
standardization = 2,
activation = 3,
epsilon = 4,
- weight_decay = 5,
- weight_decay_lambda = 6,
+ weight_regularizer = 5,
+ weight_regularizer_constant = 6,
unit = 7,
weight_initializer = 8,
bias_initializer = 9,
* @brief set weight decay parameters
* @param[in] w struct for weight decay
*/
- void setWeightDecay(WeightDecayParam w) { weight_decay = w; }
+ void setWeightRegularizer(WeightRegularizerParam w) {
+ weight_regularizer = w;
+ }
/**
* @brief set Weight Initialization Type
* @brief check if current layer's weight decay type is l2norm
* @return bool is weightdecay type is L2 Norm
*/
- bool isWeightDecayL2Norm() {
- return weight_decay.type == WeightDecayType::l2norm;
+ bool isWeightRegularizerL2Norm() {
+ return weight_regularizer.type == WeightRegularizerType::l2norm;
}
/**
* @brief Input Tensor
ActiType activation_type;
- WeightDecayParam weight_decay;
+ WeightRegularizerParam weight_regularizer;
WeightInitializer weight_initializer; /** initializer for weights */
* 1. Regression
* 2. Unknown (equivalent to none)
*/
-enum class WeightDecayType { l2norm = 0, regression = 1, unknown = 2 };
+enum class WeightRegularizerType { l2norm = 0, regression = 1, unknown = 2 };
/**
* @brief type for the Weight Decay hyper-parameter
*/
-typedef struct WeightDecayParam_ {
- WeightDecayType type;
- float lambda;
-
- WeightDecayParam_() : type(WeightDecayType::unknown), lambda(0.0f) {}
-} WeightDecayParam;
+typedef struct WeightRegularizerParam_ {
+ WeightRegularizerType type;
+ float constant;
+
+ WeightRegularizerParam_() :
+ type(WeightRegularizerType::unknown),
+ constant(0.0f) {}
+} WeightRegularizerParam;
/**
* @brief type for the Optimizor to save hyper-parameter
* 3. ACTI ( Activation Token )
* 4. LAYER ( Layer Token )
* 5. WEIGHT_INIT ( Weight Initializer Token )
- * 7. WEIGHT_DECAY ( Weight Decay Token )
+ * 7. WEIGHT_REGULARIZER ( Weight Decay Token )
* 8. PADDING ( Padding Token )
* 9. POOLING ( Pooling Token )
* 9. UNKNOWN
TOKEN_ACTI,
TOKEN_LAYER,
TOKEN_WEIGHT_INIT,
- TOKEN_WEIGHT_DECAY,
+ TOKEN_WEIGHT_REGULARIZER,
TOKEN_PADDING,
TOKEN_POOLING,
TOKEN_UNKNOWN
#endif
loss = 0.0f;
- if (weight_decay.type == WeightDecayType::l2norm) {
+ if (weight_regularizer.type == WeightRegularizerType::l2norm) {
for (unsigned int i = 0; i < filter_size; ++i) {
Tensor &weight = paramsAt(i).weight;
- loss += weight_decay.lambda * 0.5f * (weight.l2norm());
+ loss += weight_regularizer.constant * 0.5f * (weight.l2norm());
}
loss /= filter_size;
}
Tensor &filters = paramsAt(i).weight;
delK = delK.chain()
- .applyIf(this->isWeightDecayL2Norm(), _LIFT(add_i), filters,
- weight_decay.lambda)
+ .applyIf(this->isWeightRegularizerL2Norm(), _LIFT(add_i),
+ filters, weight_regularizer.constant)
.run();
}
hidden = input.dot(weight);
hidden.add_i(bias);
- if (weight_decay.type == WeightDecayType::l2norm) {
- loss = weight_decay.lambda * 0.5f * (weight.l2norm());
+ if (weight_regularizer.type == WeightRegularizerType::l2norm) {
+ loss = weight_regularizer.constant * 0.5f * (weight.l2norm());
}
return MAKE_SHARED_TENSOR(hidden);
djdb = derivative->sum(0);
djdw = input.dot(*derivative, true, false);
- if (isWeightDecayL2Norm())
- djdw.add_i(weight, weight_decay.lambda);
+ if (isWeightRegularizerL2Norm())
+ djdw.add_i(weight, weight_regularizer.constant);
djdw = djdw.sum(0);
if (trainable) {
throw_status(status);
}
break;
- case PropertyType::weight_decay:
+ case PropertyType::weight_regularizer:
if (!value.empty()) {
- weight_decay.type = (WeightDecayType)parseType(value, TOKEN_WEIGHT_DECAY);
- if (weight_decay.type == WeightDecayType::unknown) {
+ weight_regularizer.type =
+ (WeightRegularizerType)parseType(value, TOKEN_WEIGHT_REGULARIZER);
+ if (weight_regularizer.type == WeightRegularizerType::unknown) {
throw std::invalid_argument("[Layer] Unknown Weight decay");
}
}
break;
- case PropertyType::weight_decay_lambda:
+ case PropertyType::weight_regularizer_constant:
if (!value.empty()) {
- status = setFloat(weight_decay.lambda, value);
+ status = setFloat(weight_regularizer.constant, value);
throw_status(status);
}
break;
void Layer::printProperties(std::ostream &out) {
out << "Trainable: " << trainable << std::endl;
- printIfValid(out, PropertyType::weight_decay,
- static_cast<int>(weight_decay.type));
- printIfValid(out, PropertyType::weight_decay_lambda, weight_decay.lambda);
+ printIfValid(out, PropertyType::weight_regularizer,
+ static_cast<int>(weight_regularizer.type));
+ printIfValid(out, PropertyType::weight_regularizer_constant,
+ weight_regularizer.constant);
}
void Layer::printMetric(std::ostream &out) {
* "L2Norm" : squared norm regularization
* "Regression" : Regression
*/
- std::array<std::string, 2> weight_decay_string = {"l2norm", "regression"};
+ std::array<std::string, 2> weight_regularizer_string = {"l2norm",
+ "regression"};
/**
* @brief Weight Decay String from configure file
}
ret = (unsigned int)WeightInitializer::WEIGHT_UNKNOWN;
break;
- case TOKEN_WEIGHT_DECAY:
- for (i = 0; i < weight_decay_string.size(); i++) {
- if (!strncasecmp(weight_decay_string[i].c_str(), ll.c_str(),
- weight_decay_string[i].size())) {
+ case TOKEN_WEIGHT_REGULARIZER:
+ for (i = 0; i < weight_regularizer_string.size(); i++) {
+ if (!strncasecmp(weight_regularizer_string[i].c_str(), ll.c_str(),
+ weight_regularizer_string[i].size())) {
return (i);
}
}
- ret = (unsigned int)WeightDecayType::unknown;
+ ret = (unsigned int)WeightRegularizerType::unknown;
break;
case TOKEN_PADDING:
for (i = 0; i < padding_string.size(); i++) {
* standardization = 2
* activation = 3
* epsilon = 4
- * weight_decay = 5
- * weight_decay_lambda = 6
+ * weight_regularizer = 5
+ * weight_regularizer_constant = 6
* unit = 7
* weight_initializer = 8
* bias_initializer = 9
* Pooling2DLayer has 12, 13, 14, 15 properties.
* BatchNormalizationLayer has 0, 1, 5, 6, 7 properties.
*/
-static std::array<std::string, 22> property_string = {"input_shape",
- "normalization",
- "standardization",
- "activation",
- "epsilon",
- "weight_decay",
- "weight_decay_lambda",
- "unit",
- "weight_initializer",
- "bias_initializer",
- "filters",
- "kernel_size",
- "stride",
- "padding",
- "pool_size",
- "pooling",
- "flatten",
- "name",
- "num_inputs",
- "num_outputs",
- "batch_size",
- "unknown"};
+static std::array<std::string, 22> property_string = {
+ "input_shape",
+ "normalization",
+ "standardization",
+ "activation",
+ "epsilon",
+ "weight_regularizer",
+ "weight_regularizer_constant",
+ "unit",
+ "weight_initializer",
+ "bias_initializer",
+ "filters",
+ "kernel_size",
+ "stride",
+ "padding",
+ "pool_size",
+ "pooling",
+ "flatten",
+ "name",
+ "num_inputs",
+ "num_outputs",
+ "batch_size",
+ "unknown"};
unsigned int parseLayerProperty(std::string property) {
unsigned int i;
"\n"
"Loss = cross"
"\n"
- "Weight_Decay = l2norm"
+ "Weight_Regularizer = l2norm"
"\n"
- "weight_Decay_Lambda = 0.005"
+ "weight_regularizer_constant = 0.005"
"\n"
"Save_Path = 'model.bin'"
"\n"
"\n"
"Loss = cross"
"\n"
- "Weight_Decay = l2norm"
+ "Weight_Regularizer = l2norm"
"\n"
- "weight_Decay_Lambda = 0.005"
+ "weight_regularizer_constant = 0.005"
"\n"
"Model = 'model.bin'"
"\n"
"\n"
"Activation = sigmoid"
"\n"
- "weight_decay=l2norm"
+ "weight_regularizer=l2norm"
"\n"
- "weight_decay_lambda=0.005"
+ "weight_regularizer_constant=0.005"
"\n"
"filters=6"
"\n"
status = ml_train_layer_set_property(
layers[1], "unit= 10", "activation=softmax", "bias_initializer=zeros",
- "weight_decay=l2norm", "weight_decay_lambda=0.005",
+ "weight_regularizer=l2norm", "weight_regularizer_constant=0.005",
"weight_initializer=xavier_uniform", "name=fc100", NULL);
EXPECT_EQ(status, ML_ERROR_NONE);
status = ml_train_layer_set_property(
layers[1], "unit= 10", "activation=softmax", "bias_initializer=zeros",
- "weight_decay=l2norm", "weight_decay_lambda=0.005", NULL);
+ "weight_regularizer=l2norm", "weight_regularizer_constant=0.005", NULL);
EXPECT_EQ(status, ML_ERROR_NONE);
status = ml_train_model_add_layer(model, layers[1]);
status = ml_train_layer_set_property(
layer, "unit= 10", "activation=softmax", "bias_initializer=zeros",
- "weight_decay=l2norm", "weight_decay_lambda=0.005", NULL);
+ "weight_regularizer=l2norm", "weight_regularizer_constant=0.005", NULL);
EXPECT_EQ(status, ML_ERROR_NONE);
status = ml_train_model_add_layer(model, layer);
status = ml_train_layer_set_property(
layers[1], "unit= 10", "activation=softmax", "bias_initializer=zeros",
- "weight_decay=l2norm", "weight_decay_lambda=0.005", NULL);
+ "weight_regularizer=l2norm", "weight_regularizer_constant=0.005", NULL);
EXPECT_EQ(status, ML_ERROR_NONE);
status = ml_train_model_add_layer(model, layers[1]);
status = ml_train_layer_set_property(
layers[1], "unit= 10", "activation=softmax", "bias_initializer=zeros",
- "weight_decay=l2norm", "weight_decay_lambda=0.005", NULL);
+ "weight_regularizer=l2norm", "weight_regularizer_constant=0.005", NULL);
EXPECT_EQ(status, ML_ERROR_NONE);
status = ml_train_model_add_layer(model, layers[1]);
status = ml_train_layer_set_property(
layers[1], "unit= 10", "activation=softmax", "bias_initializer=zeros",
- "weight_decay=l2norm", "weight_decay_lambda=0.005",
+ "weight_regularizer=l2norm", "weight_regularizer_constant=0.005",
"weight_initializer=xavier_uniform", NULL);
EXPECT_EQ(status, ML_ERROR_NONE);
status = ml_train_layer_set_property(
layers[1], "unit= 10", "activation=softmax", "bias_initializer=zeros",
- "weight_decay=l2norm", "weight_decay_lambda=0.005",
+ "weight_regularizer=l2norm", "weight_regularizer_constant=0.005",
"weight_initializer=xavier_uniform", NULL);
EXPECT_EQ(status, ML_ERROR_NONE);
int status;
status = ml_train_layer_create(&handle, ML_TRAIN_LAYER_TYPE_FC);
EXPECT_EQ(status, ML_ERROR_NONE);
- status = ml_train_layer_set_property(handle, "weight_decay=l2norm",
- "weight_decay_lambda=0.0001", NULL);
+ status =
+ ml_train_layer_set_property(handle, "weight_regularizer=l2norm",
+ "weight_regularizer_constant=0.0001", NULL);
EXPECT_EQ(status, ML_ERROR_NONE);
status = ml_train_layer_destroy(handle);
EXPECT_EQ(status, ML_ERROR_NONE);
int status;
status = ml_train_layer_create(&handle, ML_TRAIN_LAYER_TYPE_FC);
EXPECT_EQ(status, ML_ERROR_NONE);
- status = ml_train_layer_set_property(handle, "weight_decay=asdfasd",
- "weight_decay_lambda=0.0001", NULL);
+ status =
+ ml_train_layer_set_property(handle, "weight_regularizer=asdfasd",
+ "weight_regularizer_constant=0.0001", NULL);
EXPECT_EQ(status, ML_ERROR_INVALID_PARAMETER);
status = ml_train_layer_destroy(handle);
EXPECT_EQ(status, ML_ERROR_NONE);
setProperty("input_shape=3:28:28 | batch_size=32 |"
"bias_initializer=zeros |"
"activation=sigmoid |"
- "weight_decay=l2norm |"
- "weight_decay_lambda= 0.005 |"
+ "weight_regularizer=l2norm |"
+ "weight_regularizer_constant= 0.005 |"
"weight_initializer=xavier_uniform |"
"normalization=true |"
"filters=12 | kernel_size= 5,5 | stride=3,3 | padding=1,1");
projects / platform / core / ml / nntrainer.git / commitdiff