From 97a308d5ca5ac132d58dc0fdb2edf5237afe0141 Mon Sep 17 00:00:00 2001 From: Parichay Kapoor Date: Fri, 19 Mar 2021 20:38:16 +0900 Subject: [PATCH] [optimizer] Add optimizer section to INI Add a new optimizer section to INI This is essential to allow custom optimizers as the properties of optimizers needs to be iterables. Currently, optimizer needs to specified in the model, which along being not the best design, limits the properties to be set for the optimizer as the model loader has to know which properties are for optimizer and which for model. This patch separates the optimizer as a new section and updates the unittests. Note that this removes the default optimizer from adam. Now, it is necessary for an optimizer to be defined in the model file, if to be used for training. For inference, model can be defined without an optimizer. Updated documentation for the changes made. See also #986 **Self evaluation:** 1. Build test: [x]Passed [ ]Failed [ ]Skipped 2. Run test: [x]Passed [ ]Failed [ ]Skipped Signed-off-by: Parichay Kapoor --- .../Custom/LayerClient/res/custom_layer_client.ini | 6 +- .../LogisticRegression/res/LogisticRegression.ini | 9 +- Applications/MNIST/res/mnist.ini | 7 +- Applications/MNIST/res/mnist_valid.ini | 7 +- .../ReinforcementLearning/DeepQ/jni/DeepQ.ini | 8 +- Applications/ResNet/res/resnet18.ini | 6 +- .../CIFAR_Classification/res/Classification.ini | 11 +- .../res/Classification_func.ini | 11 +- .../res/Classification_new.ini | 15 +- .../Draw_Classification/res/Training.ini | 8 +- Applications/VGG/res/vgg.ini | 8 +- Applications/VGG/res/vgg_small.ini | 7 +- docs/configuration-ini.md | 64 ++++--- nntrainer/models/model_loader.cpp | 120 ++++++------ nntrainer/models/model_loader.h | 10 + test/ccapi/unittest_ccapi.cpp | 23 ++- test/test_models/models/mnist.ini | 6 +- test/tizen_capi/unittest_tizen_capi.cpp | 41 ++--- test/unittest/unittest_nntrainer_graph.cpp | 13 +- test/unittest/unittest_nntrainer_modelfile.cpp | 204 +++++++++++---------- test/unittest/unittest_nntrainer_models.cpp | 43 +++-- 21 files changed, 333 insertions(+), 294 deletions(-) diff --git a/Applications/Custom/LayerClient/res/custom_layer_client.ini b/Applications/Custom/LayerClient/res/custom_layer_client.ini index 3832b26..a8776cd 100644 --- a/Applications/Custom/LayerClient/res/custom_layer_client.ini +++ b/Applications/Custom/LayerClient/res/custom_layer_client.ini @@ -2,12 +2,14 @@ # Network Section : Network [Model] Type = NeuralNetwork -Learning_rate = 0.001 Epochs = 100 -Optimizer = sgd Loss = cross batch_size = 10 +[Optimizer] +Type = sgd +Learning_rate = 0.001 + # Layer Section : Name [inputlayer] Type = Input diff --git a/Applications/LogisticRegression/res/LogisticRegression.ini b/Applications/LogisticRegression/res/LogisticRegression.ini index c25cdb5..58611ea 100644 --- a/Applications/LogisticRegression/res/LogisticRegression.ini +++ b/Applications/LogisticRegression/res/LogisticRegression.ini @@ -1,15 +1,14 @@ # Model Section : Model [Model] Type = Regression # Network Type : Regression, KNN, NeuralNetwork -Learning_rate = 0.001 # Learning Rate Epochs = 500 # Epochs -Optimizer = sgd # Optimizer : sgd (stochastic gradien decent), - # adam (Adamtive Moment Estimation) Loss = cross # Loss function : mse (mean squared error) - # cross ( cross entropy ) + # cross ( cross entropy ) Save_Path = "logistic_model.bin" # model path to save / read batch_size = 16 # batch size -epsilon = 1e-5 + +[Optimizer] +Type = sgd # Optimizer : sgd (stochastic gradien decent) # Layer Section : Name [inputlayer] diff --git a/Applications/MNIST/res/mnist.ini b/Applications/MNIST/res/mnist.ini index 828ebf8..0a0b229 100644 --- a/Applications/MNIST/res/mnist.ini +++ b/Applications/MNIST/res/mnist.ini @@ -1,14 +1,15 @@ # Network Section : Network [Model] Type = NeuralNetwork # Network Type : Regression, KNN, NeuralNetwork -Learning_rate = 1e-4 # Learning Rate Epochs = 1500 # Epochs -Optimizer = adam # Optimizer : sgd (stochastic gradien decent), - # adam (Adamtive Moment Estimation) Loss = cross # Loss function : mse (mean squared error) # cross ( for cross entropy ) # Save_Path = "mnist_model.bin" # model path to save / read batch_size = 32 # batch size + +[Optimizer] +Type = adam +Learning_rate = 1e-4 # Learning Rate beta1 = 0.9 # beta 1 for adam beta2 = 0.999 # beta 2 for adam epsilon = 1e-7 # epsilon for adam diff --git a/Applications/MNIST/res/mnist_valid.ini b/Applications/MNIST/res/mnist_valid.ini index 2adef51..6008a19 100644 --- a/Applications/MNIST/res/mnist_valid.ini +++ b/Applications/MNIST/res/mnist_valid.ini @@ -1,14 +1,15 @@ # Network Section : Network [Model] Type = NeuralNetwork # Network Type : Regression, KNN, NeuralNetwork -Learning_rate = 1e-4 # Learning Rate Epochs = 100 # Epochs -Optimizer = adam # Optimizer : sgd (stochastic gradien decent), - # adam (Adamtive Moment Estimation) Loss = cross # Loss function : mse (mean squared error) # cross ( for cross entropy ) Save_Path = "model.bin" # model path to save / read batch_size = 32 # batch size + +[Optimizer] +Type = adam +Learning_rate = 1e-4 # Learning Rate beta1 = 0.9 # beta 1 for adam beta2 = 0.999 # beta 2 for adam epsilon = 1e-7 # epsilon for adam diff --git a/Applications/ReinforcementLearning/DeepQ/jni/DeepQ.ini b/Applications/ReinforcementLearning/DeepQ/jni/DeepQ.ini index b6643f8..798fd24 100644 --- a/Applications/ReinforcementLearning/DeepQ/jni/DeepQ.ini +++ b/Applications/ReinforcementLearning/DeepQ/jni/DeepQ.ini @@ -1,15 +1,15 @@ # Network Section : Network [Model] Type = NeuralNetwork # Network Type : Regression, KNN, NeuralNetwork -Learning_rate = 0.001 # Learning Rate Epochs = 10000 # Epochs -Optimizer = adam # Optimizer : sgd (stochastic gradien decent), - # adam (Adamtive Moment Estimation) - Loss = mse # Loss function : mse (mean squared error) # cross (cross entropy) Save_Path = "model_deepq.bin" # model path to save / read batch_size = 32 # batch size + +[Optimizer] +Type = adam +Learning_rate = 0.001 # Learning Rate beta1 = 0.9 # beta 1 for adam beta2 = 0.9999 # beta 2 for adam epsilon = 1e-8 # epsilon for adam diff --git a/Applications/ResNet/res/resnet18.ini b/Applications/ResNet/res/resnet18.ini index 77d6f45..6b65ec6 100644 --- a/Applications/ResNet/res/resnet18.ini +++ b/Applications/ResNet/res/resnet18.ini @@ -8,12 +8,14 @@ [Model] Type = NeuralNetwork -Learning_rate = 1e-4 Epochs = 2 -Optimizer = adam Loss = cross Save_Path = "model.bin" batch_size = 128 + +[Optimizer] +Type = adam +Learning_rate = 1e-4 beta1 = 0.9 beta2 = 0.999 epsilon = 1e-7 diff --git a/Applications/TransferLearning/CIFAR_Classification/res/Classification.ini b/Applications/TransferLearning/CIFAR_Classification/res/Classification.ini index 94afc31..8417d03 100644 --- a/Applications/TransferLearning/CIFAR_Classification/res/Classification.ini +++ b/Applications/TransferLearning/CIFAR_Classification/res/Classification.ini @@ -1,16 +1,17 @@ #Network Section : Network [Model] Type = NeuralNetwork # Network Type : Regression, KNN, NeuralNetwork -Learning_rate = 0.0001 # Learning Rate -Decay_rate = 0.96 # for the decay_rate for the decayed learning rate -Decay_steps = 1000 # decay step for the exponential decayed learning rate Epochs = 30000 # Epochs -Optimizer = adam # Optimizer : sgd (stochastic gradien decent), - # adam(Adamtive Moment Estimation) Loss = cross # Loss function : mse (mean squared error) # cross(cross entropy) Save_Path = "model_cls.bin" # model path to save / read batch_size = 32 # batch size + +[Optimizer] +Type = adam +Learning_rate = 0.0001 # Learning Rate +Decay_rate = 0.96 # for the decay_rate for the decayed learning rate +Decay_steps = 1000 # decay step for the exponential decayed learning rate beta1 = 0.9 # beta 1 for adam beta2 = 0.9999 # beta 2 for adam epsilon = 1e-7 # epsilon for adam diff --git a/Applications/TransferLearning/CIFAR_Classification/res/Classification_func.ini b/Applications/TransferLearning/CIFAR_Classification/res/Classification_func.ini index 4d93ffe..d94464e 100644 --- a/Applications/TransferLearning/CIFAR_Classification/res/Classification_func.ini +++ b/Applications/TransferLearning/CIFAR_Classification/res/Classification_func.ini @@ -1,19 +1,20 @@ # Network Section : Network [Model] Type = NeuralNetwork # Network Type : Regression, KNN, NeuralNetwork -Learning_rate = 0.0001 # Learning Rate -Decay_rate = 0.96 # for the decay_rate for the decayed learning rate -Decay_steps = 1000 # decay step for the exponential decayed learning rate Epochs = 30000 # Epochs -Optimizer = adam # Optimizer : sgd (stochastic gradien decent), - # adam (Adamtive Moment Estimation) Loss = cross # Loss function : mse (mean squared error) # cross (cross entropy) Save_Path = "model_cls_func.bin" # model path to save / read batch_size = 32 # batch size + +[Optimizer] +Type = adam beta1 = 0.9 # beta 1 for adam beta2 = 0.9999 # beta 2 for adam epsilon = 1e-7 # epsilon for adam +Learning_rate = 0.0001 # Learning Rate +Decay_rate = 0.96 # for the decay_rate for the decayed learning rate +Decay_steps = 1000 # decay step for the exponential decayed learning rate # Layer Section : Name [mobilenetv2] diff --git a/Applications/TransferLearning/CIFAR_Classification/res/Classification_new.ini b/Applications/TransferLearning/CIFAR_Classification/res/Classification_new.ini index 8b4eda3..e29b33b 100644 --- a/Applications/TransferLearning/CIFAR_Classification/res/Classification_new.ini +++ b/Applications/TransferLearning/CIFAR_Classification/res/Classification_new.ini @@ -1,20 +1,17 @@ # Network Section : Network [Model] Type = NeuralNetwork # Network Type : Regression, KNN, NeuralNetwork -Learning_rate = 0.001 # Learning Rate -Decay_rate = 0.96 # for the decay_rate for the decayed learning rate -Decay_steps = 1000 # decay step for the exponential decayed learning rate Epochs = 30000 # Epochs -Optimizer = sgd # Optimizer : sgd (stochastic gradien decent), - # adam (Adamtive Moment Estimation) -Activation = sigmoid # activation : sigmoid, tanh Loss = cross # Loss function : mse (mean squared error) # cross (cross entropy) Save_Path = "model_cls_new.bin" # model path to save / read batch_size = 32 # batch size -beta1 = 0.9 # beta 1 for adam -beta2 = 0.9999 # beta 2 for adam -epsilon = 1e-8 # epsilon for adam + +[Optimizer] +Type = sgd +Learning_rate = 0.001 # Learning Rate +Decay_rate = 0.96 # for the decay_rate for the decayed learning rate +Decay_steps = 1000 # decay step for the exponential decayed learning rate # Layer Section : Name [inputlayer] diff --git a/Applications/TransferLearning/Draw_Classification/res/Training.ini b/Applications/TransferLearning/Draw_Classification/res/Training.ini index 870467a..7364017 100644 --- a/Applications/TransferLearning/Draw_Classification/res/Training.ini +++ b/Applications/TransferLearning/Draw_Classification/res/Training.ini @@ -1,16 +1,18 @@ # Network Section : Network [Model] Type = NeuralNetwork # Network Type : Regression, KNN, NeuralNetwork -Learning_rate = 0.01 # Learning Rate # Epochs = 1000 # Epochs Epochs = 10 # Epochs -Optimizer = sgd # Optimizer : sgd (stochastic gradien decent), - # adam (Adamtive Moment Estimation) Loss = cross # Loss function : mse (mean squared error) # cross (cross entropy) Save_Path = "model_draw_cls.bin" # model path to save / read batch_size = 1 # batch size +[Optimizer] +Type = sgd # Optimizer : sgd (stochastic gradien decent), + # adam (Adamtive Moment Estimation) +Learning_rate = 0.01 # Learning Rate + [MobilenetV2Backbone] backbone = ssd_mobilenet_v2_coco_feature.tflite Input_Shape = 300:300:3 # Input Dimension with NHWC for tflite diff --git a/Applications/VGG/res/vgg.ini b/Applications/VGG/res/vgg.ini index 6966e6c..4cd0a46 100644 --- a/Applications/VGG/res/vgg.ini +++ b/Applications/VGG/res/vgg.ini @@ -8,14 +8,16 @@ [Model] Type = NeuralNetwork # Network Type : Regression, KNN, NeuralNetwork -Learning_rate = 1e-4 # Learning Rate Epochs = 1 # Epochs -Optimizer = adam # Optimizer : sgd (stochastic gradien decent), - # adam (Adamtive Moment Estimation) Loss = cross # Loss function : mse (mean squared error) # cross ( for cross entropy ) Save_Path = "model.bin" # model path to save / read batch_size = 128 # batch size + +[Optimizer] +Type = adam # Optimizer : sgd (stochastic gradien decent), + # adam (Adamtive Moment Estimation) +Learning_rate = 1e-4 # Learning Rate beta1 = 0.9 # beta 1 for adam beta2 = 0.999 # beta 2 for adam epsilon = 1e-7 # epsilon for adam diff --git a/Applications/VGG/res/vgg_small.ini b/Applications/VGG/res/vgg_small.ini index 31b968a..79c7745 100644 --- a/Applications/VGG/res/vgg_small.ini +++ b/Applications/VGG/res/vgg_small.ini @@ -8,14 +8,15 @@ [Model] Type = NeuralNetwork # Network Type : Regression, KNN, NeuralNetwork -Learning_rate = 1e-2 # Learning Rate Epochs = 3000 # Epochs -Optimizer = adam # Optimizer : sgd (stochastic gradien decent), - # adam (Adamtive Moment Estimation) Loss = cross # Loss function : mse (mean squared error) # cross ( for cross entropy ) Save_Path = "model.bin" # model path to save / read batch_size = 128 # batch size + +[Optimizer] +Type = adam +Learning_rate = 1e-2 # Learning Rate beta1 = 0.9 # beta 1 for adam beta2 = 0.999 # beta 2 for adam epsilon = 1e-7 # epsilon for adam diff --git a/docs/configuration-ini.md b/docs/configuration-ini.md index a088ac1..f67b7cb 100644 --- a/docs/configuration-ini.md +++ b/docs/configuration-ini.md @@ -26,21 +26,13 @@ Start with "[Model]" * knn : K-nearest neighbor * neuralnetwork : Deep Neural Network -2. ```learning_rate = ``` - - Initial learning rate to decay - -3. ```epochs = ``` +2. ```epochs = ``` Number of epochs to train -4. ```optimizer = ``` - - Optimizer to apply the gradients to weights. - * adam : Adaptive Moment Estimation - * sgd : stochastic gradient decent + Create a new section for this -5. ```loss = ``` +3. ```loss = ``` Loss function * mse : mean squared error @@ -48,37 +40,61 @@ Start with "[Model]" Only allowed with sigmoid and softmax activation function * none : no loss for the model (this model will only support inference) -6. ```save_path = ``` +4. ```save_path = ``` Model file path to save updated weights -7. ```batch_size = ``` +5. ```batch_size = ``` Mini batch size -8. ```beta1 = ``` +Below is sample Network section. + +```ini +# Network Section : Network +[Model] +Type = NeuralNetwork +Epochs = 1500 +Loss = cross +Save_Path = "model.bin" +batch_size = 32 +``` + +### Optimizer Section + +Define the optimizer to be used for training. This is an optional section needed only for training, and can be skipped for inference. + +Start with "[ Optimizer ]" + +1. ```type = ``` + + Optimizer type to apply the gradients to weights. + * adam : Adaptive Moment Estimation + * sgd : stochastic gradient decent + +2. ```learning_rate = ``` + + Initial learning rate to decay + +3. ```beta1 = ``` beta1 parameter for adam optimizer. Only valid for adam. 0.9 is default. -9. ```beta2 = ``` +4. ```beta2 = ``` beta2 parameter for adam optimizer. Only valid for adam. 0.999 is default. -10. ```epsilon = ``` +5. ```epsilon = ``` Epsilon parameter for adam optimizer. Only valid for adam. 1.0e-7 is default. -Below is sample Network section. +Below is a sample Optimizer section. ```ini -# Network Section : Network -[Model] -Type = NeuralNetwork +# Optimizer Section +[Optimizer] +Type = adam Learning_rate = 1e-4 -Epochs = 1500 -Optimizer = adam -Loss = cross -Save_Path = "model.bin" batch_size = 32 beta1 = 0.9 beta2 = 0.999 diff --git a/nntrainer/models/model_loader.cpp b/nntrainer/models/model_loader.cpp index d76f4e2..812eb11 100644 --- a/nntrainer/models/model_loader.cpp +++ b/nntrainer/models/model_loader.cpp @@ -45,6 +45,34 @@ namespace nntrainer { +int ModelLoader::loadOptimizerConfigIni(dictionary *ini, NeuralNetwork &model) { + int status = ML_ERROR_NONE; + + if (iniparser_find_entry(ini, "Optimizer") == 0) { + ml_logw("there is no [Optimizer] section in given ini file." + "This model can only be used for inference."); + return ML_ERROR_NONE; + } + + /** Default to adam optimizer */ + const char *opt_type = iniparser_getstring(ini, "Optimizer:Type", "adam"); + std::vector properties = parseProperties(ini, "Optimizer"); + + try { + std::shared_ptr optimizer = + app_context.createObject(opt_type, properties); + model.setOptimizer(optimizer); + } catch (std::exception &e) { + ml_loge("%s %s", typeid(e).name(), e.what()); + return ML_ERROR_INVALID_PARAMETER; + } catch (...) { + ml_loge("Creating the optimizer failed"); + return ML_ERROR_INVALID_PARAMETER; + } + + return status; +} + /** * @brief load model config from ini */ @@ -70,61 +98,6 @@ int ModelLoader::loadModelConfigIni(dictionary *ini, NeuralNetwork &model) { model.batch_size = iniparser_getint(ini, "Model:Batch_Size", model.batch_size); - /** Default to adam optimizer */ - const char *opt_type = iniparser_getstring(ini, "Model:Optimizer", "adam"); - - try { - model.opt = nntrainer::createOptimizer(opt_type); - } catch (std::exception &e) { - ml_loge("%s %s", typeid(e).name(), e.what()); - return ML_ERROR_INVALID_PARAMETER; - } catch (...) { - ml_loge("Creating the optimizer failed"); - return ML_ERROR_INVALID_PARAMETER; - } - - std::vector optimizer_prop = {}; - optimizer_prop.push_back( - {"learning_rate=" + - std::string(iniparser_getstring( - ini, "Model:Learning_rate", - std::to_string(model.opt->getLearningRate()).c_str()))}); - - // TODO: create a optimizer section in the INI - if (model.opt->getType() == SGD::type || model.opt->getType() == Adam::type) { - std::shared_ptr opt_impl = - std::static_pointer_cast(model.opt); - - optimizer_prop.push_back( - {"decay_steps=" + std::string(iniparser_getstring( - ini, "Model:Decay_steps", - std::to_string(opt_impl->getDecaySteps()).c_str()))}); - optimizer_prop.push_back( - {"decay_rate=" + std::string(iniparser_getstring( - ini, "Model:Decay_rate", - std::to_string(opt_impl->getDecayRate()).c_str()))}); - - if (opt_impl->getType() == "adam") { - std::shared_ptr opt_adam = std::static_pointer_cast(opt_impl); - - optimizer_prop.push_back( - {"beta1=" + - std::string(iniparser_getstring( - ini, "Model:Beta1", std::to_string(opt_adam->getBeta1()).c_str()))}); - optimizer_prop.push_back( - {"beta2=" + - std::string(iniparser_getstring( - ini, "Model:Beta2", std::to_string(opt_adam->getBeta2()).c_str()))}); - optimizer_prop.push_back( - {"epsilon=" + std::string(iniparser_getstring( - ini, "Model:Epsilon", - std::to_string(opt_adam->getEpsilon()).c_str()))}); - } - } - - status = model.opt->setProperty(optimizer_prop); - NN_RETURN_STATUS(); - return status; } @@ -181,27 +154,24 @@ int ModelLoader::loadDatasetConfigIni(dictionary *ini, NeuralNetwork &model) { return status; } -int ModelLoader::loadLayerConfigIniCommon(dictionary *ini, - std::shared_ptr &layer, - const std::string &layer_name, - const std::string &layer_type) { - int status = ML_ERROR_NONE; - int num_entries = iniparser_getsecnkeys(ini, layer_name.c_str()); +std::vector +ModelLoader::parseProperties(dictionary *ini, const std::string §ion_name) { + int num_entries = iniparser_getsecnkeys(ini, section_name.c_str()); - ml_logd("number of entries for %s: %d", layer_name.c_str(), num_entries); + ml_logd("number of entries for %s: %d", section_name.c_str(), num_entries); if (num_entries < 1) { std::stringstream ss; - ss << "there are no entries in the layer section: " << layer_name; + ss << "there are no entries in the section: " << section_name; throw std::invalid_argument(ss.str()); } std::unique_ptr key_refs(new const char *[num_entries]); - if (iniparser_getseckeys(ini, layer_name.c_str(), key_refs.get()) == + if (iniparser_getseckeys(ini, section_name.c_str(), key_refs.get()) == nullptr) { std::stringstream ss; - ss << "failed to fetch key for layer: " << layer_name; + ss << "failed to fetch key for section: " << section_name; throw std::invalid_argument(ss.str()); } @@ -209,7 +179,7 @@ int ModelLoader::loadLayerConfigIniCommon(dictionary *ini, properties.reserve(num_entries - 1); for (int i = 0; i < num_entries; ++i) { - /// key is ini section key, which is layer_name + ":" + prop_key + /// key is ini section key, which is section_name + ":" + prop_key std::string key(key_refs[i]); std::string prop_key = key.substr(key.find(":") + 1); @@ -235,6 +205,16 @@ int ModelLoader::loadLayerConfigIniCommon(dictionary *ini, properties.push_back(prop_key + "=" + value); } + return properties; +} + +int ModelLoader::loadLayerConfigIniCommon(dictionary *ini, + std::shared_ptr &layer, + const std::string &layer_name, + const std::string &layer_type) { + int status = ML_ERROR_NONE; + std::vector properties = parseProperties(ini, layer_name); + try { std::shared_ptr layer_ = app_context.createObject(layer_type, properties); @@ -371,6 +351,8 @@ int ModelLoader::loadFromIni(std::string ini_file, NeuralNetwork &model, unsigned int model_len = strlen(model_str); const char dataset_str[] = "dataset"; unsigned int dataset_len = strlen(dataset_str); + const char optimizer_str[] = "optimizer"; + unsigned int optimizer_len = strlen(optimizer_str); if (ini_file.empty()) { ml_loge("Error: Configuration File is not defined"); @@ -404,6 +386,9 @@ int ModelLoader::loadFromIni(std::string ini_file, NeuralNetwork &model, status = loadDatasetConfigIni(ini, model); NN_INI_RETURN_STATUS(); + + status = loadOptimizerConfigIni(ini, model); + NN_INI_RETURN_STATUS(); } ml_logd("parsing ini started"); @@ -427,6 +412,9 @@ int ModelLoader::loadFromIni(std::string ini_file, NeuralNetwork &model, if (strncasecmp(dataset_str, sec_name.c_str(), dataset_len) == 0) continue; + if (strncasecmp(optimizer_str, sec_name.c_str(), optimizer_len) == 0) + continue; + /** Parse all the layers defined as sections in order */ std::shared_ptr layer; diff --git a/nntrainer/models/model_loader.h b/nntrainer/models/model_loader.h index d82e58a..75506e1 100644 --- a/nntrainer/models/model_loader.h +++ b/nntrainer/models/model_loader.h @@ -80,6 +80,13 @@ private: int loadModelConfigIni(dictionary *ini, NeuralNetwork &model); /** + * @brief load optimizer config from ini + * @param[in] ini dictionary containing the config + * @param[in/out] model model to be loaded + */ + int loadOptimizerConfigIni(dictionary *ini, NeuralNetwork &model); + + /** * @brief load layer config from ini given the layer type * @param[in] ini dictionary containing the config * @param[in/out] layer layer to be loaded @@ -163,6 +170,9 @@ private: return model_file_context->getWorkingPath(app_context_resolved_path); } + std::vector parseProperties(dictionary *ini, + const std::string §ion_name); + const char *unknown = "Unknown"; const char *none = "none"; diff --git a/test/ccapi/unittest_ccapi.cpp b/test/ccapi/unittest_ccapi.cpp index c3d358b..646b30a 100644 --- a/test/ccapi/unittest_ccapi.cpp +++ b/test/ccapi/unittest_ccapi.cpp @@ -157,19 +157,18 @@ TEST(nntrainer_ccapi, train_with_config_01_p) { std::unique_ptr model; static IniSection model_base("Model", "Type = NeuralNetwork" - " | Learning_rate = 0.0001" - " | Decay_rate = 0.96" - " | Decay_steps = 1000" " | Epochs = 1" - " | Optimizer = adam" " | Loss = cross" - " | Weight_Regularizer = l2norm" - " | weight_regularizer_constant = 0.005" " | Save_Path = 'model.bin'" - " | batch_size = 32" - " | beta1 = 0.9" - " | beta2 = 0.9999" - " | epsilon = 1e-7"); + " | batch_size = 32"); + + static IniSection optimizer("Optimizer", "Type = adam" + " | Learning_rate = 0.0001" + " | Decay_rate = 0.96" + " | Decay_steps = 1000" + " | beta1 = 0.9" + " | beta2 = 0.9999" + " | epsilon = 1e-7"); static IniSection dataset("Dataset", "BufferSize=100" " | TrainData = trainingSet.dat" @@ -189,8 +188,8 @@ TEST(nntrainer_ccapi, train_with_config_01_p) { "| Activation = softmax"); ScopedIni s("test_train_01_p", - {model_base + "batch_size = 16", dataset + "-BufferSize", - inputlayer, outputlayer}); + {model_base + "batch_size = 16", optimizer, + dataset + "-BufferSize", inputlayer, outputlayer}); EXPECT_NO_THROW(model = ml::train::createModel(ml::train::ModelType::NEURAL_NET)); diff --git a/test/test_models/models/mnist.ini b/test/test_models/models/mnist.ini index 6de77b6..449f8bf 100644 --- a/test/test_models/models/mnist.ini +++ b/test/test_models/models/mnist.ini @@ -1,13 +1,15 @@ # Network Section : Network [Model] Type = NeuralNetwork # Network Type : Regression, KNN, NeuralNetwork -Learning_rate = 1e-4 # Learning Rate Epochs = 1500 # Epochs -Optimizer = adam # adam (Adamtive Moment Estimation) Loss = cross # Loss function : mse (mean squared error) # cross ( for cross entropy ) Save_Path = "model.bin" # model path to save / read batch_size = 32 # batch size + +[Optimizer] +Type = adam # adam (Adamtive Moment Estimation) +Learning_rate = 1e-4 # Learning Rate beta1 = 0.9 # beta 1 for adam beta2 = 0.999 # beta 2 for adam epsilon = 1e-7 # epsilon for adam diff --git a/test/tizen_capi/unittest_tizen_capi.cpp b/test/tizen_capi/unittest_tizen_capi.cpp index bdd65d0..253de93 100644 --- a/test/tizen_capi/unittest_tizen_capi.cpp +++ b/test/tizen_capi/unittest_tizen_capi.cpp @@ -30,19 +30,18 @@ static const std::string getTestResPath(const std::string &file) { } static IniSection model_base("Model", "Type = NeuralNetwork" - " | Learning_rate = 0.0001" - " | Decay_rate = 0.96" - " | Decay_steps = 1000" " | Epochs = 1" - " | Optimizer = adam" " | Loss = cross" - " | Weight_Regularizer = l2norm" - " | weight_regularizer_constant = 0.005" " | Save_Path = 'model.bin'" - " | batch_size = 32" - " | beta1 = 0.9" - " | beta2 = 0.9999" - " | epsilon = 1e-7"); + " | batch_size = 32"); + +static IniSection optimizer("Optimizer", "Type = adam" + " | Learning_rate = 0.0001" + " | Decay_rate = 0.96" + " | Decay_steps = 1000" + " | beta1 = 0.9" + " | beta2 = 0.9999" + " | epsilon = 1e-7"); static IniSection dataset("Dataset", "BufferSize=100" " | TrainData = trainingSet.dat" @@ -136,7 +135,7 @@ TEST(nntrainer_capi_nnmodel, compile_01_p) { int status = ML_ERROR_NONE; ScopedIni s("test_compile_01_p", - {model_base, dataset, inputlayer, outputlayer}); + {model_base, optimizer, dataset, inputlayer, outputlayer}); status = ml_train_model_construct_with_conf(s.getIniName().c_str(), &handle); EXPECT_EQ(status, ML_ERROR_NONE); @@ -164,9 +163,9 @@ TEST(nntrainer_capi_nnmodel, construct_conf_02_n) { ml_train_model_h handle = NULL; int status = ML_ERROR_NONE; - ScopedIni s( - "test_compile_03_n", - {model_base, dataset, inputlayer + "Input_Shape=1:1:0", outputlayer}); + ScopedIni s("test_compile_03_n", + {model_base, optimizer, dataset, inputlayer + "Input_Shape=1:1:0", + outputlayer}); status = ml_train_model_construct_with_conf(s.getIniName().c_str(), &handle); EXPECT_EQ(status, ML_ERROR_INVALID_PARAMETER); @@ -335,8 +334,8 @@ TEST(nntrainer_capi_nnmodel, train_01_p) { int status = ML_ERROR_NONE; ScopedIni s("test_train_01_p", - {model_base + "batch_size = 16", dataset + "-BufferSize", - inputlayer, outputlayer}); + {model_base + "batch_size = 16", optimizer, + dataset + "-BufferSize", inputlayer, outputlayer}); status = ml_train_model_construct_with_conf(s.getIniName().c_str(), &handle); EXPECT_EQ(status, ML_ERROR_NONE); @@ -370,8 +369,8 @@ TEST(nntrainer_capi_nnmodel, train_03_n) { ml_train_model_h handle = NULL; int status = ML_ERROR_NONE; ScopedIni s("test_train_01_p", - {model_base + "batch_size = 16", dataset + "-BufferSize", - inputlayer, outputlayer}); + {model_base + "batch_size = 16", optimizer, + dataset + "-BufferSize", inputlayer, outputlayer}); status = ml_train_model_construct_with_conf(s.getIniName().c_str(), &handle); EXPECT_EQ(status, ML_ERROR_NONE); @@ -520,7 +519,7 @@ TEST(nntrainer_capi_nnmodel, addLayer_05_n) { ml_train_layer_h layer = NULL; ScopedIni s("test_compile_01_p", - {model_base, dataset, inputlayer, outputlayer}); + {model_base, optimizer, dataset, inputlayer, outputlayer}); status = ml_train_model_construct_with_conf(s.getIniName().c_str(), &model); EXPECT_EQ(status, ML_ERROR_NONE); @@ -930,7 +929,7 @@ TEST(nntrainer_capi_summary, summary_01_p) { int status = ML_ERROR_NONE; ScopedIni s("test_compile_01_p", - {model_base, dataset, inputlayer, outputlayer}); + {model_base, optimizer, dataset, inputlayer, outputlayer}); status = ml_train_model_construct_with_conf(s.getIniName().c_str(), &handle); EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_model_compile(handle, NULL); @@ -956,7 +955,7 @@ TEST(nntrainer_capi_summary, summary_02_n) { int status = ML_ERROR_NONE; ScopedIni s("test_compile_01_p", - {model_base, dataset, inputlayer, outputlayer}); + {model_base, optimizer, dataset, inputlayer, outputlayer}); status = ml_train_model_construct_with_conf(s.getIniName().c_str(), &handle); EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_model_compile(handle, NULL); diff --git a/test/unittest/unittest_nntrainer_graph.cpp b/test/unittest/unittest_nntrainer_graph.cpp index 024ab44..19d533e 100644 --- a/test/unittest/unittest_nntrainer_graph.cpp +++ b/test/unittest/unittest_nntrainer_graph.cpp @@ -113,11 +113,10 @@ TEST_P(nntrainerGraphTest, loadConfig) { static IniSection nw_base("model", "Type = NeuralNetwork | " "batch_size = 16 | " - "epsilon = 1e-7 | " "loss = cross"); -static IniSection nw_sgd = nw_base + "Optimizer = sgd |" - "Learning_rate = 1"; +static IniSection sgd("Optimizer", "Type = sgd |" + "Learning_rate = 1"); static IniSection input("inputlayer", "Type = input |" "Input_Shape = 3:32:32"); @@ -236,10 +235,10 @@ mkIniTc(const char *name, const IniTestWrapper::Sections vec, int flag) { INSTANTIATE_TEST_CASE_P( nntrainerIniAutoTests, nntrainerGraphTest, ::testing::Values(mkIniTc("basic_p", - {nw_sgd, input, conv2d8, conv2d9, pooling2, out0, - conv2d10, conv2d11, addition0, out1, conv2d12, - conv2d13, addition1, conv2d14, pooling3, fclayer0, - fclayer1}, + {nw_base, sgd, input, conv2d8, conv2d9, pooling2, + out0, conv2d10, conv2d11, addition0, out1, + conv2d12, conv2d13, addition1, conv2d14, pooling3, + fclayer0, fclayer1}, SUCCESS))); int main(int argc, char **argv) { diff --git a/test/unittest/unittest_nntrainer_modelfile.cpp b/test/unittest/unittest_nntrainer_modelfile.cpp index 6366e54..633ee6f 100644 --- a/test/unittest/unittest_nntrainer_modelfile.cpp +++ b/test/unittest/unittest_nntrainer_modelfile.cpp @@ -161,25 +161,28 @@ TEST_P(nntrainerIniTest, initThreetime_n) { /// @todo add run test could be added with iniTest flag to control skip static IniSection nw_base("model", "Type = NeuralNetwork | " "batch_size = 32 | " - "epsilon = 1e-7 | " "loss = cross"); static IniSection nw_base_mse("model", "Type = NeuralNetwork | " "batch_size = 32 | " - "epsilon = 1e-7 | " "loss = mse"); -static IniSection adam("adam", "Optimizer = adam |" - "Learning_rate = 0.00001 |" - "Decay_rate = 0.96 |" - "Decay_steps = 1000"); +static IniSection adam("Optimizer", "Type = adam |" + "epsilon = 1e-7 | " + "Learning_rate = 0.00001 |" + "Decay_rate = 0.96 |" + "Decay_steps = 1000"); -static IniSection nw_sgd = nw_base + "Optimizer = sgd |" - "Learning_rate = 1"; +static IniSection sgd("Optimizer", "Type = sgd |" + "Learning_rate = 1"); -static IniSection nw_adam = nw_base + adam; +// static IniSection nw_sgd = nw_base + "Optimizer = sgd |" +// "Learning_rate = 1"; -static IniSection nw_adam_n = nw_base + "Learning_rate = -1"; +// static IniSection nw_adam = nw_base + adam; + +// static IniSection nw_adam_n = nw_base + "Learning_rate = -1"; +// static IniSection adam_n = adam + "Learning_rate = -1"; static IniSection dataset("DataSet", "BufferSize = 100 |" "TrainData = trainingSet.dat | " @@ -286,56 +289,56 @@ mkIniTc(const char *name, const IniTestWrapper::Sections vec, int flag) { INSTANTIATE_TEST_CASE_P( nntrainerIniAutoTests, nntrainerIniTest, ::testing::Values( /**< positive: basic valid scenarios (2 positive and 3 negative cases) */ - mkIniTc("basic_p", {nw_adam, input, out+"input_layers=inputlayer"}, SUCCESS), - mkIniTc("basic2_p", {nw_sgd, input, out+"input_layers=inputlayer"}, SUCCESS), - mkIniTc("basic_act_p", {nw_sgd, input + "-Activation", act_relu+"input_layers=inputlayer", out+"input_layers=activation_relu" }, SUCCESS), - mkIniTc("basic_bn_p", {nw_sgd, input + "-Activation", batch_normal+"input_layers=inputlayer", act_relu+"input_layers=bn", out+"input_layers=activation_relu" }, SUCCESS), - mkIniTc("basic_bn2_p", {nw_sgd, input + "-Activation", batch_normal + "Activation = relu"+"input_layers=inputlayer", out+"input_layers=bn" }, SUCCESS), - mkIniTc("basic_dataset_p", {nw_adam, dataset, input, out+"input_layers=inputlayer"}, SUCCESS), - mkIniTc("basic_dataset2_p", {nw_sgd, input, out+"input_layers=inputlayer", dataset}, SUCCESS), - mkIniTc("basic_dataset3_p", {dataset, nw_sgd, input, out+"input_layers=inputlayer"}, SUCCESS), - mkIniTc("basic_conv2d_p", {nw_adam, conv2d + "input_shape = 1:10:10"}, SUCCESS), - mkIniTc("no_testSet_p", {nw_adam, dataset + "-TestData", input, out+"input_layers=inputlayer"}, SUCCESS), - mkIniTc("no_validSet_p", {nw_adam, dataset + "-ValidData", input, out+"input_layers=inputlayer"}, SUCCESS), - mkIniTc("no_bufferSize_p", {nw_adam, dataset + "-BufferSize", input, out+"input_layers=inputlayer"}, SUCCESS), - mkIniTc("buffer_size_smaller_than_batch_size_p", {nw_adam, dataset + "BufferSize=26", input, out+"input_layers=inputlayer"}, SUCCESS), - mkIniTc("buffer_size_smaller_than_batch_size2_p", {nw_adam, input, out+"input_layers=inputlayer", dataset + "BufferSize=26"}, SUCCESS), + mkIniTc("basic_p", {nw_base, adam, input, out+"input_layers=inputlayer"}, SUCCESS), + mkIniTc("basic2_p", {nw_base, sgd, input, out+"input_layers=inputlayer"}, SUCCESS), + mkIniTc("basic_act_p", {nw_base, sgd, input + "-Activation", act_relu+"input_layers=inputlayer", out+"input_layers=activation_relu" }, SUCCESS), + mkIniTc("basic_bn_p", {nw_base, sgd, input + "-Activation", batch_normal+"input_layers=inputlayer", act_relu+"input_layers=bn", out+"input_layers=activation_relu" }, SUCCESS), + mkIniTc("basic_bn2_p", {nw_base, sgd, input + "-Activation", batch_normal + "Activation = relu"+"input_layers=inputlayer", out+"input_layers=bn" }, SUCCESS), + mkIniTc("basic_dataset_p", {nw_base, adam, dataset, input, out+"input_layers=inputlayer"}, SUCCESS), + mkIniTc("basic_dataset2_p", {nw_base, sgd, input, out+"input_layers=inputlayer", dataset}, SUCCESS), + mkIniTc("basic_dataset3_p", {dataset, nw_base, sgd, input, out+"input_layers=inputlayer"}, SUCCESS), + mkIniTc("basic_conv2d_p", {nw_base, adam, conv2d + "input_shape = 1:10:10"}, SUCCESS), + mkIniTc("no_testSet_p", {nw_base, adam, dataset + "-TestData", input, out+"input_layers=inputlayer"}, SUCCESS), + mkIniTc("no_validSet_p", {nw_base, adam, dataset + "-ValidData", input, out+"input_layers=inputlayer"}, SUCCESS), + mkIniTc("no_bufferSize_p", {nw_base, adam, dataset + "-BufferSize", input, out+"input_layers=inputlayer"}, SUCCESS), + mkIniTc("buffer_size_smaller_than_batch_size_p", {nw_base, adam, dataset + "BufferSize=26", input, out+"input_layers=inputlayer"}, SUCCESS), + mkIniTc("buffer_size_smaller_than_batch_size2_p", {nw_base, adam, input, out+"input_layers=inputlayer", dataset + "BufferSize=26"}, SUCCESS), /**< half negative: init fail cases (1 positive and 4 negative cases) */ - mkIniTc("unknown_loss_n", {nw_adam + "loss = unknown", input, out+"input_layers=inputlayer"}, COMPFAIL | INITFAIL), - mkIniTc("activation_very_first_n", {nw_sgd, act_relu, input+"input_layers=activation_relu", out+"input_layers=inputlayer"}, COMPFAIL | INITFAIL), - mkIniTc("bnlayer_very_first_n", {nw_sgd, batch_normal, input+"input_layers=bn", out+"input_layers=inputlayer"}, COMPFAIL | INITFAIL), - mkIniTc("act_layer_after_act_n", {nw_sgd, input, act_relu+"input_layers=inputlayer", out+"input_layers=activation_relu"}, INITFAIL), - mkIniTc("act_layer_after_act_bn_n", {nw_sgd, input, act_relu+"input_layers=inputlayer", batch_normal+"input_layers=activation_relu", out+"input_layers=bn" }, INITFAIL), - mkIniTc("last_act_layer_relu_n", {nw_sgd, input, out+"input_layers=inputlayer", act_relu+"input_layers=fclayer" }, COMPFAIL | INITFAIL), - mkIniTc("last_act_layer_relu2_n", {nw_sgd, input, out+"input_layers=inputlayer" + "-Activation", act_relu+"input_layers=fclayer" }, COMPFAIL | INITFAIL), + mkIniTc("unknown_loss_n", {nw_base + "loss = unknown", adam, input, out+"input_layers=inputlayer"}, COMPFAIL | INITFAIL), + mkIniTc("activation_very_first_n", {nw_base, sgd, act_relu, input+"input_layers=activation_relu", out+"input_layers=inputlayer"}, COMPFAIL | INITFAIL), + mkIniTc("bnlayer_very_first_n", {nw_base, sgd, batch_normal, input+"input_layers=bn", out+"input_layers=inputlayer"}, COMPFAIL | INITFAIL), + mkIniTc("act_layer_after_act_n", {nw_base, sgd, input, act_relu+"input_layers=inputlayer", out+"input_layers=activation_relu"}, INITFAIL), + mkIniTc("act_layer_after_act_bn_n", {nw_base, sgd, input, act_relu+"input_layers=inputlayer", batch_normal+"input_layers=activation_relu", out+"input_layers=bn" }, INITFAIL), + mkIniTc("last_act_layer_relu_n", {nw_base, sgd, input, out+"input_layers=inputlayer", act_relu+"input_layers=fclayer" }, COMPFAIL | INITFAIL), + mkIniTc("last_act_layer_relu2_n", {nw_base, sgd, input, out+"input_layers=inputlayer" + "-Activation", act_relu+"input_layers=fclayer" }, COMPFAIL | INITFAIL), /**< negative: basic invalid scenarios (5 negative cases) */ - mkIniTc("no_model_sec_name_n", {I(nw_adam, "-", "")}, ALLFAIL), + mkIniTc("no_model_sec_name_n", {I(nw_base, "-", "")}, ALLFAIL), mkIniTc("no_model_sec_n", {input, out+"input_layers=inputlayer"}, ALLFAIL), mkIniTc("empty_n", {}, ALLFAIL), - mkIniTc("no_layers_n", {nw_adam}, ALLFAIL), - mkIniTc("no_layers_2_n", {nw_adam, dataset}, ALLFAIL), + mkIniTc("no_layers_n", {nw_base, adam}, ALLFAIL), + mkIniTc("no_layers_2_n", {nw_base, adam, dataset}, ALLFAIL), /// #391 - // mkIniTc("ini_has_empty_value_n", {nw_adam + "epsilon = _", input, out}, ALLFAIL), + // mkIniTc("ini_has_empty_value_n", {nw_base, adam + "epsilon = _", input, out}, ALLFAIL), /**< negative: property(hyperparam) validation (5 negative cases) */ - mkIniTc("wrong_opt_type_n", {nw_adam + "Optimizer = wrong_opt", input, out+"input_layers=inputlayer"}, ALLFAIL), - mkIniTc("adam_minus_lr_n", {nw_adam + "Learning_rate = -0.1", input, out+"input_layers=inputlayer"}, ALLFAIL), - mkIniTc("sgd_minus_lr_n", {nw_sgd + "Learning_rate = -0.1", input, out+"input_layers=inputlayer"}, ALLFAIL), - mkIniTc("no_loss_p", {nw_adam + "-loss", input, out+"input_layers=inputlayer"}, SUCCESS), - mkIniTc("unknown_layer_type_n", {nw_adam, input + "Type = asdf", out+"input_layers=inputlayer"}, ALLFAIL), - mkIniTc("unknown_layer_type2_n", {nw_adam, input, out + "Type = asdf"+"input_layers=inputlayer", I(out, "outlayer", "")}, ALLFAIL), + mkIniTc("wrong_opt_type_n", {nw_base, adam + "Type = wrong_opt", input, out+"input_layers=inputlayer"}, ALLFAIL), + mkIniTc("adam_minus_lr_n", {nw_base, adam + "Learning_rate = -0.1", input, out+"input_layers=inputlayer"}, ALLFAIL), + mkIniTc("sgd_minus_lr_n", {nw_base, sgd + "Learning_rate = -0.1", input, out+"input_layers=inputlayer"}, ALLFAIL), + mkIniTc("no_loss_p", {nw_base + "-loss", adam, input, out+"input_layers=inputlayer"}, SUCCESS), + mkIniTc("unknown_layer_type_n", {nw_base, adam, input + "Type = asdf", out+"input_layers=inputlayer"}, ALLFAIL), + mkIniTc("unknown_layer_type2_n", {nw_base, adam, input, out + "Type = asdf"+"input_layers=inputlayer", I(out, "outlayer", "")}, ALLFAIL), /**< negative: little bit of tweeks to check determinancy (5 negative cases) */ - mkIniTc("wrong_nw_dataset_n", {nw_adam, input, out+"input_layers=inputlayer", dataset + "-LabelData"}, ALLFAIL), - mkIniTc("wrong_nw_dataset2_n", {nw_adam, dataset + "-LabelData", input, out+"input_layers=inputlayer"}, ALLFAIL), + mkIniTc("wrong_nw_dataset_n", {nw_base, adam, input, out+"input_layers=inputlayer", dataset + "-LabelData"}, ALLFAIL), + mkIniTc("wrong_nw_dataset2_n", {nw_base, adam, dataset + "-LabelData", input, out+"input_layers=inputlayer"}, ALLFAIL), /**< negative: dataset is not complete (5 negative cases) */ - mkIniTc("no_trainingSet_n", {nw_adam, dataset + "-TrainData", input, out+"input_layers=inputlayer"}, ALLFAIL), - mkIniTc("no_labelSet_n", {nw_adam, dataset + "-LabelData", input, out+"input_layers=inputlayer"}, ALLFAIL), + mkIniTc("no_trainingSet_n", {nw_base, adam, dataset + "-TrainData", input, out+"input_layers=inputlayer"}, ALLFAIL), + mkIniTc("no_labelSet_n", {nw_base, adam, dataset + "-LabelData", input, out+"input_layers=inputlayer"}, ALLFAIL), - mkIniTc("backbone_filemissing_n", {nw_adam, dataset + "-LabelData", input, out+"input_layers=inputlayer"}, ALLFAIL) + mkIniTc("backbone_filemissing_n", {nw_base, adam, dataset + "-LabelData", input, out+"input_layers=inputlayer"}, ALLFAIL) /// #if gtest_version <= 1.7.0 )); /// #else gtest_version > 1.8.0 @@ -349,7 +352,7 @@ INSTANTIATE_TEST_CASE_P( * @brief Ini file unittest with backbone with wrong file */ TEST(nntrainerIniTest, backbone_n_01) { - ScopedIni s{"backbone_n1", {nw_base, backbone_random}}; + ScopedIni s{"backbone_n1", {nw_base, adam, backbone_random}}; nntrainer::NeuralNetwork NN; EXPECT_EQ(NN.loadFromConfig(s.getIniName()), ML_ERROR_INVALID_PARAMETER); @@ -360,7 +363,7 @@ TEST(nntrainerIniTest, backbone_n_01) { */ TEST(nntrainerIniTest, backbone_n_02) { ScopedIni b{"base", {nw_base}}; - ScopedIni s{"backbone_n2", {nw_base, backbone_valid}}; + ScopedIni s{"backbone_n2", {nw_base, adam, backbone_valid}}; nntrainer::NeuralNetwork NN; EXPECT_EQ(NN.loadFromConfig(s.getIniName()), ML_ERROR_INVALID_PARAMETER); @@ -371,7 +374,7 @@ TEST(nntrainerIniTest, backbone_n_02) { */ TEST(nntrainerIniTest, backbone_p_03) { ScopedIni b{"base", {nw_base, batch_normal}}; - ScopedIni s{"backbone_p3", {nw_base, backbone_valid}}; + ScopedIni s{"backbone_p3", {nw_base, adam, backbone_valid}}; nntrainer::NeuralNetwork NN; EXPECT_EQ(NN.loadFromConfig(s.getIniName()), ML_ERROR_NONE); @@ -382,7 +385,7 @@ TEST(nntrainerIniTest, backbone_p_03) { */ TEST(nntrainerIniTest, backbone_p_04) { ScopedIni b{"base", {flatten, conv2d}}; - ScopedIni s{"backbone_p4", {nw_base, backbone_valid}}; + ScopedIni s{"backbone_p4", {nw_base, adam, backbone_valid}}; nntrainer::NeuralNetwork NN; EXPECT_EQ(NN.loadFromConfig(s.getIniName()), ML_ERROR_NONE); @@ -394,15 +397,15 @@ TEST(nntrainerIniTest, backbone_p_04) { TEST(nntrainerIniTest, backbone_p_05) { /** Create a backbone.ini */ - ScopedIni b("base", {nw_adam, conv2d}); + ScopedIni b("base", {nw_base, conv2d}); /** Create a model of 4 conv layers using backbone */ ScopedIni backbone_made( - "backbone_made", - {nw_sgd, input2d, I("block1") + backbone_valid + "input_layers=inputlayer", - I("block2") + backbone_valid + "input_layers=block1", - I("block3") + backbone_valid + "input_layers=block2", - I("block4") + backbone_valid + "input_layers=block3"}); + "backbone_made", {nw_base, sgd, input2d, + I("block1") + backbone_valid + "input_layers=inputlayer", + I("block2") + backbone_valid + "input_layers=block1", + I("block3") + backbone_valid + "input_layers=block2", + I("block4") + backbone_valid + "input_layers=block3"}); nntrainer::NeuralNetwork NN_backbone; EXPECT_EQ(NN_backbone.loadFromConfig(backbone_made.getIniName()), @@ -419,11 +422,11 @@ TEST(nntrainerIniTest, backbone_p_05) { /** Create the same model directly without using backbone */ // std::string conv2d_orig_name = conv2d.getName(); ScopedIni direct_made( - "direct_made", - {nw_sgd, input2d, I("block1conv2d") + conv2d + "input_layers=inputlayer", - I("block2conv2d") + conv2d + "input_layers=block1conv2d", - I("block3conv2d") + conv2d + "input_layers=block2conv2d", - I("block4conv2d") + conv2d + "input_layers=block3conv2d"}); + "direct_made", {nw_base, sgd, input2d, + I("block1conv2d") + conv2d + "input_layers=inputlayer", + I("block2conv2d") + conv2d + "input_layers=block1conv2d", + I("block3conv2d") + conv2d + "input_layers=block2conv2d", + I("block4conv2d") + conv2d + "input_layers=block3conv2d"}); nntrainer::NeuralNetwork NN_direct; EXPECT_EQ(NN_direct.loadFromConfig(direct_made.getIniName()), ML_ERROR_NONE); @@ -457,7 +460,7 @@ TEST(nntrainerIniTest, backbone_p_05) { */ TEST(nntrainerIniTest, backbone_p_06) { ScopedIni b("base", {flatten, conv2d}); - ScopedIni s("backbone_p6", {nw_base, backbone_valid}); + ScopedIni s("backbone_p6", {nw_base, adam, backbone_valid}); nntrainer::NeuralNetwork NN; EXPECT_EQ(NN.loadFromConfig(s.getIniName()), ML_ERROR_NONE); @@ -473,7 +476,7 @@ TEST(nntrainerIniTest, backbone_p_06) { */ TEST(nntrainerIniTest, backbone_p_07) { ScopedIni b("base", {conv2d}); - ScopedIni s("backbone_p7", {nw_base, backbone_notrain, backbone_train}); + ScopedIni s("backbone_p7", {nw_base, adam, backbone_notrain, backbone_train}); nntrainer::NeuralNetwork NN; EXPECT_EQ(NN.loadFromConfig(s.getIniName()), ML_ERROR_NONE); @@ -488,7 +491,7 @@ TEST(nntrainerIniTest, backbone_p_07) { * @brief Ini file unittest with backbone with normal backbone */ TEST(nntrainerIniTest, backbone_n_08) { - ScopedIni s("backbone_n8", {nw_base, backbone_random_external}); + ScopedIni s("backbone_n8", {nw_base, adam, backbone_random_external}); nntrainer::NeuralNetwork NN; @@ -506,7 +509,7 @@ TEST(nntrainerIniTest, backbone_n_08) { */ TEST(nntrainerIniTest, backbone_p_09) { ScopedIni s("backbone_p9", - {nw_base_mse + "-batch_size", backbone_valid_external}); + {nw_base_mse + "-batch_size", adam, backbone_valid_external}); nntrainer::NeuralNetwork NN; #if defined(ENABLE_NNSTREAMER_BACKBONE) || defined(ENABLE_TFLITE_BACKBONE) @@ -523,7 +526,8 @@ TEST(nntrainerIniTest, backbone_p_09) { */ // Enable after sepearet memory assign and initialization of graph TEST(nntrainerIniTest, backbone_p_10) { - ScopedIni s("backbone_p10", {nw_base_mse, backbone_valid_external_no_shape}); + ScopedIni s("backbone_p10", + {nw_base_mse, adam, backbone_valid_external_no_shape}); nntrainer::NeuralNetwork NN; #if defined(ENABLE_NNSTREAMER_BACKBONE) || defined(ENABLE_TFLITE_BACKBONE) @@ -547,11 +551,11 @@ TEST(nntrainerIniTest, backbone_p_11) { ScopedIni ini_scaled_half( "backbone_p11_scaled_half", - {nw_base_mse, input2d, backbone_scaled + "input_layers=inputlayer"}); + {nw_base_mse, adam, input2d, backbone_scaled + "input_layers=inputlayer"}); ScopedIni ini_full( "backbone_p11_full", - {nw_base_mse, input2d, backbone_valid + "input_layers=inputlayer"}); + {nw_base_mse, adam, input2d, backbone_valid + "input_layers=inputlayer"}); nntrainer::NeuralNetwork NN_scaled_half, NN_full; @@ -580,10 +584,10 @@ TEST(nntrainerIniTest, backbone_p_12) { ScopedIni b("base", {out, batch_normal + "input_layers=fclayer"}); ScopedIni scaled_half( "backbone_p12_scaled_half", - {{nw_base_mse, input, backbone_scaled + "input_layers=inputlayer"}}); + {{nw_base_mse, adam, input, backbone_scaled + "input_layers=inputlayer"}}); ScopedIni scaled_full( "backbone_p12_scaled_full", - {nw_base_mse, input, backbone_valid + "input_layers=inputlayer"}); + {nw_base_mse, adam, input, backbone_valid + "input_layers=inputlayer"}); nntrainer::NeuralNetwork NN_scaled_half, NN_full; @@ -614,11 +618,11 @@ TEST(nntrainerIniTest, backbone_p_13) { ScopedIni scaled_half( "backbone_p13_scaled_half", - {nw_base_mse, input2d, backbone_scaled + "input_layers=inputlayer"}); + {nw_base_mse, adam, input2d, backbone_scaled + "input_layers=inputlayer"}); ScopedIni scaled_full( "backbone_p13_full", - {nw_base_mse, input2d, backbone_valid + "input_layers=inputlayer"}); + {nw_base_mse, adam, input2d, backbone_valid + "input_layers=inputlayer"}); nntrainer::NeuralNetwork NN_scaled_half, NN_full; @@ -645,13 +649,13 @@ TEST(nntrainerIniTest, backbone_p_13) { TEST(nntrainerIniTest, backbone_p_14) { ScopedIni base("base", {conv2d_shape, conv2d + "input_layers=conv2d_shape"}); - ScopedIni scaled_zero( - "backbone_p14_scaled_zero", - {nw_base_mse, input2d, backbone_scaled_zero + "input_layers=inputlayer"}); + ScopedIni scaled_zero("backbone_p14_scaled_zero", + {nw_base_mse, adam, input2d, + backbone_scaled_zero + "input_layers=inputlayer"}); ScopedIni scaled_full( "backbone_p14_full", - {nw_base_mse, input2d, backbone_valid + "input_layers=inputlayer"}); + {nw_base_mse, adam, input2d, backbone_valid + "input_layers=inputlayer"}); nntrainer::NeuralNetwork NN_scaled_zero, NN_full; @@ -676,14 +680,14 @@ TEST(nntrainerIniTest, backbone_p_14) { TEST(nntrainerIniTest, backbone_n_15) { ScopedIni base("base", {conv2d, conv2d}); - ScopedIni full("backbone_n15_scaled", {nw_base_mse, backbone_valid}); + ScopedIni full("backbone_n15_scaled", {nw_base_mse, adam, backbone_valid}); nntrainer::NeuralNetwork NN_scaled, NN_full; EXPECT_EQ(NN_full.loadFromConfig(full.getIniName()), ML_ERROR_NONE); EXPECT_EQ(NN_full.compile(), ML_ERROR_INVALID_PARAMETER); EXPECT_EQ(NN_full.initialize(), ML_ERROR_NOT_SUPPORTED); - ScopedIni scaled("backbone_n15_scaled", {nw_base_mse, backbone_scaled}); + ScopedIni scaled("backbone_n15_scaled", {nw_base_mse, adam, backbone_scaled}); EXPECT_EQ(NN_scaled.loadFromConfig(scaled.getIniName()), ML_ERROR_NONE); EXPECT_EQ(NN_scaled.compile(), ML_ERROR_INVALID_PARAMETER); @@ -699,13 +703,13 @@ TEST(nntrainerIniTest, backbone_n_16) { ScopedIni base("base", {conv2d_shape, conv2d + "input_layers=conv2d_shape"}); - ScopedIni full("backbone_n16_full", {nw_base_mse, backbone_valid}); + ScopedIni full("backbone_n16_full", {nw_base_mse, adam, backbone_valid}); EXPECT_EQ(NN_full.loadFromConfig(full.getIniName()), ML_ERROR_NONE); EXPECT_EQ(NN_full.compile(), ML_ERROR_INVALID_PARAMETER); EXPECT_EQ(NN_full.initialize(), ML_ERROR_NOT_SUPPORTED); - ScopedIni scaled("backbone_n16_full", {nw_base_mse, backbone_scaled}); + ScopedIni scaled("backbone_n16_full", {nw_base_mse, adam, backbone_scaled}); EXPECT_EQ(NN_scaled.loadFromConfig(scaled.getIniName()), ML_ERROR_NONE); EXPECT_EQ(NN_scaled.compile(), ML_ERROR_INVALID_PARAMETER); @@ -722,7 +726,7 @@ TEST(nntrainerIniTest, backbone_p_17) { ScopedIni full( "backbone_p17_full", - {nw_base_mse, input2d, backbone_valid + "input_layers=inputlayer"}); + {nw_base_mse, adam, input2d, backbone_valid + "input_layers=inputlayer"}); EXPECT_EQ(NN_full.loadFromConfig(full.getIniName()), ML_ERROR_NONE); EXPECT_EQ(NN_full.compile(), ML_ERROR_NONE); @@ -730,7 +734,7 @@ TEST(nntrainerIniTest, backbone_p_17) { ScopedIni scaled( "backbone_p17_scaled", - {nw_base_mse, input2d, backbone_scaled + "input_layers=inputlayer"}); + {nw_base_mse, adam, input2d, backbone_scaled + "input_layers=inputlayer"}); EXPECT_EQ(NN_scaled.loadFromConfig(scaled.getIniName()), ML_ERROR_NONE); EXPECT_EQ(NN_scaled.compile(), ML_ERROR_NONE); @@ -746,9 +750,9 @@ TEST(nntrainerIniTest, backbone_n_18) { ScopedIni base("base", {input2d, conv2d + "input_layers=inputlayer", flatten + "input_layers=conv2d"}); - ScopedIni backbone( - "Backbone_n18", - {nw_base_mse, input, backbone_valid_inout + "input_layers=inputlayer"}); + ScopedIni backbone("Backbone_n18", + {nw_base_mse, adam, input, + backbone_valid_inout + "input_layers=inputlayer"}); EXPECT_EQ(NN.loadFromConfig(backbone.getIniName()), ML_ERROR_INVALID_PARAMETER); @@ -764,9 +768,9 @@ TEST(nntrainerIniTest, backbone_n_19) { ScopedIni base("base", {input2d, conv2d + "input_layers=inputlayer", batch_normal + "input_layers=conv2d"}); - ScopedIni backbone( - "backbone_n19", - {nw_base_mse, input, backbone_valid_inout + "input_layers=inputlayer"}); + ScopedIni backbone("backbone_n19", + {nw_base_mse, adam, input, + backbone_valid_inout + "input_layers=inputlayer"}); EXPECT_EQ(NN.loadFromConfig(backbone.getIniName()), ML_ERROR_INVALID_PARAMETER); @@ -783,9 +787,9 @@ TEST(nntrainerIniTest, backbone_p_20) { {input2d, conv2d + "input_layers=inputlayer", flatten + "input_layers=conv2d", out + "input_layers=flat"}); - ScopedIni backbone( - "backbone_p20", - {nw_base_mse, input, backbone_valid_inout + "input_layers=inputlayer"}); + ScopedIni backbone("backbone_p20", + {nw_base_mse, adam, input, + backbone_valid_inout + "input_layers=inputlayer"}); EXPECT_EQ(NN.loadFromConfig(backbone.getIniName()), ML_ERROR_NONE); EXPECT_EQ(NN.compile(), ML_ERROR_NONE); @@ -800,9 +804,9 @@ TEST(nntrainerIniTest, backbone_p_20) { */ TEST(nntrainerIniTest, backbone_relative_to_ini_p) { ScopedIni b{getResPath("base"), {nw_base, batch_normal}}; - ScopedIni s{ - getResPath("original"), - {nw_base + "loss=mse", input, backbone_valid + "input_layers=inputlayer"}}; + ScopedIni s{getResPath("original"), + {nw_base + "loss=mse", adam, input, + backbone_valid + "input_layers=inputlayer"}}; nntrainer::NeuralNetwork NN; @@ -818,9 +822,9 @@ TEST(nntrainerIniTest, backbone_relative_to_ini_p) { */ TEST(nntrainerIniTest, backbone_from_different_directory_n) { ScopedIni b{"base", {nw_base, batch_normal}}; - ScopedIni s{ - getResPath("original"), - {nw_base + "loss=mse", input, backbone_valid + "input_layers=inputlayer"}}; + ScopedIni s{getResPath("original"), + {nw_base + "loss=mse", adam, input, + backbone_valid + "input_layers=inputlayer"}}; nntrainer::NeuralNetwork NN; @@ -834,9 +838,9 @@ TEST(nntrainerIniTest, backbone_from_different_directory_n) { */ TEST(nntrainerIniTest, backbone_based_on_working_directory_p) { ScopedIni b{getResPath("base", {"test"}), {nw_base, batch_normal}}; - ScopedIni s{ - getResPath("original"), - {nw_base + "loss=mse", input, backbone_valid + "input_layers=inputlayer"}}; + ScopedIni s{getResPath("original"), + {nw_base + "loss=mse", adam, input, + backbone_valid + "input_layers=inputlayer"}}; nntrainer::AppContext ac(nntrainer::AppContext::Global()); ac.setWorkingDirectory(getResPath("", {"test"})); diff --git a/test/unittest/unittest_nntrainer_models.cpp b/test/unittest/unittest_nntrainer_models.cpp index e88a590..f372c09 100644 --- a/test/unittest/unittest_nntrainer_models.cpp +++ b/test/unittest/unittest_nntrainer_models.cpp @@ -555,6 +555,7 @@ static IniSection softmax_base = act_base + "Activation = softmax"; static IniSection sigmoid_base = act_base + "Activation = sigmoid"; static IniSection relu_base = act_base + "Activation = relu"; static IniSection bn_base("bn", "Type=batch_normalization"); +static IniSection sgd_base("optimizer", "Type = sgd"); using I = IniSection; using INI = IniTestWrapper; @@ -594,7 +595,8 @@ using INI = IniTestWrapper; // clang-format off INI fc_sigmoid_mse( "fc_sigmoid_mse", - {nn_base + "learning_rate=1 | optimizer=sgd | loss=mse | batch_size = 3", + {nn_base + "loss=mse | batch_size = 3", + sgd_base + "learning_rate = 1", I("input") + input_base + "input_shape = 1:1:3", I("dense") + fc_base + "unit = 5", I("act") + sigmoid_base, @@ -606,7 +608,8 @@ INI fc_sigmoid_cross = INI fc_relu_mse( "fc_relu_mse", - {nn_base + "Learning_rate=0.1 | Optimizer=sgd | Loss=mse | batch_size = 3", + {nn_base + "Loss=mse | batch_size = 3", + sgd_base + "learning_rate = 0.1", I("input") + input_base + "input_shape = 1:1:3", I("dense") + fc_base + "unit = 10", I("act") + relu_base, @@ -615,7 +618,8 @@ INI fc_relu_mse( INI fc_bn_sigmoid_cross( "fc_bn_sigmoid_cross", - {nn_base + "learning_rate=1 | optimizer=sgd | loss=cross | batch_size = 3", + {nn_base + "loss=cross | batch_size = 3", + sgd_base + "learning_rate = 1", I("input") + input_base + "input_shape = 1:1:3", I("dense") + fc_base + "unit = 10" + "input_layers=input", I("bn") + bn_base + "input_layers=dense", @@ -632,7 +636,8 @@ std::string mnist_pooling = INI mnist_conv_cross( "mnist_conv_cross", { - nn_base + "learning_rate=0.1 | optimizer=sgd | loss=cross | batch_size=3", + nn_base + "loss=cross | batch_size=3", + sgd_base + "learning_rate = 0.1", I("input") + input_base + "input_shape=2:4:5", I("conv2d_c1_layer") + conv_base + "kernel_size=3,4 | filters=2" +"input_layers=input", I("act_1") + sigmoid_base +"input_layers=conv2d_c1_layer", @@ -646,7 +651,8 @@ INI mnist_conv_cross( INI conv_1x1( "conv_1x1", { - nn_base + "learning_rate=0.1 | optimizer=sgd | loss=cross | batch_size=3", + nn_base + "loss=cross | batch_size=3", + sgd_base + "learning_rate = 0.1", I("input") + input_base + "input_shape=2:4:5", I("conv2d_c1_layer") + conv_base + "kernel_size=1,1 | filters=4", I("act_1") + sigmoid_base, @@ -659,7 +665,8 @@ INI conv_1x1( INI conv_input_matches_kernel( "conv_input_matches_kernel", { - nn_base + "learning_rate=0.1 | optimizer=sgd | loss=cross | batch_size=3", + nn_base + "loss=cross | batch_size=3", + sgd_base + "learning_rate = 0.1", I("input") + input_base + "input_shape=2:4:5", I("conv2d_c1_layer") + conv_base + "kernel_size=4,5 | filters=4" +"input_layers=input", I("act_1") + sigmoid_base +"input_layers=conv2d_c1_layer", @@ -672,8 +679,9 @@ INI conv_input_matches_kernel( INI conv_basic( "conv_basic", { - nn_base + "learning_rate=0.1 | optimizer=sgd | loss=cross | batch_size=3", - I("input") + input_base + "input_shape=2:5:3", + nn_base + "loss=cross | batch_size=3", + sgd_base + "learning_rate = 0.1", + I("input") + input_base + "input_shape=2:5:3", I("conv2d_c1") + conv_base + "kernel_size = 3,3 | filters=4" + "input_layers=input", I("act_1") + sigmoid_base +"input_layers=conv2d_c1", @@ -687,7 +695,8 @@ INI conv_same_padding( "conv_same_padding", { nn_base + "learning_rate=0.1 | optimizer=sgd | loss=cross | batch_size=3", - I("input") + input_base + "input_shape=2:5:3", + sgd_base + "learning_rate = 0.1", + I("input") + input_base + "input_shape=2:5:3", I("conv2d_c1") + conv_base + "kernel_size = 3,3 | filters=4 | padding =1,1" + "input_layers=input", I("act_1") + sigmoid_base +"input_layers=conv2d_c1", @@ -700,8 +709,9 @@ INI conv_same_padding( INI conv_multi_stride( "conv_multi_stride", { - nn_base + "learning_rate=0.1 | optimizer=sgd | loss=cross | batch_size=3", - I("input") + input_base + "input_shape=2:5:3", + nn_base + "loss=cross | batch_size=3", + sgd_base + "learning_rate = 0.1", + I("input") + input_base + "input_shape=2:5:3", I("conv2d_c1") + conv_base + "kernel_size = 3,3 | filters=4 | stride=2,2" + "input_layers=input", I("act_1") + sigmoid_base +"input_layers=conv2d_c1", @@ -729,8 +739,9 @@ INI conv_uneven_strides( INI conv_same_padding_multi_stride( "conv_same_padding_multi_stride", { - nn_base + "learning_rate=0.1 | optimizer=sgd | loss=cross | batch_size=3", - I("input") + input_base + "input_shape=2:5:3", + nn_base + "loss=cross | batch_size=3", + sgd_base + "learning_rate = 0.1", + I("input") + input_base + "input_shape=2:5:3", I("conv2d_c1") + conv_base + "kernel_size = 3,3 | filters=4 | stride=2,2 | padding=1,1" + "input_layers=input", I("act_1") + sigmoid_base +"input_layers=conv2d_c1", @@ -743,7 +754,8 @@ INI conv_same_padding_multi_stride( INI conv_no_loss_validate( "conv_no_loss_validate", { - nn_base + "learning_rate=0.1 | optimizer=sgd | batch_size=3", + nn_base + "batch_size=3", + sgd_base + "learning_rate = 0.1", I("input") + input_base + "input_shape=2:4:5", I("conv2d_c1_layer") + conv_base + "kernel_size=4,5 | filters=4" +"input_layers=input", I("act_1") + sigmoid_base +"input_layers=conv2d_c1_layer", @@ -756,7 +768,8 @@ INI conv_no_loss_validate( INI conv_none_loss_validate( "conv_none_loss_validate", { - nn_base + "learning_rate=0.1 | optimizer=sgd | loss=none | batch_size=3", + nn_base + "loss=none | batch_size=3", + sgd_base + "learning_rate = 0.1", I("input") + input_base + "input_shape=2:4:5", I("conv2d_c1_layer") + conv_base + "kernel_size=4,5 | filters=4" +"input_layers=input", I("act_1") + sigmoid_base +"input_layers=conv2d_c1_layer", -- 2.7.4