From: Piotr Kosko/Tizen API (PLT) /SRPOL/Engineer/Samsung Electronics Date: Thu, 10 Feb 2022 14:36:51 +0000 (+0100) Subject: [ML] Train - added a test method for trained model X-Git-Tag: submit/tizen/20220223.080409~3^2 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=c6dc0f0fb43bd1211e84a1e2d32cabfaf3bc336c;p=platform%2Fcore%2Fapi%2Fwebapi-plugins.git [ML] Train - added a test method for trained model [ACR] https://code.sec.samsung.net/jira/browse/TWDAPI-285 [Verification] Code compiles without errors. Verification method is available in JS console. var trainsetFile = "documents/trainingSet.dat"; var validsetFile = "documents/valSet.dat"; // TODO should support virtual roots var outputFile = "/home/owner/media/Documents/webapi_tizen_model.bin" var m = tizen.ml.trainer.createModel() var l1 = tizen.ml.trainer.createLayer("LAYER_IN") l1.setProperty("input_shape", "1:1:62720") l1.setProperty("normalization", "true") l1.setProperty("name", "inputlayer") m.addLayer(l1) var l2 = tizen.ml.trainer.createLayer("LAYER_FC") l2.setProperty("unit", "10") l2.setProperty("activation", "softmax") l2.setProperty("bias_initializer", "zeros") l2.setProperty("weight_regularizer", "l2norm") l2.setProperty("weight_regularizer_constant", "0.005") l2.setProperty("weight_initializer", "xavier_uniform") l2.setProperty("name", "fc1") l2.setProperty("input_layers", "inputlayer") m.addLayer(l2) var opt = tizen.ml.trainer.createOptimizer("OPTIMIZER_ADAM") opt.setProperty("learning_rate", "0.0001") opt.setProperty("decay_rate", "0.96") opt.setProperty("decay_steps", "1000") opt.setProperty("beta1", "0.002") opt.setProperty("beta2", "0.001") opt.setProperty("epsilon", "1e-7") m.setOptimizer(opt); var dataset = tizen.ml.trainer.createFileDataset(trainsetFile, validsetFile, /*no test file*/); dataset.setProperty("buffer_size", "100", "MODE_TRAIN"); dataset.setProperty("buffer_size", "100", "MODE_VALID"); m.setDataset(dataset); var compileOpts = { loss: "cross", batch_size: "16" } m.compile(compileOpts); var runOpts = { epochs: "2", save_path: outputFile } m.run(runOpts, (s) => { console.log("success"); console.log("Test result: " + m._checkMetrics(2.163000, 2.267410, 16.666700)); }, (e) => console.log("error " + JSON.stringify(e))); Change-Id: I4760fe341f58f84c985c6e4e4b609bafe36fb4be --- diff --git a/src/ml/js/ml_trainer.js b/src/ml/js/ml_trainer.js index 4c3a8412..f1e0712b 100755 --- a/src/ml/js/ml_trainer.js +++ b/src/ml/js/ml_trainer.js @@ -430,6 +430,30 @@ Model.prototype.summarize = function() { return result.summary }; +/* +Private method used for verification of training results. +It returns true if the results match given values (with tolerance 1.0e-5), false otherwise. +*/ +Model.prototype._checkMetrics = function (trainLoss, validLoss, validAccuracy) { + var callArgs = { + trainLoss: trainLoss, validLoss: validLoss, validAccuracy: validAccuracy, + id: this._id + } + + var result = native_.callSync('MLTrainerModelCheckMetrics', callArgs); + + if (native_.isFailure(result)) { + throw native_.getErrorObjectAndValidate( + result, + ValidBasicExceptions, + AbortError + ); + } + + return result.result +}; + + var ValidModelSaveExceptions = [ 'InvalidValuesError', 'TypeMismatchError', diff --git a/src/ml/ml_instance.cc b/src/ml/ml_instance.cc index 250eced0..bfff53d7 100644 --- a/src/ml/ml_instance.cc +++ b/src/ml/ml_instance.cc @@ -193,6 +193,7 @@ MlInstance::MlInstance() REGISTER_METHOD(MLTrainerModelAddLayer); REGISTER_METHOD(MLTrainerModelRun); REGISTER_METHOD(MLTrainerModelSummarize); + REGISTER_METHOD(MLTrainerModelCheckMetrics); REGISTER_METHOD(MLTrainerModelSave); REGISTER_METHOD(MLTrainerModelSetDataset); REGISTER_METHOD(MLTrainerModelSetOptimizer); @@ -1952,6 +1953,33 @@ void MlInstance::MLTrainerModelSummarize(const picojson::value& args, picojson:: ReportSuccess(out); } +void MlInstance::MLTrainerModelCheckMetrics(const picojson::value& args, + picojson::object& out) { + const std::string kTrainLoss = "trainLoss"; + const std::string kValidLoss = "validLoss"; + const std::string kValidAccuracy = "validAccuracy"; + ScopeLogger("args: %s", args.serialize().c_str()); + CHECK_ARGS(args, kId, double, out); + CHECK_ARGS(args, kTrainLoss, double, out); + CHECK_ARGS(args, kValidLoss, double, out); + CHECK_ARGS(args, kValidAccuracy, double, out); + + auto id = static_cast(args.get(kId).get()); + auto train_loss = args.get(kTrainLoss).get(); + auto valid_loss = args.get(kValidLoss).get(); + auto valid_accuracy = args.get(kValidAccuracy).get(); + + bool as_expected = false; + PlatformResult result = trainer_manager_.CheckMetrics( + id, train_loss, valid_loss, valid_accuracy, &as_expected); + if (!result) { + ReportError(result, &out); + return; + } + + ReportSuccess(picojson::value(as_expected), out); +} + void MlInstance::MLTrainerModelSave(const picojson::value& args, picojson::object& out) { ScopeLogger("args: %s", args.serialize().c_str()); diff --git a/src/ml/ml_instance.h b/src/ml/ml_instance.h index cfec0a7b..5e3095da 100644 --- a/src/ml/ml_instance.h +++ b/src/ml/ml_instance.h @@ -166,6 +166,8 @@ class MlInstance : public common::ParsedInstance { void MLTrainerModelAddLayer(const picojson::value& args, picojson::object& out); void MLTrainerModelRun(const picojson::value& args, picojson::object& out); void MLTrainerModelSummarize(const picojson::value& args, picojson::object& out); + void MLTrainerModelCheckMetrics(const picojson::value& args, + picojson::object& out); void MLTrainerModelSave(const picojson::value& args, picojson::object& out); void MLTrainerModelSetDataset(const picojson::value& args, picojson::object& out); void MLTrainerModelSetOptimizer(const picojson::value& args, picojson::object& out); diff --git a/src/ml/ml_trainer_manager.cc b/src/ml/ml_trainer_manager.cc index 00795d51..d819d561 100644 --- a/src/ml/ml_trainer_manager.cc +++ b/src/ml/ml_trainer_manager.cc @@ -334,8 +334,74 @@ PlatformResult TrainerManager::ModelSummarize(int id, return PlatformResult(); } -PlatformResult TrainerManager::ModelSave(int id, - const std::string& path, +// These values are defined in nntrainer tests +// nntrainer/include/nntrainer_test_util.h file. These defines are just copied +// because mentioned header is not public. +#define ML_TRAIN_SUMMARY_MODEL_TRAIN_LOSS 101 +#define ML_TRAIN_SUMMARY_MODEL_VALID_LOSS 102 +#define ML_TRAIN_SUMMARY_MODEL_VALID_ACCURACY 103 + +PlatformResult TrainerManager::CheckMetrics(int id, double train_loss, + double valid_loss, + double valid_accuracy, bool* res) { + ScopeLogger("Expected train_loss %f, valid_loss: %f, valid_accuracy: %f", + train_loss, valid_loss, valid_accuracy); + const double tolerance = 1.0e-5; + + if (models_.find(id) == models_.end()) { + LoggerE("Could not find model with id: %d", id); + return PlatformResult(ErrorCode::ABORT_ERR, "Could not find model"); + } + + auto& model = models_[id]; + + int status = ML_ERROR_NONE; + char *summary1 = nullptr, *summary2 = nullptr, *summary3 = nullptr; + status = ml_train_model_get_summary( + model->getNative(), + (ml_train_summary_type_e)ML_TRAIN_SUMMARY_MODEL_TRAIN_LOSS, &summary1); + if (status != ML_ERROR_NONE) { + LoggerE("Could not get summary for model: %d (%s)", status, + ml_strerror(status)); + return PlatformResult(ErrorCode::ABORT_ERR, ml_strerror(status)); + } + double train_loss_real = std::strtof(summary1, nullptr); + LoggerE("Train loss value: %f", train_loss_real); + bool result = (train_loss_real - train_loss) < tolerance; + free(summary1); + + status = ml_train_model_get_summary( + model->getNative(), + (ml_train_summary_type_e)ML_TRAIN_SUMMARY_MODEL_VALID_LOSS, &summary2); + if (status != ML_ERROR_NONE) { + LoggerE("Could not get summary for model: %d (%s)", status, + ml_strerror(status)); + return PlatformResult(ErrorCode::ABORT_ERR, ml_strerror(status)); + } + double valid_loss_real = std::strtof(summary2, nullptr); + LoggerE("Valid loss value: %f", valid_loss_real); + result = result && ((valid_loss_real - valid_loss) < tolerance); + free(summary2); + + status = ml_train_model_get_summary( + model->getNative(), + (ml_train_summary_type_e)ML_TRAIN_SUMMARY_MODEL_VALID_ACCURACY, + &summary3); + if (status != ML_ERROR_NONE) { + LoggerE("Could not get summary for model: %d (%s)", status, + ml_strerror(status)); + return PlatformResult(ErrorCode::ABORT_ERR, ml_strerror(status)); + } + double valid_accuracy_real = std::strtof(summary3, nullptr); + LoggerE("Valid accuracy value: %f", valid_accuracy_real); + result = result && ((valid_accuracy_real - valid_accuracy) < tolerance); + free(summary3); + + *res = result; + return PlatformResult(); +}; + +PlatformResult TrainerManager::ModelSave(int id, const std::string& path, ml_train_model_format_e format) { ScopeLogger(); diff --git a/src/ml/ml_trainer_manager.h b/src/ml/ml_trainer_manager.h index 4e38c5a6..efd2fca6 100644 --- a/src/ml/ml_trainer_manager.h +++ b/src/ml/ml_trainer_manager.h @@ -45,6 +45,8 @@ class TrainerManager { PlatformResult ModelSetDataset(int id, int datasetId); PlatformResult ModelSummarize(int id, ml_train_summary_type_e level, std::string& summary); + PlatformResult CheckMetrics(int id, double train_loss, double valid_loss, + double valid_accuracy, bool* result); PlatformResult ModelSave(int id, const std::string& path, ml_train_model_format_e format);