From 332417f57c2eaf7c8905d0e5e17958188d2363fd Mon Sep 17 00:00:00 2001 From: "Piotr Kosko/Native/Web API (PLT) /SRPOL/Engineer/Samsung Electronics" Date: Fri, 12 Feb 2021 10:36:44 +0100 Subject: [PATCH] [ML] Added SingleShot openModelAsync() implementation [ACR] https://code.sec.samsung.net/jira/browse/TWDAPI-273 [Verification] Code compiles without errors. Checked in chrome console with few calls: // Success calls: tizen.ml.single.openModelAsync("documents/mobilenet_v1_1.0_224_quant.tflite", (s) => console.log(s)) tizen.ml.single.openModelAsync("documents/mobilenet_v1_1.0_224_quant.tflite", (s) => console.log(s), null, null, null, "TENSORFLOW_LITE") tizen.ml.single.openModelAsync("documents/mobilenet_v1_1.0_224_quant.tflite", (s) => console.log(s), null, null, null, "TENSORFLOW_LITE", "ANY", true) // Fails for invalid or not-existing file: tizen.ml.single.openModelAsync("documents/fail.tflite", (s) => console.log(s), (e) => console.log(e)) // Abort Error tizen.ml.single.openModelAsync("documents/notexisting", (s) => console.log(s), (e) => console.log(e)) // NotFound tizen.ml.single.openModelAsync("notexisting", (s) => console.log(s), (e) => console.log(e)) // NotFound // ASYNC version verification // // without storage privilege tizen.ml.single.openModelAsync("notexistingfile", (s) => console.log(s), (e) => console.log("Async: " + JSON.stringify(e))) // async NotFoundError tizen.ml.single.openModelAsync("/dddd/notexistingfile", (s) => console.log(s), (e) => console.log("Async: " + JSON.stringify(e))) // async NotFoundError tizen.ml.single.openModelAsync("documents/notexistingfile", (s) => console.log(s), (e) => console.log("Async: " + JSON.stringify(e))) // throws SecurityError tizen.ml.single.openModelAsync("documents/mobilenet_v1_1.0_224_quant.tflite", (s) => console.log(s), (e) => console.log("Async: " + JSON.stringify(e))) // throws SecurityError // with storage privilege tizen.ml.single.openModelAsync("notexistingfile", (s) => console.log(s), (e) => console.log("Async: " + JSON.stringify(e))) // async NotFoundError tizen.ml.single.openModelAsync("/dddd/notexistingfile", (s) => console.log(s), (e) => console.log("Async: " + JSON.stringify(e))) // async NotFoundError tizen.ml.single.openModelAsync("documents/notexistingfile", (s) => console.log(s), (e) => console.log("Async: " + JSON.stringify(e))) // async NotFoundError tizen.ml.single.openModelAsync("documents/mobilenet_v1_1.0_224_quant.tflite", (s) => console.log(s), (e) => console.log("Async: " + JSON.stringify(e))) // success // SYNC version verification // // without storage privilege tizen.ml.single.openModel("notexistingfile") // throws NotFoundError tizen.ml.single.openModel("/dddd/notexistingfile") // throws NotFoundError tizen.ml.single.openModel("documents/notexistingfile") // throws SecurityError tizen.ml.single.openModel("documents/mobilenet_v1_1.0_224_quant.tflite") // throws SecurityError // with storage privilege tizen.ml.single.openModel("notexistingfile") // throws NotFoundError tizen.ml.single.openModel("/dddd/notexistingfile") // throws NotFoundError tizen.ml.single.openModel("documents/notexistingfile") // throws NotFoundError tizen.ml.single.openModel("documents/mobilenet_v1_1.0_224_quant.tflite") // success Change-Id: I93e009efec4e1bcbe6d6c3b381b34cada910d8aa --- src/ml/js/ml_single.js | 102 +++++++++++++++++++++++++++++++++++- src/ml/ml_instance.cc | 72 ++++++++++++++++--------- src/ml/ml_instance.h | 4 +- src/ml/ml_single_manager.cc | 4 +- src/ml/ml_single_manager.h | 3 ++ 5 files changed, 156 insertions(+), 29 deletions(-) diff --git a/src/ml/js/ml_single.js b/src/ml/js/ml_single.js index 503d1b17..d516997b 100755 --- a/src/ml/js/ml_single.js +++ b/src/ml/js/ml_single.js @@ -93,8 +93,108 @@ MachineLearningSingle.prototype.openModel = function() { }; // MachineLearningSingle::openModelAsync() +var ValidOpenModelAsyncCallbackErrors = [ + 'InvalidValuesError', + 'NotFoundError', + 'NotSupportedError', + 'AbortError' +]; +var ValidOpenModelAsyncExceptions = ['TypeMismatchError', 'SecurityError', 'AbortError']; +MachineLearningSingle.prototype.openModelAsync = function() { + var args = validator_.validateArgs(arguments, [ + { + name: 'modelPath', + type: types_.STRING + }, + { + name: 'successCallback', + type: types_.FUNCTION + }, + { + name: 'errorCallback', + type: types_.FUNCTION, + optional: true, + nullable: true + }, + { + name: 'inTensorsInfo', + type: types_.PLATFORM_OBJECT, + values: TensorsInfo, + optional: true, + nullable: true + }, + { + name: 'outTensorsInfo', + type: types_.PLATFORM_OBJECT, + values: TensorsInfo, + optional: true, + nullable: true + }, + { + name: 'fwType', + type: types_.ENUM, + values: Object.keys(NNFWType), + optional: true + }, + { + name: 'hwType', + type: types_.ENUM, + values: Object.keys(HWType), + optional: true + }, + { + name: 'isDynamicMode', + type: types_.BOOLEAN, + optional: true + } + ]); + try { + // get normalized URI of a file + args.modelPath = tizen.filesystem.toURI(args.modelPath); + } catch (e) { + setTimeout(function() { + native_.callIfPossible( + args.errorCallback, + new WebAPIException(WebAPIException.NOT_FOUND_ERR, 'Path is invalid') + ); + }, 0); + return; + } -// OpenModelSuccessCallback + var nativeArgs = { + modelPath: args.modelPath, + inTensorsInfo: args.inTensorsInfo ? args.inTensorsInfo._id : NO_ID, + outTensorsInfo: args.outTensorsInfo ? args.outTensorsInfo._id : NO_ID, + fwType: args.fwType ? args.fwType : 'ANY', + hwType: args.hwType ? args.hwType : 'ANY', + isDynamicMode: args.isDynamicMode ? args.isDynamicMode : false, + async: true + }; + + var callback = function(result) { + if (native_.isFailure(result)) { + native_.callIfPossible( + args.errorCallback, + native_.getErrorObjectAndValidate( + result, + ValidOpenModelAsyncCallbackErrors, + AbortError + ) + ); + } else { + args.successCallback(new SingleShot(result.id)); + } + }; + + var result = native_.call('MLSingleManagerOpenModel', nativeArgs, callback); + if (native_.isFailure(result)) { + throw native_.getErrorObjectAndValidate( + result, + ValidOpenModelAsyncExceptions, + AbortError + ); + } +}; // SingleShot interface (input & output) var ValidInputExceptions = ['TypeMismatchError', 'AbortError']; diff --git a/src/ml/ml_instance.cc b/src/ml/ml_instance.cc index 7c489022..aee61d0a 100644 --- a/src/ml/ml_instance.cc +++ b/src/ml/ml_instance.cc @@ -117,8 +117,6 @@ MlInstance::MlInstance() // Single API begin REGISTER_METHOD(MLSingleManagerOpenModel); - // MachineLearningSingle::openModelAsync() - // OpenModelSuccessCallback REGISTER_METHOD(MLSingleShotGetTensorsInfo); REGISTER_METHOD(MLSingleShotSetInputInfo); // SingleShot::invoke() @@ -154,6 +152,7 @@ MlInstance::MlInstance() MlInstance::~MlInstance() { ScopeLogger(); + worker_.stop(); } TensorsInfoManager& MlInstance::GetTensorsInfoManager() { @@ -187,7 +186,7 @@ void MlInstance::MLTensorsInfoCreate(const picojson::value& args, picojson::obje ("Could not create new TensorsInfo handle")); return; } - out["id"] = picojson::value(static_cast(tensorsInfo->Id())); + out[kId] = picojson::value(static_cast(tensorsInfo->Id())); ReportSuccess(out); } @@ -730,6 +729,8 @@ const std::string kFwType = "fwType"; const std::string kHwType = "hwType"; const std::string kIsDynamicMode = "isDynamicMode"; const std::string kValue = "value"; +const std::string kCallbackId = "callbackId"; +const std::string kAsync = "async"; const int kNoId = -1; } // namespace @@ -744,13 +745,7 @@ void MlInstance::MLSingleManagerOpenModel(const picojson::value& args, picojson: CHECK_ARGS(args, kIsDynamicMode, bool, out); const auto& model_path = common::tools::ConvertUriToPath(args.get(kModelPath).get()); - PlatformResult result = common::tools::CheckFileAvailability(model_path); - if (!result) { - LogAndReportError( - PlatformResult(ErrorCode::NOT_FOUND_ERR, "File does not exist or is not accessible"), &out, - ("File does not exist or is not accessible: %s", model_path.c_str())); - return; - } + CHECK_STORAGE_ACCESS(model_path, &out); TensorsInfo* in_tensors_info = nullptr; auto inTensorId = static_cast(args.get(kInTensorsInfo).get()); @@ -775,7 +770,8 @@ void MlInstance::MLSingleManagerOpenModel(const picojson::value& args, picojson: } ml_nnfw_type_e nnfw_e = ML_NNFW_TYPE_ANY; - result = types::NNFWTypeEnum.getValue(args.get(kFwType).get(), &nnfw_e); + PlatformResult result = + types::NNFWTypeEnum.getValue(args.get(kFwType).get(), &nnfw_e); if (!result) { LogAndReportError(result, &out); return; @@ -790,21 +786,49 @@ void MlInstance::MLSingleManagerOpenModel(const picojson::value& args, picojson: auto is_dynamic_mode = args.get(kIsDynamicMode).get(); - int res_id = -1; - result = single_manager_.OpenModel(model_path, in_tensors_info, out_tensors_info, nnfw_e, hw_e, - is_dynamic_mode, &res_id); - if (!result) { - ReportError(result, &out); - return; - } + auto logic = [this, model_path, in_tensors_info, out_tensors_info, nnfw_e, hw_e, + is_dynamic_mode](decltype(out) out) { + PlatformResult result = common::tools::CheckFileAvailability(model_path); + if (!result) { + LogAndReportError( + PlatformResult(ErrorCode::NOT_FOUND_ERR, "File does not exist or is not accessible"), + &out, ("File does not exist or is not accessible: %s", model_path.c_str())); + return; + } - out["id"] = picojson::value(static_cast(res_id)); - ReportSuccess(out); -} + int res_id = -1; + result = single_manager_.OpenModel(model_path, in_tensors_info, out_tensors_info, nnfw_e, hw_e, + is_dynamic_mode, &res_id); + if (!result) { + ReportError(result, &out); + return; + } -// MachineLearningSingle::openModelAsync() + out[kId] = picojson::value(static_cast(res_id)); + ReportSuccess(out); + }; -// OpenModelSuccessCallback + bool async = + (args.contains(kAsync) && args.get(kAsync).is()) ? args.get(kAsync).get() : false; + + if (!async) { + logic(out); + } else { + // Async logic + CHECK_ARGS(args, kCallbackId, double, out); + double callback_id = args.get(kCallbackId).get(); + this->worker_.add_job([this, callback_id, logic] { + picojson::value response = picojson::value(picojson::object()); + picojson::object& async_out = response.get(); + async_out[kCallbackId] = picojson::value(callback_id); + logic(async_out); + this->PostMessage(response.serialize().c_str()); + }); + + // Sync return + ReportSuccess(out); + } +} // SingleShot input/output // TODO move to the up section with field names @@ -827,7 +851,7 @@ void MlInstance::MLSingleShotGetTensorsInfo(const picojson::value& args, picojso return; } - out["id"] = picojson::value(static_cast(res_id)); + out[kId] = picojson::value(static_cast(res_id)); ReportSuccess(out); } diff --git a/src/ml/ml_instance.h b/src/ml/ml_instance.h index eaa7cfc5..a44ce690 100644 --- a/src/ml/ml_instance.h +++ b/src/ml/ml_instance.h @@ -18,6 +18,7 @@ #define ML_ML_INSTANCE_H_ #include "common/extension.h" +#include "common/worker.h" #include "ml/ml_pipeline_manager.h" #include "ml/ml_single_manager.h" @@ -66,13 +67,12 @@ class MlInstance : public common::ParsedInstance { */ TensorsInfoManager tensors_info_manager_; TensorsDataManager tensors_data_manager_; + common::Worker worker_; // Common ML API end // Single API begin SingleManager single_manager_; void MLSingleManagerOpenModel(const picojson::value& args, picojson::object& out); - // MachineLearningSingle::openModelAsync() - // OpenModelSuccessCallback void MLSingleShotGetTensorsInfo(const picojson::value& args, picojson::object& out); void MLSingleShotSetInputInfo(const picojson::value& args, picojson::object& out); // SingleShot::invoke() diff --git a/src/ml/ml_single_manager.cc b/src/ml/ml_single_manager.cc index e21eaf2e..68c262b4 100644 --- a/src/ml/ml_single_manager.cc +++ b/src/ml/ml_single_manager.cc @@ -48,18 +48,18 @@ PlatformResult SingleManager::OpenModel(const std::string& modelPath, TensorsInf return util::ToPlatformResult(ret, "Failed to open model"); } + std::lock_guard singles_lock(singles_mutex_); int id = nextId_++; singles_[id] = std::make_unique(id, handle); *res_id = id; return PlatformResult{}; } -// MachineLearningSingle::openModelAsync() -// OpenModelSuccessCallback // SingleShot input SingleShot* SingleManager::GetSingleShot(int id) { ScopeLogger("id: %d", id); + std::lock_guard singles_lock(singles_mutex_); if (singles_.end() != singles_.find(id)) { return singles_[id].get(); } diff --git a/src/ml/ml_single_manager.h b/src/ml/ml_single_manager.h index 3482c3e6..ad232aa9 100644 --- a/src/ml/ml_single_manager.h +++ b/src/ml/ml_single_manager.h @@ -17,6 +17,8 @@ #ifndef ML_ML_SINGLE_MANAGER_H_ #define ML_ML_SINGLE_MANAGER_H_ +#include + #include "common/platform_result.h" #include "ml_singleshot.h" @@ -54,6 +56,7 @@ class SingleManager { private: int nextId_; std::map> singles_; + std::mutex singles_mutex_; TensorsInfoManager* tim_; SingleShot* GetSingleShot(int id); }; -- 2.34.1