From e7987a76f1bcc5396a61db94721e11074331a338 Mon Sep 17 00:00:00 2001 From: Inki Dae Date: Fri, 12 May 2023 16:23:53 +0900 Subject: [PATCH] mv_machine_learning: add async API for face detection task group [Issue type] : new feature Add async API support for face detection task group. For async API support, it implements '_incoming_queue' to store input tensors requested by user and '_outgoing_queue' to retrieve the final result after decoding the inference results obtained by processing each input tensor from the '_incoming_queue'. When a user requests the result, the final result is retrieved from '_outgoing_queue' and stored in '_current_result' for consistency because user can use two more native API to get the result, which is maintained until the next inference request is made. In default, async API has frame skip behavior if previous request is being performed. And which in turn, it minimizes time delay between input frame and inference result. There is no longer a need to perform complex tasks in application code to reduce the time delay with this patch. Change-Id: If46e7f9a3a8e5ffbe564cea0439ff3dfe762e6c9 Signed-off-by: Inki Dae --- include/mv_face_detection_internal.h | 98 ++++++++++++++-------- include/mv_face_detection_type.h | 2 + .../include/face_detection_adapter.h | 2 + .../src/face_detection_adapter.cpp | 17 +++- .../object_detection/src/mv_face_detection.cpp | 36 +++++++- .../test_object_detection_async.cpp | 93 ++++++++++++++++++-- 6 files changed, 204 insertions(+), 44 deletions(-) diff --git a/include/mv_face_detection_internal.h b/include/mv_face_detection_internal.h index e8d4518..208a246 100644 --- a/include/mv_face_detection_internal.h +++ b/include/mv_face_detection_internal.h @@ -37,9 +37,9 @@ extern "C" { /** * @internal - * @brief Creates a inference handle for object detection object. + * @brief Creates a inference handle for face detection object. * @details Use this function to create a inference handle. After the creation - * the object detection 3d task has to be prepared with + * the face detection task has to be prepared with * mv_face_detection_prepare() function to prepare a network * for the inference. * @@ -86,7 +86,7 @@ int mv_face_detection_destroy(mv_face_detection_h handle); * * @since_tizen 8.0 * - * @param[in] handle The handle to the object detection object. + * @param[in] handle The handle to the face detection object. * @param[in] model_name Model name. * @param[in] model_file Model file name. * @param[in] meta_file Model meta file name. @@ -97,14 +97,14 @@ int mv_face_detection_destroy(mv_face_detection_h handle); * @retval #MEDIA_VISION_ERROR_INVALID_PARAMETER Invalid parameter * @retval #MEDIA_VISION_ERROR_INVALID_OPERATION Invalid operation * - * @pre Create a object detection handle by calling @ref mv_face_detection_create() + * @pre Create a face detection handle by calling @ref mv_face_detection_create() */ int mv_face_detection_set_model(mv_face_detection_h handle, const char *model_name, const char *model_file, const char *meta_file, const char *label_file); /** * @internal - * @brief Configures the backend for the object detection inference. + * @brief Configures the backend for the face detection inference. * * @since_tizen 8.0 * @@ -120,8 +120,8 @@ int mv_face_detection_configure(mv_face_detection_h handle); /** * @internal - * @brief Prepares the object detection inference - * @details Use this function to prepare the object detection inference based on + * @brief Prepares the face detection inference + * @details Use this function to prepare the face detection inference based on * the configured network. * * @since_tizen 8.0 @@ -142,7 +142,7 @@ int mv_face_detection_prepare(mv_face_detection_h handle); /** * @internal - * @brief Performs the object detection inference on the @a source. + * @brief Performs the face detection inference on the @a source. * * @since_tizen 8.0 * @remarks This function is synchronous and may take considerable time to run. @@ -159,15 +159,45 @@ int mv_face_detection_prepare(mv_face_detection_h handle); * isn't supported * * @pre Create a source handle by calling mv_create_source() - * @pre Create an inference handle by calling mv_object_detect_create() - * @pre Prepare an inference by calling mv_object_detect_configure() - * @pre Prepare an inference by calling mv_object_detect_prepare() + * @pre Create an inference handle by calling mv_face_detect_create() + * @pre Prepare an inference by calling mv_face_detect_configure() + * @pre Prepare an inference by calling mv_face_detect_prepare() */ int mv_face_detection_inference(mv_face_detection_h handle, mv_source_h source); /** * @internal - * @brief Gets the object detection inference result on the @a source. + * @brief Performs asynchronously the face detection inference on the @a source. + * + * @since_tizen 7.5 + * @remarks This function operates asynchronously, so it returns immediately upon invocation. + * Therefore, user needs to receive the result though a given callback function. + * + * @param[in] handle The handle to the inference + * @param[in] source The handle to the source of the media + * @param[in] completion_cb A callback which is called internally by the framework + * once the given inference request is completed. + * @param[in] user_data A pointer to user data object. + * + * @return @c 0 on success, otherwise a negative error value + * @retval #MEDIA_VISION_ERROR_NONE Successful + * @retval #MEDIA_VISION_ERROR_NOT_SUPPORTED Not supported + * @retval #MEDIA_VISION_ERROR_INVALID_PARAMETER Invalid parameter + * @retval #MEDIA_VISION_ERROR_INTERNAL Internal error + * @retval #MEDIA_VISION_ERROR_NOT_SUPPORTED_FORMAT Source colorspace + * isn't supported + * + * @pre Create a source handle by calling mv_create_source() + * @pre Create an inference handle by calling mv_face_detect_create() + * @pre Prepare an inference by calling mv_face_detect_configure() + * @pre Prepare an inference by calling mv_face_detect_prepare() + */ +int mv_face_detection_inference_async(mv_face_detection_h handle, mv_source_h source, mv_completion_cb completion_cb, + void *user_data); + +/** + * @internal + * @brief Gets the face detection inference result on the @a source. * * @since_tizen 8.0 * @@ -187,10 +217,10 @@ int mv_face_detection_inference(mv_face_detection_h handle, mv_source_h source); * @retval #MEDIA_VISION_ERROR_INTERNAL Internal error * * @pre Create a source handle by calling mv_create_source() - * @pre Create an inference handle by calling mv_object_detect_create() - * @pre Prepare an inference by calling mv_object_detect_configure() - * @pre Prepare an inference by calling mv_object_detect_prepare() - * @pre Prepare an inference by calling mv_object_detect_inference() + * @pre Create an inference handle by calling mv_face_detect_create() + * @pre Prepare an inference by calling mv_face_detect_configure() + * @pre Prepare an inference by calling mv_face_detect_prepare() + * @pre Prepare an inference by calling mv_face_detect_inference() */ int mv_face_detection_get_result(mv_face_detection_h handle, unsigned int *number_of_objects, const unsigned int **indices, const float **confidences, const int **left, @@ -213,10 +243,10 @@ int mv_face_detection_get_result(mv_face_detection_h handle, unsigned int *numbe * @retval #MEDIA_VISION_ERROR_INTERNAL Internal error * * @pre Create a source handle by calling mv_create_source() - * @pre Create an inference handle by calling mv_object_detect_create() - * @pre Prepare an inference by calling mv_object_detect_configure() - * @pre Prepare an inference by calling mv_object_detect_prepare() - * @pre Prepare an inference by calling mv_object_detect_inference() + * @pre Create an inference handle by calling mv_face_detect_create() + * @pre Prepare an inference by calling mv_face_detect_configure() + * @pre Prepare an inference by calling mv_face_detect_prepare() + * @pre Prepare an inference by calling mv_face_detect_inference() */ int mv_face_detection_get_label(mv_face_detection_h handle, const unsigned int index, const char **label); @@ -227,7 +257,7 @@ int mv_face_detection_get_label(mv_face_detection_h handle, const unsigned int i * * @since_tizen 8.0 * - * @param[in] handle The handle to the object detection object. + * @param[in] handle The handle to the face detection object. * @param[in] engine_type A string of inference engine type. * @param[in] device_type A string of device type. * @@ -236,26 +266,26 @@ int mv_face_detection_get_label(mv_face_detection_h handle, const unsigned int i * @retval #MEDIA_VISION_ERROR_INVALID_PARAMETER Invalid parameter * @retval #MEDIA_VISION_ERROR_INVALID_OPERATION Invalid operation * - * @pre Create a object detection handle by calling @ref mv_face_detection_create() + * @pre Create a face detection handle by calling @ref mv_face_detection_create() */ int mv_face_detection_set_engine(mv_face_detection_h handle, const char *engine_type, const char *device_type); /** * @internal - * @brief Gets a number of inference engines available for object detection task API. - * @details Use this function to get how many inference engines are supported for object detection after calling @ref mv_face_detection_create(). + * @brief Gets a number of inference engines available for face detection task API. + * @details Use this function to get how many inference engines are supported for face detection after calling @ref mv_face_detection_create(). * * @since_tizen 8.0 * - * @param[in] handle The handle to the object detection object. - * @param[out] engine_count A number of inference engines available for object detection API. + * @param[in] handle The handle to the face detection object. + * @param[out] engine_count A number of inference engines available for face detection API. * * @return @c 0 on success, otherwise a negative error value * @retval #MEDIA_VISION_ERROR_NONE Successful * @retval #MEDIA_VISION_ERROR_INVALID_PARAMETER Invalid parameter * @retval #MEDIA_VISION_ERROR_INVALID_OPERATION Invalid operation * - * @pre Create a object detection handle by calling @ref mv_face_detection_create() + * @pre Create a face detection handle by calling @ref mv_face_detection_create() */ int mv_face_detection_get_engine_count(mv_face_detection_h handle, unsigned int *engine_count); @@ -266,7 +296,7 @@ int mv_face_detection_get_engine_count(mv_face_detection_h handle, unsigned int * * @since_tizen 8.0 * - * @param[in] handle The handle to the object detection object. + * @param[in] handle The handle to the face detection object. * @param[in] engine_index A inference engine index for getting the inference engine type. * @param[out] engine_type A string to inference engine. * @@ -275,7 +305,7 @@ int mv_face_detection_get_engine_count(mv_face_detection_h handle, unsigned int * @retval #MEDIA_VISION_ERROR_INVALID_PARAMETER Invalid parameter * @retval #MEDIA_VISION_ERROR_INVALID_OPERATION Invalid operation * - * @pre Get a number of inference engines available for object detection task API by calling @ref mv_face_detection_get_engine_count() + * @pre Get a number of inference engines available for face detection task API by calling @ref mv_face_detection_get_engine_count() */ int mv_face_detection_get_engine_type(mv_face_detection_h handle, const unsigned int engine_index, char **engine_type); @@ -286,7 +316,7 @@ int mv_face_detection_get_engine_type(mv_face_detection_h handle, const unsigned * * @since_tizen 8.0 * - * @param[in] handle The handle to the object detection object. + * @param[in] handle The handle to the face detection object. * @param[in] engine_type A inference engine string. * @param[out] device_count A number of device types available for a given inference engine. * @@ -295,7 +325,7 @@ int mv_face_detection_get_engine_type(mv_face_detection_h handle, const unsigned * @retval #MEDIA_VISION_ERROR_INVALID_PARAMETER Invalid parameter * @retval #MEDIA_VISION_ERROR_INVALID_OPERATION Invalid operation * - * @pre Create a object detection handle by calling @ref mv_face_detection_create() + * @pre Create a face detection handle by calling @ref mv_face_detection_create() */ int mv_face_detection_get_device_count(mv_face_detection_h handle, const char *engine_type, unsigned int *device_count); @@ -306,7 +336,7 @@ int mv_face_detection_get_device_count(mv_face_detection_h handle, const char *e * * @since_tizen 8.0 * - * @param[in] handle The handle to the object detection object. + * @param[in] handle The handle to the face detection object. * @param[in] engine_type A inference engine string. * @param[in] device_index A device index for getting the device type. * @param[out] device_type A string to device type. @@ -316,8 +346,8 @@ int mv_face_detection_get_device_count(mv_face_detection_h handle, const char *e * @retval #MEDIA_VISION_ERROR_INVALID_PARAMETER Invalid parameter * @retval #MEDIA_VISION_ERROR_INVALID_OPERATION Invalid operation * - * @pre Create a object detection handle by calling @ref mv_face_detection_create() - * @pre Configure object detection task by calling @ref mv_face_detection_configure() + * @pre Create a face detection handle by calling @ref mv_face_detection_create() + * @pre Configure face detection task by calling @ref mv_face_detection_configure() */ int mv_face_detection_get_device_type(mv_face_detection_h handle, const char *engine_type, const unsigned int device_index, char **device_type); diff --git a/include/mv_face_detection_type.h b/include/mv_face_detection_type.h index 6a4e514..be68132 100644 --- a/include/mv_face_detection_type.h +++ b/include/mv_face_detection_type.h @@ -40,6 +40,8 @@ extern "C" { */ typedef void *mv_face_detection_h; +typedef void (*mv_completion_cb)(mv_face_detection_h handle, void *user_data); + /** * @} */ diff --git a/mv_machine_learning/object_detection/include/face_detection_adapter.h b/mv_machine_learning/object_detection/include/face_detection_adapter.h index ec2536d..5c19d0d 100644 --- a/mv_machine_learning/object_detection/include/face_detection_adapter.h +++ b/mv_machine_learning/object_detection/include/face_detection_adapter.h @@ -37,6 +37,8 @@ private: std::string _meta_file; std::string _label_file; + void updateResult(ObjectDetectionResult &result); + public: FaceDetectionAdapter(); ~FaceDetectionAdapter(); diff --git a/mv_machine_learning/object_detection/src/face_detection_adapter.cpp b/mv_machine_learning/object_detection/src/face_detection_adapter.cpp index 3484566..31bd39b 100644 --- a/mv_machine_learning/object_detection/src/face_detection_adapter.cpp +++ b/mv_machine_learning/object_detection/src/face_detection_adapter.cpp @@ -35,7 +35,9 @@ template FaceDetectionAdapter::FaceDetectionAdapte } template FaceDetectionAdapter::~FaceDetectionAdapter() -{} +{ + _object_detection->preDestroy(); +} template void FaceDetectionAdapter::create(int type) { @@ -140,12 +142,21 @@ template void FaceDetectionAdapter::perform() template void FaceDetectionAdapter::performAsync(T &t) { - throw InvalidOperation("Not support yet."); + shared_ptr metaInfo = _object_detection->getInputMetaInfo(); + + if (metaInfo->dataType == MV_INFERENCE_DATA_UINT8) { + _object_detection->performAsync(t, metaInfo); + } else if (metaInfo->dataType == MV_INFERENCE_DATA_FLOAT32) { + _object_detection->performAsync(t, metaInfo); + // TODO + } else { + throw InvalidOperation("Invalid model data type."); + } } template V &FaceDetectionAdapter::getOutput() { - return _object_detection->result(); + return _object_detection->getOutput(); } template class FaceDetectionAdapter; diff --git a/mv_machine_learning/object_detection/src/mv_face_detection.cpp b/mv_machine_learning/object_detection/src/mv_face_detection.cpp index 40d6779..9850978 100644 --- a/mv_machine_learning/object_detection/src/mv_face_detection.cpp +++ b/mv_machine_learning/object_detection/src/mv_face_detection.cpp @@ -67,7 +67,12 @@ int mv_face_detection_create(mv_face_detection_h *handle) int mv_face_detection_destroy(mv_face_detection_h handle) { - lock_guard lock(g_face_detection_mutex); + // TODO. find proper solution later. + // For thread safety, lock is needed here but if async API is used then dead lock occurs + // because mv_face_detection_destroy_open function acquires a lock and, + // while waiting for the thread loop to finish, the same lock is also acquired + // within functions - mv_face_detection_get_result_open and mv_face_detection_get_label_open + // - called to obtain results from the thread loop. MEDIA_VISION_SUPPORT_CHECK(_mv_inference_check_system_info_feature_supported()); MEDIA_VISION_INSTANCE_CHECK(handle); @@ -329,6 +334,35 @@ int mv_face_detection_inference(mv_face_detection_h handle, mv_source_h source) return MEDIA_VISION_ERROR_NONE; } +int mv_face_detection_inference_async(mv_face_detection_h handle, mv_source_h source, mv_completion_cb completion_cb, + void *user_data) +{ + LOGD("ENTER"); + + lock_guard lock(g_face_detection_mutex); + + if (!handle) { + LOGE("Handle is NULL."); + return MEDIA_VISION_ERROR_INVALID_PARAMETER; + } + + try { + auto context = static_cast(handle); + auto task = static_cast(context->__tasks.at("face_detection")); + + ObjectDetectionInput input = { handle, source, completion_cb, user_data }; + + task->performAsync(input); + } catch (const BaseException &e) { + LOGE("%s", e.what()); + return e.getError(); + } + + LOGD("LEAVE"); + + return MEDIA_VISION_ERROR_NONE; +} + int mv_face_detection_get_result(mv_face_detection_h handle, unsigned int *number_of_objects, const unsigned int **indices, const float **confidences, const int **left, const int **top, const int **right, const int **bottom) diff --git a/test/testsuites/machine_learning/object_detection/test_object_detection_async.cpp b/test/testsuites/machine_learning/object_detection/test_object_detection_async.cpp index e5f0536..65cca28 100644 --- a/test/testsuites/machine_learning/object_detection/test_object_detection_async.cpp +++ b/test/testsuites/machine_learning/object_detection/test_object_detection_async.cpp @@ -22,8 +22,10 @@ #include "ImageHelper.h" #include "mv_object_detection_internal.h" +#include "mv_face_detection_internal.h" #define IMG_DOG TEST_RES_PATH "/res/inference/images/dog2.jpg" +#define IMG_FACE TEST_RES_PATH "/res/inference/images/faceDetection.jpg" #define MAX_INFERENCE_ITERATION 20 using namespace testing; @@ -40,7 +42,7 @@ struct model_info { mv_source_h source; }; -void completion_callback(mv_object_detection_h handle, void *user_data) +void object_detection_completion_callback(mv_object_detection_h handle, void *user_data) { unsigned int number_of_objects; const int *left, *top, *right, *bottom; @@ -50,7 +52,7 @@ void completion_callback(mv_object_detection_h handle, void *user_data) int ret = mv_object_detection_get_result(handle, &number_of_objects, &indices, &confidences, &left, &top, &right, &bottom); - ASSERT_EQ(ret, 0); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); for (unsigned int idx = 0; idx < number_of_objects; ++idx) { cout << "index = " << indices[idx] << " probability = " << confidences[idx] << " " << left[idx] << " x " @@ -61,14 +63,14 @@ void completion_callback(mv_object_detection_h handle, void *user_data) const char *label; ret = mv_object_detection_get_label(handle, indices[idx], &label); - ASSERT_EQ(ret, 0); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); cout << "index = " << indices[idx] << " label = " << label << endl; string label_str(label); transform(label_str.begin(), label_str.end(), label_str.begin(), ::toupper); - ASSERT_TRUE(label_str == test_model->answer); + ASSERT_EQ(label_str, test_model->answer); } } @@ -108,9 +110,9 @@ TEST(ObjectDetectionAsyncTest, InferenceShouldBeOk) model.source = mv_source; - ret = mv_object_detection_inference_async(handle, mv_source, completion_callback, + ret = mv_object_detection_inference_async(handle, mv_source, object_detection_completion_callback, reinterpret_cast(&model)); - ASSERT_EQ(ret, 0); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); ret = mv_destroy_source(mv_source); ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); @@ -119,4 +121,83 @@ TEST(ObjectDetectionAsyncTest, InferenceShouldBeOk) ret = mv_object_detection_destroy(handle); ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); } +} + +void face_detection_completion_callback(mv_object_detection_h handle, void *user_data) +{ + unsigned int number_of_objects; + const int *left, *top, *right, *bottom; + const unsigned int *indices; + const float *confidences; + model_info *test_model(static_cast(user_data)); + + int ret = mv_face_detection_get_result(handle, &number_of_objects, &indices, &confidences, &left, &top, &right, + &bottom); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + for (unsigned int idx = 0; idx < number_of_objects; ++idx) { + cout << "index = " << indices[idx] << " probability = " << confidences[idx] << " " << left[idx] << " x " + << top[idx] << " ~ " << right[idx] << " x " << bottom[idx] << endl; + } + + for (unsigned int idx = 0; idx < number_of_objects; ++idx) { + const char *label; + + ret = mv_face_detection_get_label(handle, indices[idx], &label); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + cout << "index = " << indices[idx] << " label = " << label << endl; + + string label_str(label); + + transform(label_str.begin(), label_str.end(), label_str.begin(), ::toupper); + + ASSERT_EQ(label_str, test_model->answer); + } +} + +TEST(FaceDetectionAsyncTest, InferenceShouldBeOk) +{ + mv_object_detection_h handle; + vector test_models { + { "", "", "", "", "FACE" } // If empty then default model will be used. + // TODO. + }; + + for (auto &model : test_models) { + int ret = mv_face_detection_create(&handle); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + cout << "model name : " << model.model_file << endl; + + mv_face_detection_set_model(handle, model.model_name.c_str(), model.model_file.c_str(), model.meta_file.c_str(), + model.label_file.c_str()); + mv_face_detection_set_engine(handle, "tflite", "cpu"); + + ret = mv_face_detection_configure(handle); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + ret = mv_face_detection_prepare(handle); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + for (unsigned int iter = 0; iter < MAX_INFERENCE_ITERATION; ++iter) { + mv_source_h mv_source = NULL; + ret = mv_create_source(&mv_source); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + ret = ImageHelper::loadImageToSource(IMG_FACE, mv_source); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + model.source = mv_source; + + ret = mv_face_detection_inference_async(handle, mv_source, face_detection_completion_callback, + reinterpret_cast(&model)); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + ret = mv_destroy_source(mv_source); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + } + + ret = mv_face_detection_destroy(handle); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + } } \ No newline at end of file -- 2.7.4