From: Inki Dae Date: Fri, 30 Dec 2022 02:21:50 +0000 (+0900) Subject: mv_machine_learning: face recognition: consider multiple threads X-Git-Tag: accepted/tizen/unified/20230113.091231~4^2 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=4316ca5d4ba4f99badba3aa6a02b14cc5940dcb5;p=platform%2Fcore%2Fapi%2Fmediavision.git mv_machine_learning: face recognition: consider multiple threads [Issue type] : new feature Considered multiple threads to face recognigion task API. In application, it can request an inference and register even unregister by each thread context. So make sure to guarantee such requests using mutex. And also this patch introduces test_face_recognition_multi_threads app which adds two test cases, InferenceAndRegisterShouldBeOk and InferenceAndRegisterAndUnregisterShouldBeOk, to test multiple threads. Change-Id: I9217a572f6746591eeda774f113e9f2b23c82a8a Signed-off-by: Inki Dae --- diff --git a/mv_machine_learning/common/include/context.h b/mv_machine_learning/common/include/context.h index 847bbdf2..3f9027c3 100644 --- a/mv_machine_learning/common/include/context.h +++ b/mv_machine_learning/common/include/context.h @@ -17,8 +17,9 @@ #ifndef __CONTEXT_H__ #define __CONTEXT_H__ -#include "itask.h" #include +#include +#include "itask.h" namespace mediavision { @@ -33,6 +34,7 @@ public: {} std::map __tasks; + std::mutex _mutex; }; } // namespace } // namespace diff --git a/mv_machine_learning/face_recognition/include/face_recognition.h b/mv_machine_learning/face_recognition/include/face_recognition.h index e181c441..f927031c 100644 --- a/mv_machine_learning/face_recognition/include/face_recognition.h +++ b/mv_machine_learning/face_recognition/include/face_recognition.h @@ -71,6 +71,7 @@ struct mv_face_recognition_result_s { unsigned int label_idx; /**< label index of label file. */ std::vector raw_data; /**< raw data to each label. */ std::string label; /**< label string. */ + bool is_valid; /**< inference result is valid or not. */ }; struct FaceRecognitionConfig { diff --git a/mv_machine_learning/face_recognition/src/face_recognition.cpp b/mv_machine_learning/face_recognition/src/face_recognition.cpp index e4e3ea38..5ec22451 100644 --- a/mv_machine_learning/face_recognition/src/face_recognition.cpp +++ b/mv_machine_learning/face_recognition/src/face_recognition.cpp @@ -311,6 +311,7 @@ int FaceRecognition::GetAnswer() } _result.label_idx = answer_idx; + _result.is_valid = true; } catch (const BaseException &e) { LOGE("%s", e.what()); return e.getError(); @@ -525,7 +526,7 @@ int FaceRecognition::DeleteLabel(string label_name) int FaceRecognition::GetLabel(const char **out_label) { if (_status != INFERENCED) { - LOGE("Inference not completed yet."); + LOGE("Inference not completed yet. (%d)", _status); return MEDIA_VISION_ERROR_INVALID_OPERATION; } @@ -543,8 +544,10 @@ int FaceRecognition::GetLabel(const char **out_label) mv_face_recognition_result_s &FaceRecognition::GetResult() { - if (_status != INFERENCED) - throw InvalidOperation("Inference not completed yet."); + if (!_result.is_valid) + throw NoData("Inference result not ready yet."); + + ImportLabel(); if (!_label_manager) throw NoData("Label file doesn't exist."); diff --git a/mv_machine_learning/face_recognition/src/mv_face_recognition_open.cpp b/mv_machine_learning/face_recognition/src/mv_face_recognition_open.cpp index 40249a74..1a9d7028 100644 --- a/mv_machine_learning/face_recognition/src/mv_face_recognition_open.cpp +++ b/mv_machine_learning/face_recognition/src/mv_face_recognition_open.cpp @@ -98,6 +98,8 @@ int mv_face_recognition_destroy_open(mv_face_recognition_h handle) Context *context = static_cast(handle); map::iterator iter; + std::lock_guard lock(context->_mutex); + for (iter = context->__tasks.begin(); iter != context->__tasks.end(); ++iter) { if (iter->first.compare("face_recognition") == 0) { auto face_recognition_task = static_cast(iter->second); @@ -131,6 +133,8 @@ int mv_face_recognition_prepare_open(mv_face_recognition_h handle) auto face_recognition_task = static_cast(context->__tasks["face_recognition"]); auto facenet_task = static_cast(context->__tasks["facenet"]); + std::lock_guard lock(context->_mutex); + face_recognition_task->configure(); facenet_task->configure(); face_recognition_task->prepare(); @@ -159,6 +163,8 @@ int mv_face_recognition_register_open(mv_face_recognition_h handle, mv_source_h auto face_recognition_task = static_cast(context->__tasks["face_recognition"]); auto facenet_task = static_cast(context->__tasks["facenet"]); + std::lock_guard lock(context->_mutex); + facenet_input_s facenet_input = { { source } }; facenet_task->setInput(facenet_input); @@ -195,6 +201,8 @@ int mv_face_recognition_unregister_open(mv_face_recognition_h handle, const char Context *context = static_cast(handle); auto face_recognition_task = static_cast(context->__tasks["face_recognition"]); + std::lock_guard lock(context->_mutex); + mv_face_recognition_input_s input = { mode::DELETE }; input.labels.clear(); @@ -225,6 +233,8 @@ int mv_face_recognition_inference_open(mv_face_recognition_h handle, mv_source_h auto face_recognition_task = static_cast(context->__tasks["face_recognition"]); auto facenet_task = static_cast(context->__tasks["facenet"]); + std::lock_guard lock(context->_mutex); + facenet_input_s facenet_input = { { source } }; facenet_task->setInput(facenet_input); @@ -259,6 +269,8 @@ int mv_face_recognition_get_label_open(mv_face_recognition_h handle, const char Context *context = static_cast(handle); auto face_recognition_task = static_cast(context->__tasks["face_recognition"]); + std::lock_guard lock(context->_mutex); + *out_label = face_recognition_task->getOutput().label.c_str(); } catch (const BaseException &e) { LOGE("%s", e.what()); diff --git a/packaging/capi-media-vision.spec b/packaging/capi-media-vision.spec index ef22857b..d52012ea 100644 --- a/packaging/capi-media-vision.spec +++ b/packaging/capi-media-vision.spec @@ -448,6 +448,7 @@ find . -name '*.gcno' -exec cp --parents '{}' "$gcno_obj_dir" ';' %{_bindir}/test_object_detection_3d %if "%{enable_ml_face_recognition}" == "1" %{_bindir}/test_face_recognition +%{_bindir}/test_face_recognition_multi_threads %{_bindir}/measure_face_recognition %endif %{_bindir}/tizen-unittests/%{name}/run-unittest.sh diff --git a/test/testsuites/machine_learning/face_recognition/CMakeLists.txt b/test/testsuites/machine_learning/face_recognition/CMakeLists.txt index a7e87d21..262c8516 100644 --- a/test/testsuites/machine_learning/face_recognition/CMakeLists.txt +++ b/test/testsuites/machine_learning/face_recognition/CMakeLists.txt @@ -2,19 +2,26 @@ project(mv_face_recognition_test_suite) cmake_minimum_required(VERSION 2.6...3.13) set(TEST_FACE_RECOGNITION test_face_recognition) +set(TEST_FACE_RECOGNITION_MULTI_THREADS test_face_recognition_multi_threads) set(MEASURE_ACCURACY measure_face_recognition) add_executable(${TEST_FACE_RECOGNITION} face_recognition_test_util.cpp test_face_recognition.cpp) +add_executable(${TEST_FACE_RECOGNITION_MULTI_THREADS} face_recognition_test_util.cpp test_face_recognition_multi_threads.cpp) add_executable(${MEASURE_ACCURACY} face_recognition_test_util.cpp measure_face_recognition.cpp) target_link_libraries(${TEST_FACE_RECOGNITION} gtest gtest_main mv_face_recognition mv_image_helper ) - target_link_libraries(${MEASURE_ACCURACY} gtest gtest_main +target_link_libraries(${TEST_FACE_RECOGNITION_MULTI_THREADS} gtest gtest_main pthread + mv_face_recognition + mv_image_helper +) +target_link_libraries(${MEASURE_ACCURACY} gtest gtest_main mv_face_recognition mv_image_helper ) install(TARGETS ${TEST_FACE_RECOGNITION} DESTINATION ${CMAKE_INSTALL_BINDIR}) +install(TARGETS ${TEST_FACE_RECOGNITION_MULTI_THREADS} DESTINATION ${CMAKE_INSTALL_BINDIR}) install(TARGETS ${MEASURE_ACCURACY} DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/test/testsuites/machine_learning/face_recognition/test_face_recognition.cpp b/test/testsuites/machine_learning/face_recognition/test_face_recognition.cpp index f5f004b3..5ec4159d 100644 --- a/test/testsuites/machine_learning/face_recognition/test_face_recognition.cpp +++ b/test/testsuites/machine_learning/face_recognition/test_face_recognition.cpp @@ -100,9 +100,8 @@ TEST(FaceRecognitionTest, InferenceAfterTrainingShouldBeOk) ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); ret = mv_face_recognition_inference(handle, mv_source); - if (ret != MEDIA_VISION_ERROR_NO_DATA) { + if (ret != MEDIA_VISION_ERROR_NO_DATA) ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); - } ret = mv_destroy_source(mv_source); ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); @@ -110,6 +109,12 @@ TEST(FaceRecognitionTest, InferenceAfterTrainingShouldBeOk) const char *out_label = NULL; ret = mv_face_recognition_get_label(handle, &out_label); + if (ret == MEDIA_VISION_ERROR_NO_DATA) { + image_idx++; + mv_destroy_source(mv_source); + continue; + } + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); string label_str(out_label); @@ -147,7 +152,7 @@ TEST(FaceRecognitionTest, GetLabelWithoutInferenceShouldBeError) ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); ret = mv_face_recognition_register(handle, mv_source, image.second.c_str()); - ASSERT_EQ(ret, 0); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); ret = mv_destroy_source(mv_source); ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); @@ -156,7 +161,7 @@ TEST(FaceRecognitionTest, GetLabelWithoutInferenceShouldBeError) const char *out_label = NULL; ret = mv_face_recognition_get_label(handle, &out_label); - ASSERT_EQ(ret, MEDIA_VISION_ERROR_INVALID_OPERATION); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NO_DATA); ret = mv_face_recognition_destroy(handle); ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); @@ -273,6 +278,11 @@ TEST(FaceRecognitionTest, LabelRemovalShouldBeOk) const char *out_label = NULL; ret = mv_face_recognition_get_label(handle, &out_label); + if (ret == MEDIA_VISION_ERROR_NO_DATA) { + is_no_data = true; + ret = MEDIA_VISION_ERROR_NONE; + } + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); if (is_no_data) diff --git a/test/testsuites/machine_learning/face_recognition/test_face_recognition_multi_threads.cpp b/test/testsuites/machine_learning/face_recognition/test_face_recognition_multi_threads.cpp new file mode 100644 index 00000000..8f4d1ef1 --- /dev/null +++ b/test/testsuites/machine_learning/face_recognition/test_face_recognition_multi_threads.cpp @@ -0,0 +1,196 @@ +/** + * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" + +#include "ImageHelper.h" +#include "mv_face_recognition.h" +#include "face_recognition_test_util.h" + +#define TRAINING_IMAGE_PATH "/home/owner/media/res/face_recognition/res/test/training/" +#define TEST_IMAGE_PATH "/home/owner/media/res/face_recognition/res/test/test/" +#define MAX_DATA_SET 15 + +using namespace testing; +using namespace std; + +static const map training_images = { + { "037830.png", "2929" }, { "038965.png", "2929" }, { "045978.png", "2929" }, { "050501.png", "2929" }, + { "065899.png", "2929" }, { "010348.png", "7779" }, { "029342.png", "7779" }, { "035939.png", "7779" }, + { "061310.png", "7779" }, { "062261.png", "7779" }, { "000928.png", "3448" }, { "008922.png", "3448" }, + { "029633.png", "3448" }, { "032962.png", "3448" }, { "054616.png", "3448" } +}; + +static const map test_images = { + { "068468.png", "2929" }, { "068883.png", "2929" }, { "075004.png", "2929" }, { "078125.png", "2929" }, + { "080649.png", "2929" }, { "074645.png", "7779" }, { "086536.png", "7779" }, { "089334.png", "7779" }, + { "096514.png", "7779" }, { "100336.png", "7779" }, { "054757.png", "3448" }, { "064838.png", "3448" }, + { "072749.png", "3448" }, { "073526.png", "3448" }, { "080451.png", "3448" } +}; + +using namespace MediaVision::Common; + +void Register(mv_face_recognition_h handle) +{ + for (auto &image : training_images) { + const string image_path = string(TRAINING_IMAGE_PATH) + image.first; + mv_source_h mv_source = NULL; + + int ret = mv_create_source(&mv_source); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + ret = ImageHelper::loadImageToSource(image_path.c_str(), mv_source); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + ret = mv_face_recognition_register(handle, mv_source, image.second.c_str()); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + ret = mv_destroy_source(mv_source); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + } +} + +void Recognize(mv_face_recognition_h handle) +{ + vector answers = { "3448", "3448", "2929", "2929", "3448", "3448", "7779", "2929", + "2929", "3448", "2929", "7779", "7779", "7779", "7779" }; + + unsigned int image_idx = 0; + unsigned int correct_cnt = 0; + + for (auto &image : test_images) { + const string image_path = string(TEST_IMAGE_PATH) + image.first; + mv_source_h mv_source = NULL; + + int ret = mv_create_source(&mv_source); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + ret = ImageHelper::loadImageToSource(image_path.c_str(), mv_source); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + ret = mv_face_recognition_inference(handle, mv_source); + if (ret == MEDIA_VISION_ERROR_NO_DATA) { + mv_destroy_source(mv_source); + image_idx++; + continue; + } + + ret = mv_destroy_source(mv_source); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + const char *out_label = NULL; + + ret = mv_face_recognition_get_label(handle, &out_label); + if (ret != MEDIA_VISION_ERROR_NONE) { + image_idx++; + continue; + } + + string label_str(out_label); + + if (answers[image_idx++] == label_str) + correct_cnt++; + } + + cout << "Correct/Total = " << correct_cnt << " / " << image_idx << endl; +} + +void Unregister(mv_face_recognition_h handle) +{ + vector labels = { "3448", "2929", "7779" }; + + for (auto &image : training_images) { + const string image_path = string(TRAINING_IMAGE_PATH) + image.first; + mv_source_h mv_source = NULL; + int ret = mv_create_source(&mv_source); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + ret = ImageHelper::loadImageToSource(image_path.c_str(), mv_source); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + ret = mv_face_recognition_register(handle, mv_source, image.second.c_str()); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + ret = mv_destroy_source(mv_source); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + } + + for (auto &label : labels) { + int ret = mv_face_recognition_unregister(handle, label.c_str()); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + } +} + +TEST(FaceRecognitionMultithreadTest, RegisterAndRecognizeShouldBeOk) +{ + mv_face_recognition_h handle; + + int ret = mv_face_recognition_create(&handle); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + ret = mv_face_recognition_prepare(handle); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + std::thread register_thread([&] { Register(handle); }); + + std::thread recognize_thread([&] { Recognize(handle); }); + + register_thread.join(); + recognize_thread.join(); + + ret = mv_face_recognition_destroy(handle); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + RemoveModelResources(); +} + +TEST(FaceRecognitionMultithreadTest, RegisterAndRecognizeAndUnregisterShouldBeOk) +{ + mv_face_recognition_h handle; + + int ret = mv_face_recognition_create(&handle); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + ret = mv_face_recognition_prepare(handle); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + std::thread register_thread([&] { Register(handle); }); + + std::thread recognize_thread([&] { Recognize(handle); }); + + std::thread unregister_thread([&] { Unregister(handle); }); + + register_thread.join(); + recognize_thread.join(); + unregister_thread.join(); + + ret = mv_face_recognition_destroy(handle); + ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE); + + RemoveModelResources(); +} + +int main(int argc, char **argv) +{ + InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +}