mv_machine_learning: face recognition: consider multiple threads 20/286220/4
authorInki Dae <inki.dae@samsung.com>
Fri, 30 Dec 2022 02:21:50 +0000 (11:21 +0900)
committerInki Dae <inki.dae@samsung.com>
Tue, 3 Jan 2023 04:56:29 +0000 (13:56 +0900)
[Issue type] : new feature

Considered multiple threads to face recognigion task API.

In application, it can request an inference and register even unregister
by each thread context. So make sure to guarantee such requests using mutex.

And also this patch introduces test_face_recognition_multi_threads app
which adds two test cases, InferenceAndRegisterShouldBeOk
and InferenceAndRegisterAndUnregisterShouldBeOk, to test multiple threads.

Change-Id: I9217a572f6746591eeda774f113e9f2b23c82a8a
Signed-off-by: Inki Dae <inki.dae@samsung.com>
mv_machine_learning/common/include/context.h
mv_machine_learning/face_recognition/include/face_recognition.h
mv_machine_learning/face_recognition/src/face_recognition.cpp
mv_machine_learning/face_recognition/src/mv_face_recognition_open.cpp
packaging/capi-media-vision.spec
test/testsuites/machine_learning/face_recognition/CMakeLists.txt
test/testsuites/machine_learning/face_recognition/test_face_recognition.cpp
test/testsuites/machine_learning/face_recognition/test_face_recognition_multi_threads.cpp [new file with mode: 0644]

index 847bbdf22e02667fa95672432aaf1e44b1fed66c..3f9027c3c123b84c71ad98fccf8c3ce7cbf2a807 100644 (file)
@@ -17,8 +17,9 @@
 #ifndef __CONTEXT_H__
 #define __CONTEXT_H__
 
-#include "itask.h"
 #include <map>
+#include <mutex>
+#include "itask.h"
 
 namespace mediavision
 {
@@ -33,6 +34,7 @@ public:
        {}
 
        std::map<std::string, void *> __tasks;
+       std::mutex _mutex;
 };
 } // namespace
 } // namespace
index e181c44134e05655d2f8cbcc99b487574f454671..f927031c21e152b98c9a52e29ef6423ba3824927 100644 (file)
@@ -71,6 +71,7 @@ struct mv_face_recognition_result_s {
        unsigned int label_idx; /**< label index of label file. */
        std::vector<float> raw_data; /**< raw data to each label. */
        std::string label; /**< label string. */
+       bool is_valid; /**< inference result is valid or not. */
 };
 
 struct FaceRecognitionConfig {
index e4e3ea3879cc83606f192ae3d84344945f104213..5ec22451ccf4349e196bdba78b9cd309d836c957 100644 (file)
@@ -311,6 +311,7 @@ int FaceRecognition::GetAnswer()
                }
 
                _result.label_idx = answer_idx;
+               _result.is_valid = true;
        } catch (const BaseException &e) {
                LOGE("%s", e.what());
                return e.getError();
@@ -525,7 +526,7 @@ int FaceRecognition::DeleteLabel(string label_name)
 int FaceRecognition::GetLabel(const char **out_label)
 {
        if (_status != INFERENCED) {
-               LOGE("Inference not completed yet.");
+               LOGE("Inference not completed yet. (%d)", _status);
                return MEDIA_VISION_ERROR_INVALID_OPERATION;
        }
 
@@ -543,8 +544,10 @@ int FaceRecognition::GetLabel(const char **out_label)
 
 mv_face_recognition_result_s &FaceRecognition::GetResult()
 {
-       if (_status != INFERENCED)
-               throw InvalidOperation("Inference not completed yet.");
+       if (!_result.is_valid)
+               throw NoData("Inference result not ready yet.");
+
+       ImportLabel();
 
        if (!_label_manager)
                throw NoData("Label file doesn't exist.");
index 40249a7409513bc2212979e40e8e51a85797f42f..1a9d702806a003a49fac3688a642163344fe0085 100644 (file)
@@ -98,6 +98,8 @@ int mv_face_recognition_destroy_open(mv_face_recognition_h handle)
        Context *context = static_cast<Context *>(handle);
        map<string, void *>::iterator iter;
 
+       std::lock_guard<std::mutex> lock(context->_mutex);
+
        for (iter = context->__tasks.begin(); iter != context->__tasks.end(); ++iter) {
                if (iter->first.compare("face_recognition") == 0) {
                        auto face_recognition_task = static_cast<FaceRecognitionTask *>(iter->second);
@@ -131,6 +133,8 @@ int mv_face_recognition_prepare_open(mv_face_recognition_h handle)
                auto face_recognition_task = static_cast<FaceRecognitionTask *>(context->__tasks["face_recognition"]);
                auto facenet_task = static_cast<FacenetTask *>(context->__tasks["facenet"]);
 
+               std::lock_guard<std::mutex> lock(context->_mutex);
+
                face_recognition_task->configure();
                facenet_task->configure();
                face_recognition_task->prepare();
@@ -159,6 +163,8 @@ int mv_face_recognition_register_open(mv_face_recognition_h handle, mv_source_h
                auto face_recognition_task = static_cast<FaceRecognitionTask *>(context->__tasks["face_recognition"]);
                auto facenet_task = static_cast<FacenetTask *>(context->__tasks["facenet"]);
 
+               std::lock_guard<std::mutex> lock(context->_mutex);
+
                facenet_input_s facenet_input = { { source } };
 
                facenet_task->setInput(facenet_input);
@@ -195,6 +201,8 @@ int mv_face_recognition_unregister_open(mv_face_recognition_h handle, const char
                Context *context = static_cast<Context *>(handle);
                auto face_recognition_task = static_cast<FaceRecognitionTask *>(context->__tasks["face_recognition"]);
 
+               std::lock_guard<std::mutex> lock(context->_mutex);
+
                mv_face_recognition_input_s input = { mode::DELETE };
 
                input.labels.clear();
@@ -225,6 +233,8 @@ int mv_face_recognition_inference_open(mv_face_recognition_h handle, mv_source_h
                auto face_recognition_task = static_cast<FaceRecognitionTask *>(context->__tasks["face_recognition"]);
                auto facenet_task = static_cast<FacenetTask *>(context->__tasks["facenet"]);
 
+               std::lock_guard<std::mutex> lock(context->_mutex);
+
                facenet_input_s facenet_input = { { source } };
 
                facenet_task->setInput(facenet_input);
@@ -259,6 +269,8 @@ int mv_face_recognition_get_label_open(mv_face_recognition_h handle, const char
                Context *context = static_cast<Context *>(handle);
                auto face_recognition_task = static_cast<FaceRecognitionTask *>(context->__tasks["face_recognition"]);
 
+               std::lock_guard<std::mutex> lock(context->_mutex);
+
                *out_label = face_recognition_task->getOutput().label.c_str();
        } catch (const BaseException &e) {
                LOGE("%s", e.what());
index ef22857b19b6b43ab0f59b07f4bb63d995e7c69e..d52012ea3926a57bcb169fa46229868ad2b37262 100644 (file)
@@ -448,6 +448,7 @@ find . -name '*.gcno' -exec cp --parents '{}' "$gcno_obj_dir" ';'
 %{_bindir}/test_object_detection_3d
 %if "%{enable_ml_face_recognition}" == "1"
 %{_bindir}/test_face_recognition
+%{_bindir}/test_face_recognition_multi_threads
 %{_bindir}/measure_face_recognition
 %endif
 %{_bindir}/tizen-unittests/%{name}/run-unittest.sh
index a7e87d21e76c7431b8c2901f9f859cfc3d552f8c..262c8516a52d756bf3843f53e4a9d2753850123e 100644 (file)
@@ -2,19 +2,26 @@ project(mv_face_recognition_test_suite)
 cmake_minimum_required(VERSION 2.6...3.13)
 
 set(TEST_FACE_RECOGNITION test_face_recognition)
+set(TEST_FACE_RECOGNITION_MULTI_THREADS test_face_recognition_multi_threads)
 set(MEASURE_ACCURACY measure_face_recognition)
 
 add_executable(${TEST_FACE_RECOGNITION} face_recognition_test_util.cpp test_face_recognition.cpp)
+add_executable(${TEST_FACE_RECOGNITION_MULTI_THREADS} face_recognition_test_util.cpp test_face_recognition_multi_threads.cpp)
 add_executable(${MEASURE_ACCURACY} face_recognition_test_util.cpp measure_face_recognition.cpp)
 
 target_link_libraries(${TEST_FACE_RECOGNITION} gtest gtest_main
                                                                          mv_face_recognition
                                       mv_image_helper
 )
- target_link_libraries(${MEASURE_ACCURACY} gtest gtest_main
+target_link_libraries(${TEST_FACE_RECOGNITION_MULTI_THREADS} gtest gtest_main pthread
+                                                                         mv_face_recognition
+                                      mv_image_helper
+)
+target_link_libraries(${MEASURE_ACCURACY} gtest gtest_main
                                                                          mv_face_recognition
                                       mv_image_helper
 )
 
 install(TARGETS ${TEST_FACE_RECOGNITION} DESTINATION ${CMAKE_INSTALL_BINDIR})
+install(TARGETS ${TEST_FACE_RECOGNITION_MULTI_THREADS} DESTINATION ${CMAKE_INSTALL_BINDIR})
 install(TARGETS ${MEASURE_ACCURACY} DESTINATION ${CMAKE_INSTALL_BINDIR})
index f5f004b3ca3aca6ba0396651d5a7737c0239cc80..5ec4159d80cb8a76882ecc653d9cfe2193afcd0f 100644 (file)
@@ -100,9 +100,8 @@ TEST(FaceRecognitionTest, InferenceAfterTrainingShouldBeOk)
                ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
 
                ret = mv_face_recognition_inference(handle, mv_source);
-               if (ret != MEDIA_VISION_ERROR_NO_DATA) {
+               if (ret != MEDIA_VISION_ERROR_NO_DATA)
                        ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
-               }
 
                ret = mv_destroy_source(mv_source);
                ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
@@ -110,6 +109,12 @@ TEST(FaceRecognitionTest, InferenceAfterTrainingShouldBeOk)
                const char *out_label = NULL;
 
                ret = mv_face_recognition_get_label(handle, &out_label);
+               if (ret == MEDIA_VISION_ERROR_NO_DATA) {
+                       image_idx++;
+                       mv_destroy_source(mv_source);
+                       continue;
+               }
+
                ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
 
                string label_str(out_label);
@@ -147,7 +152,7 @@ TEST(FaceRecognitionTest, GetLabelWithoutInferenceShouldBeError)
                ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
 
                ret = mv_face_recognition_register(handle, mv_source, image.second.c_str());
-               ASSERT_EQ(ret, 0);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
 
                ret = mv_destroy_source(mv_source);
                ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
@@ -156,7 +161,7 @@ TEST(FaceRecognitionTest, GetLabelWithoutInferenceShouldBeError)
        const char *out_label = NULL;
 
        ret = mv_face_recognition_get_label(handle, &out_label);
-       ASSERT_EQ(ret, MEDIA_VISION_ERROR_INVALID_OPERATION);
+       ASSERT_EQ(ret, MEDIA_VISION_ERROR_NO_DATA);
 
        ret = mv_face_recognition_destroy(handle);
        ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
@@ -273,6 +278,11 @@ TEST(FaceRecognitionTest, LabelRemovalShouldBeOk)
                        const char *out_label = NULL;
 
                        ret = mv_face_recognition_get_label(handle, &out_label);
+                       if (ret == MEDIA_VISION_ERROR_NO_DATA) {
+                               is_no_data = true;
+                               ret = MEDIA_VISION_ERROR_NONE;
+                       }
+
                        ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
 
                        if (is_no_data)
diff --git a/test/testsuites/machine_learning/face_recognition/test_face_recognition_multi_threads.cpp b/test/testsuites/machine_learning/face_recognition/test_face_recognition_multi_threads.cpp
new file mode 100644 (file)
index 0000000..8f4d1ef
--- /dev/null
@@ -0,0 +1,196 @@
+/**
+ * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <unistd.h>
+#include <iostream>
+#include <string.h>
+#include <map>
+#include <thread>
+
+#include "gtest/gtest.h"
+
+#include "ImageHelper.h"
+#include "mv_face_recognition.h"
+#include "face_recognition_test_util.h"
+
+#define TRAINING_IMAGE_PATH "/home/owner/media/res/face_recognition/res/test/training/"
+#define TEST_IMAGE_PATH "/home/owner/media/res/face_recognition/res/test/test/"
+#define MAX_DATA_SET 15
+
+using namespace testing;
+using namespace std;
+
+static const map<string, string> training_images = {
+       { "037830.png", "2929" }, { "038965.png", "2929" }, { "045978.png", "2929" }, { "050501.png", "2929" },
+       { "065899.png", "2929" }, { "010348.png", "7779" }, { "029342.png", "7779" }, { "035939.png", "7779" },
+       { "061310.png", "7779" }, { "062261.png", "7779" }, { "000928.png", "3448" }, { "008922.png", "3448" },
+       { "029633.png", "3448" }, { "032962.png", "3448" }, { "054616.png", "3448" }
+};
+
+static const map<string, string> test_images = {
+       { "068468.png", "2929" }, { "068883.png", "2929" }, { "075004.png", "2929" }, { "078125.png", "2929" },
+       { "080649.png", "2929" }, { "074645.png", "7779" }, { "086536.png", "7779" }, { "089334.png", "7779" },
+       { "096514.png", "7779" }, { "100336.png", "7779" }, { "054757.png", "3448" }, { "064838.png", "3448" },
+       { "072749.png", "3448" }, { "073526.png", "3448" }, { "080451.png", "3448" }
+};
+
+using namespace MediaVision::Common;
+
+void Register(mv_face_recognition_h handle)
+{
+       for (auto &image : training_images) {
+               const string image_path = string(TRAINING_IMAGE_PATH) + image.first;
+               mv_source_h mv_source = NULL;
+
+               int ret = mv_create_source(&mv_source);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+               ret = ImageHelper::loadImageToSource(image_path.c_str(), mv_source);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+               ret = mv_face_recognition_register(handle, mv_source, image.second.c_str());
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+               ret = mv_destroy_source(mv_source);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+       }
+}
+
+void Recognize(mv_face_recognition_h handle)
+{
+       vector<string> answers = { "3448", "3448", "2929", "2929", "3448", "3448", "7779", "2929",
+                                                          "2929", "3448", "2929", "7779", "7779", "7779", "7779" };
+
+       unsigned int image_idx = 0;
+       unsigned int correct_cnt = 0;
+
+       for (auto &image : test_images) {
+               const string image_path = string(TEST_IMAGE_PATH) + image.first;
+               mv_source_h mv_source = NULL;
+
+               int ret = mv_create_source(&mv_source);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+               ret = ImageHelper::loadImageToSource(image_path.c_str(), mv_source);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+               ret = mv_face_recognition_inference(handle, mv_source);
+               if (ret == MEDIA_VISION_ERROR_NO_DATA) {
+                       mv_destroy_source(mv_source);
+                       image_idx++;
+                       continue;
+               }
+
+               ret = mv_destroy_source(mv_source);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+               const char *out_label = NULL;
+
+               ret = mv_face_recognition_get_label(handle, &out_label);
+               if (ret != MEDIA_VISION_ERROR_NONE) {
+                       image_idx++;
+                       continue;
+               }
+
+               string label_str(out_label);
+
+               if (answers[image_idx++] == label_str)
+                       correct_cnt++;
+       }
+
+       cout << "Correct/Total = " << correct_cnt << " / " << image_idx << endl;
+}
+
+void Unregister(mv_face_recognition_h handle)
+{
+       vector<string> labels = { "3448", "2929", "7779" };
+
+       for (auto &image : training_images) {
+               const string image_path = string(TRAINING_IMAGE_PATH) + image.first;
+               mv_source_h mv_source = NULL;
+               int ret = mv_create_source(&mv_source);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+               ret = ImageHelper::loadImageToSource(image_path.c_str(), mv_source);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+               ret = mv_face_recognition_register(handle, mv_source, image.second.c_str());
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+               ret = mv_destroy_source(mv_source);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+       }
+
+       for (auto &label : labels) {
+               int ret = mv_face_recognition_unregister(handle, label.c_str());
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+       }
+}
+
+TEST(FaceRecognitionMultithreadTest, RegisterAndRecognizeShouldBeOk)
+{
+       mv_face_recognition_h handle;
+
+       int ret = mv_face_recognition_create(&handle);
+       ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+       ret = mv_face_recognition_prepare(handle);
+       ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+       std::thread register_thread([&] { Register(handle); });
+
+       std::thread recognize_thread([&] { Recognize(handle); });
+
+       register_thread.join();
+       recognize_thread.join();
+
+       ret = mv_face_recognition_destroy(handle);
+       ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+       RemoveModelResources();
+}
+
+TEST(FaceRecognitionMultithreadTest, RegisterAndRecognizeAndUnregisterShouldBeOk)
+{
+       mv_face_recognition_h handle;
+
+       int ret = mv_face_recognition_create(&handle);
+       ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+       ret = mv_face_recognition_prepare(handle);
+       ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+       std::thread register_thread([&] { Register(handle); });
+
+       std::thread recognize_thread([&] { Recognize(handle); });
+
+       std::thread unregister_thread([&] { Unregister(handle); });
+
+       register_thread.join();
+       recognize_thread.join();
+       unregister_thread.join();
+
+       ret = mv_face_recognition_destroy(handle);
+       ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+       RemoveModelResources();
+}
+
+int main(int argc, char **argv)
+{
+       InitGoogleTest(&argc, argv);
+       return RUN_ALL_TESTS();
+}