mv_machine_learning: face recognition: consider multiple threads

[Issue type] : new feature

Considered multiple threads to face recognigion task API.

In application, it can request an inference and register even unregister
by each thread context. So make sure to guarantee such requests using mutex.

And also this patch introduces test_face_recognition_multi_threads app
which adds two test cases, InferenceAndRegisterShouldBeOk
and InferenceAndRegisterAndUnregisterShouldBeOk, to test multiple threads.

Change-Id: I9217a572f6746591eeda774f113e9f2b23c82a8a
Signed-off-by: Inki Dae <inki.dae@samsung.com>

diff --git a/mv_machine_learning/common/include/context.h b/mv_machine_learning/common/include/context.h
index 847bbdf..3f9027c 100644 (file)
--- a/mv_machine_learning/common/include/context.h
+++ b/mv_machine_learning/common/include/context.h
@@ -17,8 +17,9 @@
 #ifndef __CONTEXT_H__
 #define __CONTEXT_H__
 
-#include "itask.h"
 #include <map>
+#include <mutex>
+#include "itask.h"
 
 namespace mediavision
 {
@@ -33,6 +34,7 @@ public:
        {}
 
        std::map<std::string, void *> __tasks;
+       std::mutex _mutex;
 };
 } // namespace
 } // namespace

diff --git a/mv_machine_learning/face_recognition/include/face_recognition.h b/mv_machine_learning/face_recognition/include/face_recognition.h
index e181c44..f927031 100644 (file)
--- a/mv_machine_learning/face_recognition/include/face_recognition.h
+++ b/mv_machine_learning/face_recognition/include/face_recognition.h
@@ -71,6 +71,7 @@ struct mv_face_recognition_result_s {
        unsigned int label_idx; /**< label index of label file. */
        std::vector<float> raw_data; /**< raw data to each label. */
        std::string label; /**< label string. */
+       bool is_valid; /**< inference result is valid or not. */
 };
 
 struct FaceRecognitionConfig {

diff --git a/mv_machine_learning/face_recognition/src/face_recognition.cpp b/mv_machine_learning/face_recognition/src/face_recognition.cpp
index e4e3ea3..5ec2245 100644 (file)
--- a/mv_machine_learning/face_recognition/src/face_recognition.cpp
+++ b/mv_machine_learning/face_recognition/src/face_recognition.cpp
@@ -311,6 +311,7 @@ int FaceRecognition::GetAnswer()
                }
 
                _result.label_idx = answer_idx;
+               _result.is_valid = true;
        } catch (const BaseException &e) {
                LOGE("%s", e.what());
                return e.getError();
@@ -525,7 +526,7 @@ int FaceRecognition::DeleteLabel(string label_name)
 int FaceRecognition::GetLabel(const char **out_label)
 {
        if (_status != INFERENCED) {
-               LOGE("Inference not completed yet.");
+               LOGE("Inference not completed yet. (%d)", _status);
                return MEDIA_VISION_ERROR_INVALID_OPERATION;
        }
 
@@ -543,8 +544,10 @@ int FaceRecognition::GetLabel(const char **out_label)
 
 mv_face_recognition_result_s &FaceRecognition::GetResult()
 {
-       if (_status != INFERENCED)
-               throw InvalidOperation("Inference not completed yet.");
+       if (!_result.is_valid)
+               throw NoData("Inference result not ready yet.");
+
+       ImportLabel();
 
        if (!_label_manager)
                throw NoData("Label file doesn't exist.");

diff --git a/mv_machine_learning/face_recognition/src/mv_face_recognition_open.cpp b/mv_machine_learning/face_recognition/src/mv_face_recognition_open.cpp
index 40249a7..1a9d702 100644 (file)
--- a/mv_machine_learning/face_recognition/src/mv_face_recognition_open.cpp
+++ b/mv_machine_learning/face_recognition/src/mv_face_recognition_open.cpp
@@ -98,6 +98,8 @@ int mv_face_recognition_destroy_open(mv_face_recognition_h handle)
        Context *context = static_cast<Context *>(handle);
        map<string, void *>::iterator iter;
 
+       std::lock_guard<std::mutex> lock(context->_mutex);
+
        for (iter = context->__tasks.begin(); iter != context->__tasks.end(); ++iter) {
                if (iter->first.compare("face_recognition") == 0) {
                        auto face_recognition_task = static_cast<FaceRecognitionTask *>(iter->second);
@@ -131,6 +133,8 @@ int mv_face_recognition_prepare_open(mv_face_recognition_h handle)
                auto face_recognition_task = static_cast<FaceRecognitionTask *>(context->__tasks["face_recognition"]);
                auto facenet_task = static_cast<FacenetTask *>(context->__tasks["facenet"]);
 
+               std::lock_guard<std::mutex> lock(context->_mutex);
+
                face_recognition_task->configure();
                facenet_task->configure();
                face_recognition_task->prepare();
@@ -159,6 +163,8 @@ int mv_face_recognition_register_open(mv_face_recognition_h handle, mv_source_h
                auto face_recognition_task = static_cast<FaceRecognitionTask *>(context->__tasks["face_recognition"]);
                auto facenet_task = static_cast<FacenetTask *>(context->__tasks["facenet"]);
 
+               std::lock_guard<std::mutex> lock(context->_mutex);
+
                facenet_input_s facenet_input = { { source } };
 
                facenet_task->setInput(facenet_input);
@@ -195,6 +201,8 @@ int mv_face_recognition_unregister_open(mv_face_recognition_h handle, const char
                Context *context = static_cast<Context *>(handle);
                auto face_recognition_task = static_cast<FaceRecognitionTask *>(context->__tasks["face_recognition"]);
 
+               std::lock_guard<std::mutex> lock(context->_mutex);
+
                mv_face_recognition_input_s input = { mode::DELETE };
 
                input.labels.clear();
@@ -225,6 +233,8 @@ int mv_face_recognition_inference_open(mv_face_recognition_h handle, mv_source_h
                auto face_recognition_task = static_cast<FaceRecognitionTask *>(context->__tasks["face_recognition"]);
                auto facenet_task = static_cast<FacenetTask *>(context->__tasks["facenet"]);
 
+               std::lock_guard<std::mutex> lock(context->_mutex);
+
                facenet_input_s facenet_input = { { source } };
 
                facenet_task->setInput(facenet_input);
@@ -259,6 +269,8 @@ int mv_face_recognition_get_label_open(mv_face_recognition_h handle, const char
                Context *context = static_cast<Context *>(handle);
                auto face_recognition_task = static_cast<FaceRecognitionTask *>(context->__tasks["face_recognition"]);
 
+               std::lock_guard<std::mutex> lock(context->_mutex);
+
                *out_label = face_recognition_task->getOutput().label.c_str();
        } catch (const BaseException &e) {
                LOGE("%s", e.what());

diff --git a/packaging/capi-media-vision.spec b/packaging/capi-media-vision.spec
index ef22857..d52012e 100644 (file)
--- a/packaging/capi-media-vision.spec
+++ b/packaging/capi-media-vision.spec
@@ -448,6 +448,7 @@ find . -name '*.gcno' -exec cp --parents '{}' "$gcno_obj_dir" ';'
 %{_bindir}/test_object_detection_3d
 %if "%{enable_ml_face_recognition}" == "1"
 %{_bindir}/test_face_recognition
+%{_bindir}/test_face_recognition_multi_threads
 %{_bindir}/measure_face_recognition
 %endif
 %{_bindir}/tizen-unittests/%{name}/run-unittest.sh

diff --git a/test/testsuites/machine_learning/face_recognition/CMakeLists.txt b/test/testsuites/machine_learning/face_recognition/CMakeLists.txt
index a7e87d2..262c851 100644 (file)
--- a/test/testsuites/machine_learning/face_recognition/CMakeLists.txt
+++ b/test/testsuites/machine_learning/face_recognition/CMakeLists.txt
@@ -2,19 +2,26 @@ project(mv_face_recognition_test_suite)
 cmake_minimum_required(VERSION 2.6...3.13)
 
 set(TEST_FACE_RECOGNITION test_face_recognition)
+set(TEST_FACE_RECOGNITION_MULTI_THREADS test_face_recognition_multi_threads)
 set(MEASURE_ACCURACY measure_face_recognition)
 
 add_executable(${TEST_FACE_RECOGNITION} face_recognition_test_util.cpp test_face_recognition.cpp)
+add_executable(${TEST_FACE_RECOGNITION_MULTI_THREADS} face_recognition_test_util.cpp test_face_recognition_multi_threads.cpp)
 add_executable(${MEASURE_ACCURACY} face_recognition_test_util.cpp measure_face_recognition.cpp)
 
 target_link_libraries(${TEST_FACE_RECOGNITION} gtest gtest_main
                                                                          mv_face_recognition
                                       mv_image_helper
 )
- target_link_libraries(${MEASURE_ACCURACY} gtest gtest_main
+target_link_libraries(${TEST_FACE_RECOGNITION_MULTI_THREADS} gtest gtest_main pthread
+                                                                         mv_face_recognition
+                                      mv_image_helper
+)
+target_link_libraries(${MEASURE_ACCURACY} gtest gtest_main
                                                                          mv_face_recognition
                                       mv_image_helper
 )
 
 install(TARGETS ${TEST_FACE_RECOGNITION} DESTINATION ${CMAKE_INSTALL_BINDIR})
+install(TARGETS ${TEST_FACE_RECOGNITION_MULTI_THREADS} DESTINATION ${CMAKE_INSTALL_BINDIR})
 install(TARGETS ${MEASURE_ACCURACY} DESTINATION ${CMAKE_INSTALL_BINDIR})

diff --git a/test/testsuites/machine_learning/face_recognition/test_face_recognition.cpp b/test/testsuites/machine_learning/face_recognition/test_face_recognition.cpp
index f5f004b..5ec4159 100644 (file)
--- a/test/testsuites/machine_learning/face_recognition/test_face_recognition.cpp
+++ b/test/testsuites/machine_learning/face_recognition/test_face_recognition.cpp
@@ -100,9 +100,8 @@ TEST(FaceRecognitionTest, InferenceAfterTrainingShouldBeOk)
                ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
 
                ret = mv_face_recognition_inference(handle, mv_source);
-               if (ret != MEDIA_VISION_ERROR_NO_DATA) {
+               if (ret != MEDIA_VISION_ERROR_NO_DATA)
                        ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
-               }
 
                ret = mv_destroy_source(mv_source);
                ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
@@ -110,6 +109,12 @@ TEST(FaceRecognitionTest, InferenceAfterTrainingShouldBeOk)
                const char *out_label = NULL;
 
                ret = mv_face_recognition_get_label(handle, &out_label);
+               if (ret == MEDIA_VISION_ERROR_NO_DATA) {
+                       image_idx++;
+                       mv_destroy_source(mv_source);
+                       continue;
+               }
+
                ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
 
                string label_str(out_label);
@@ -147,7 +152,7 @@ TEST(FaceRecognitionTest, GetLabelWithoutInferenceShouldBeError)
                ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
 
                ret = mv_face_recognition_register(handle, mv_source, image.second.c_str());
-               ASSERT_EQ(ret, 0);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
 
                ret = mv_destroy_source(mv_source);
                ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
@@ -156,7 +161,7 @@ TEST(FaceRecognitionTest, GetLabelWithoutInferenceShouldBeError)
        const char *out_label = NULL;
 
        ret = mv_face_recognition_get_label(handle, &out_label);
-       ASSERT_EQ(ret, MEDIA_VISION_ERROR_INVALID_OPERATION);
+       ASSERT_EQ(ret, MEDIA_VISION_ERROR_NO_DATA);
 
        ret = mv_face_recognition_destroy(handle);
        ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
@@ -273,6 +278,11 @@ TEST(FaceRecognitionTest, LabelRemovalShouldBeOk)
                        const char *out_label = NULL;
 
                        ret = mv_face_recognition_get_label(handle, &out_label);
+                       if (ret == MEDIA_VISION_ERROR_NO_DATA) {
+                               is_no_data = true;
+                               ret = MEDIA_VISION_ERROR_NONE;
+                       }
+
                        ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
 
                        if (is_no_data)

diff --git a/test/testsuites/machine_learning/face_recognition/test_face_recognition_multi_threads.cpp b/test/testsuites/machine_learning/face_recognition/test_face_recognition_multi_threads.cpp
new file mode 100644 (file)
index 0000000..8f4d1ef
--- /dev/null
+++ b/test/testsuites/machine_learning/face_recognition/test_face_recognition_multi_threads.cpp
@@ -0,0 +1,196 @@
+/**
+ * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <unistd.h>
+#include <iostream>
+#include <string.h>
+#include <map>
+#include <thread>
+
+#include "gtest/gtest.h"
+
+#include "ImageHelper.h"
+#include "mv_face_recognition.h"
+#include "face_recognition_test_util.h"
+
+#define TRAINING_IMAGE_PATH "/home/owner/media/res/face_recognition/res/test/training/"
+#define TEST_IMAGE_PATH "/home/owner/media/res/face_recognition/res/test/test/"
+#define MAX_DATA_SET 15
+
+using namespace testing;
+using namespace std;
+
+static const map<string, string> training_images = {
+       { "037830.png", "2929" }, { "038965.png", "2929" }, { "045978.png", "2929" }, { "050501.png", "2929" },
+       { "065899.png", "2929" }, { "010348.png", "7779" }, { "029342.png", "7779" }, { "035939.png", "7779" },
+       { "061310.png", "7779" }, { "062261.png", "7779" }, { "000928.png", "3448" }, { "008922.png", "3448" },
+       { "029633.png", "3448" }, { "032962.png", "3448" }, { "054616.png", "3448" }
+};
+
+static const map<string, string> test_images = {
+       { "068468.png", "2929" }, { "068883.png", "2929" }, { "075004.png", "2929" }, { "078125.png", "2929" },
+       { "080649.png", "2929" }, { "074645.png", "7779" }, { "086536.png", "7779" }, { "089334.png", "7779" },
+       { "096514.png", "7779" }, { "100336.png", "7779" }, { "054757.png", "3448" }, { "064838.png", "3448" },
+       { "072749.png", "3448" }, { "073526.png", "3448" }, { "080451.png", "3448" }
+};
+
+using namespace MediaVision::Common;
+
+void Register(mv_face_recognition_h handle)
+{
+       for (auto &image : training_images) {
+               const string image_path = string(TRAINING_IMAGE_PATH) + image.first;
+               mv_source_h mv_source = NULL;
+
+               int ret = mv_create_source(&mv_source);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+               ret = ImageHelper::loadImageToSource(image_path.c_str(), mv_source);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+               ret = mv_face_recognition_register(handle, mv_source, image.second.c_str());
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+               ret = mv_destroy_source(mv_source);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+       }
+}
+
+void Recognize(mv_face_recognition_h handle)
+{
+       vector<string> answers = { "3448", "3448", "2929", "2929", "3448", "3448", "7779", "2929",
+                                                          "2929", "3448", "2929", "7779", "7779", "7779", "7779" };
+
+       unsigned int image_idx = 0;
+       unsigned int correct_cnt = 0;
+
+       for (auto &image : test_images) {
+               const string image_path = string(TEST_IMAGE_PATH) + image.first;
+               mv_source_h mv_source = NULL;
+
+               int ret = mv_create_source(&mv_source);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+               ret = ImageHelper::loadImageToSource(image_path.c_str(), mv_source);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+               ret = mv_face_recognition_inference(handle, mv_source);
+               if (ret == MEDIA_VISION_ERROR_NO_DATA) {
+                       mv_destroy_source(mv_source);
+                       image_idx++;
+                       continue;
+               }
+
+               ret = mv_destroy_source(mv_source);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+               const char *out_label = NULL;
+
+               ret = mv_face_recognition_get_label(handle, &out_label);
+               if (ret != MEDIA_VISION_ERROR_NONE) {
+                       image_idx++;
+                       continue;
+               }
+
+               string label_str(out_label);
+
+               if (answers[image_idx++] == label_str)
+                       correct_cnt++;
+       }
+
+       cout << "Correct/Total = " << correct_cnt << " / " << image_idx << endl;
+}
+
+void Unregister(mv_face_recognition_h handle)
+{
+       vector<string> labels = { "3448", "2929", "7779" };
+
+       for (auto &image : training_images) {
+               const string image_path = string(TRAINING_IMAGE_PATH) + image.first;
+               mv_source_h mv_source = NULL;
+               int ret = mv_create_source(&mv_source);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+               ret = ImageHelper::loadImageToSource(image_path.c_str(), mv_source);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+               ret = mv_face_recognition_register(handle, mv_source, image.second.c_str());
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+               ret = mv_destroy_source(mv_source);
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+       }
+
+       for (auto &label : labels) {
+               int ret = mv_face_recognition_unregister(handle, label.c_str());
+               ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+       }
+}
+
+TEST(FaceRecognitionMultithreadTest, RegisterAndRecognizeShouldBeOk)
+{
+       mv_face_recognition_h handle;
+
+       int ret = mv_face_recognition_create(&handle);
+       ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+       ret = mv_face_recognition_prepare(handle);
+       ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+       std::thread register_thread([&] { Register(handle); });
+
+       std::thread recognize_thread([&] { Recognize(handle); });
+
+       register_thread.join();
+       recognize_thread.join();
+
+       ret = mv_face_recognition_destroy(handle);
+       ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+       RemoveModelResources();
+}
+
+TEST(FaceRecognitionMultithreadTest, RegisterAndRecognizeAndUnregisterShouldBeOk)
+{
+       mv_face_recognition_h handle;
+
+       int ret = mv_face_recognition_create(&handle);
+       ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+       ret = mv_face_recognition_prepare(handle);
+       ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+       std::thread register_thread([&] { Register(handle); });
+
+       std::thread recognize_thread([&] { Recognize(handle); });
+
+       std::thread unregister_thread([&] { Unregister(handle); });
+
+       register_thread.join();
+       recognize_thread.join();
+       unregister_thread.join();
+
+       ret = mv_face_recognition_destroy(handle);
+       ASSERT_EQ(ret, MEDIA_VISION_ERROR_NONE);
+
+       RemoveModelResources();
+}
+
+int main(int argc, char **argv)
+{
+       InitGoogleTest(&argc, argv);
+       return RUN_ALL_TESTS();
+}