Support HandGestureRecognizer

Change-Id: I78b130e413cacfb04607c8d600321ff620882616
Signed-off-by: Tae-Young Chung <ty83.chung@samsung.com>

diff --git a/capi/singleo_native_capi_internal.h b/capi/singleo_native_capi_internal.h
index a3c350ce69a17759452ad6d7fd8528d69e763a4d..36776888a886367cc23fd1069503cabafc63fad7 100644 (file)
--- a/capi/singleo_native_capi_internal.h
+++ b/capi/singleo_native_capi_internal.h
@@ -19,7 +19,7 @@
 #include "singleo_native_capi.h"
 
 typedef void (*singleo_user_cb_t)(unsigned char *buffer, unsigned int width, unsigned int height,
-                                                                 unsigned int bytes_per_pixel, void *user_data);
+                                                                 unsigned int bytes_per_pixel, void *user_data, void *user_data2);
 
 /**
  * @internal
@@ -45,7 +45,7 @@ typedef void (*singleo_user_cb_t)(unsigned char *buffer, unsigned int width, uns
 int singleo_service_register_user_callback(singleo_service_h handle,
                                                                                   void (*callback_ptr)(unsigned char *buffer, unsigned int width,
                                                                                                                                unsigned int height, unsigned int bytes_per_pixel,
-                                                                                                                               void *user_data),
+                                                                                                                               void *user_data, void *user_data2),
                                                                                   void *user_data);
 
 /**

diff --git a/inference/CMakeLists.txt b/inference/CMakeLists.txt
index 3567f21216ddb0aba2f6ad3f77372ab0dca28e2a..13478d6a8465633268883d2182541968ae357557 100644 (file)
--- a/inference/CMakeLists.txt
+++ b/inference/CMakeLists.txt
@@ -7,7 +7,7 @@ IF (${USE_EXTERNAL_INFERENCE_SERVICE})
     SET(SINGLEO_INFERENCE_HEADER_DIRECTORY "")
     SET(SINGLEO_INFERENCE_SERVUCE_FILE "{PROJECT_SOURCE_DIR}/src/InferenceServiceExternal.cpp")
 ELSE()
-    SET(SINGLEO_INFERENCE_HEADER_DIRECTORY /usr/include/media backends/mediavision/include backends/private/include)
+    SET(SINGLEO_INFERENCE_HEADER_DIRECTORY /usr/include/media /usr/include/mediapipe backends/mediavision/include backends/private/include)
     SET(SINGLEO_INFERENCE_SERVUCE_FILE
         ${SINGLEO_INFERENCE_SERVUCE_FILE}
         ${PROJECT_SOURCE_DIR}/src/MvInferenceServiceFactory.cpp

diff --git a/inference/backends/CMakeLists.txt b/inference/backends/CMakeLists.txt
index 47ad6c104827933686bf2010347d65f0ced892b0..4f9f7837754b15cd0ddfe3b88d58a4da2d8a442a 100644 (file)
--- a/inference/backends/CMakeLists.txt
+++ b/inference/backends/CMakeLists.txt
@@ -12,7 +12,7 @@ FIND_PACKAGE(PkgConfig REQUIRED)
 PKG_CHECK_MODULES(${PROJECT_NAME}_DEP REQUIRED capi-media-vision opencv facedetection)
 
 TARGET_INCLUDE_DIRECTORIES(${PROJECT_NAME} PRIVATE ../include ../../common/include ../../log/include mediavision/include /usr/include/media private/include /usr/include/facedetection /usr/include/mediapipe)
-TARGET_LINK_LIBRARIES(${PROJECT_NAME} PRIVATE mv_common singleo_log ${${PROJECT_NAME}_DEP_LIBRARIES} )
+TARGET_LINK_LIBRARIES(${PROJECT_NAME} PRIVATE mv_common singleo_log mediapipe_gesture_recognizer_solution ${${PROJECT_NAME}_DEP_LIBRARIES} )
 
 # Install the library  
 INSTALL(TARGETS ${PROJECT_NAME} DESTINATION ${LIB_INSTALL_DIR})

diff --git a/inference/backends/private/include/PrivateInferenceHandGestureService.h b/inference/backends/private/include/PrivateInferenceHandGestureService.h
index 0cbf65f9d7b12ba2ac7bc64b554584179bba3e51..c06f6e2a2bb5082d271ce98948bf0f21d5c6ddd2 100644 (file)
--- a/inference/backends/private/include/PrivateInferenceHandGestureService.h
+++ b/inference/backends/private/include/PrivateInferenceHandGestureService.h
@@ -20,17 +20,19 @@
 #include <memory>
 #include <vector>
 #include <opencv2/core.hpp>
+#include "mediapipe_hand_solution_api.h"
 #include "IInferenceHandGestureService.h"
 
+
 namespace singleo
 {
 namespace inference
 {
 class PrivateInferenceHandGestureService : public IInferenceHandGestureService
 {
-private:    
-    
+private:
        HandGestureResult _result;
+       std::unique_ptr<mediapipe_solutions::HandGesture> _gesture;
 
 public:
        PrivateInferenceHandGestureService(std::vector<inference::TaskType> &tasks);

diff --git a/inference/backends/private/src/PrivateInferenceHandGestureService.cpp b/inference/backends/private/src/PrivateInferenceHandGestureService.cpp
index 9ed71fdf4cc294f4c5f18c3f76456f16d0a51a8f..c682b7672be07783a7ac70d72ea5fd2579f17017 100644 (file)
--- a/inference/backends/private/src/PrivateInferenceHandGestureService.cpp
+++ b/inference/backends/private/src/PrivateInferenceHandGestureService.cpp
@@ -22,9 +22,10 @@
 #include <opencv2/core.hpp>
 #include <opencv2/imgcodecs.hpp>
 #include <opencv2/imgproc.hpp>
-#include "IHand.h"
 
 using namespace std;
+using namespace mediapipe_solutions;
+
 #define DETECT_BUFFER_SIZE 0x9000
 namespace singleo
 {
@@ -34,7 +35,9 @@ namespace inference
 
 PrivateInferenceHandGestureService::PrivateInferenceHandGestureService(std::vector<inference::TaskType> &tasks)
 {
-    
+    SINGLEO_LOGI("PrivateInferenceHandGestureService: ctor");
+    _gesture = make_unique<HandGesture>();
+    SINGLEO_LOGI("PrivateInferenceHandGestureService: ctor-done");
 }
 
 PrivateInferenceHandGestureService::~PrivateInferenceHandGestureService()
@@ -49,12 +52,24 @@ void PrivateInferenceHandGestureService::configure()
 
 void PrivateInferenceHandGestureService::prepare()
 {
-    
+
 }
 
 void PrivateInferenceHandGestureService::invoke(BaseDataType &input)
 {
+    Timer timer;
+    Timer partial;
+    if (input._data_type != DataType::IMAGE) {
+               SINGLEO_LOGE("Invalid input type.");
+               throw invalid_argument("Input type not support.");
+       }
+
+    SINGLEO_LOGD("PrivateInferenceHandGestureService: invoke");
+    ImageDataType &data = dynamic_cast<ImageDataType &>(input);
+    cv::Mat cvData_(cv::Size(data.width, data.height), CV_MAKE_TYPE(CV_8U, data.byte_per_pixel), data.ptr);
 
+    _result._gestureCategory = _gesture->recognize(cvData_);
+    SINGLEO_LOGD("PrivateInferenceHandGestureService: %s recognized", _result._gestureCategory.c_str());
 }
 
 HandGestureResult& PrivateInferenceHandGestureService::result()

diff --git a/inference/include/HandGestureResult.h b/inference/include/HandGestureResult.h
index 272ac5ecd851d1a7aa31b808a62c41d922e2d7a6..aef3e11997c367b20d054696ad5243dd9471be5e 100644 (file)
--- a/inference/include/HandGestureResult.h
+++ b/inference/include/HandGestureResult.h
@@ -26,7 +26,7 @@ namespace inference
 using Points = std::vector<Point>;
 struct HandGestureResult {
     unsigned int _frame_number {};
-    int _gestureId;
+    std::string _gestureCategory;
 };
 } // inference
 } // singleo

diff --git a/packaging/singleo.spec b/packaging/singleo.spec
index 54ff2793d9dce15c86e119e5b01eb305bc6e0b41..a08a3acbc4c4160852886ebd5af1a372f3fabf7c 100644 (file)
--- a/packaging/singleo.spec
+++ b/packaging/singleo.spec
@@ -19,7 +19,7 @@ BuildRequires: pkgconfig(grpc++)
 BuildRequires: pkgconfig(re2)
 BuildRequires: pkgconfig(facedetection)
 BuildRequires: mediapipe-devel
-# BuildRequires: mediapipe-image-segmentation
+BuildRequires: mediapipe-handsolution
 
 %define enable_autozoom_api 0
 %define enable_smartpointer 1

diff --git a/services/smart_pointer/include/HandGestureRecognizer.h b/services/smart_pointer/include/HandGestureRecognizer.h
index f0f6493a75f2c183c9c7c5083f9edf4a5f19288e..f17f3efbeaa317ed928e1aac3cc589b3ab8063d9 100644 (file)
--- a/services/smart_pointer/include/HandGestureRecognizer.h
+++ b/services/smart_pointer/include/HandGestureRecognizer.h
@@ -37,7 +37,7 @@ private:
 public:
     explicit HandGestureRecognizer(input::InputConfigBase &config);
     ~HandGestureRecognizer();
-    int recognizeGesture(BaseDataType &input);
+    std::string recognizeGesture(BaseDataType &input);
 
 };
 } // smartpointer

diff --git a/services/smart_pointer/src/HandGestureRecognizer.cpp b/services/smart_pointer/src/HandGestureRecognizer.cpp
index 5ed60667c37294c29029674b07033cc5d79b0114..6997e8aed8559a107847102bb01a6db1725c1a26 100644 (file)
--- a/services/smart_pointer/src/HandGestureRecognizer.cpp
+++ b/services/smart_pointer/src/HandGestureRecognizer.cpp
@@ -35,9 +35,6 @@ HandGestureRecognizer::HandGestureRecognizer(InputConfigBase& config)
     // MvInferenceServiceFactory factory;
     PrivateInferenceServiceFactory factory;
     _recognizer = factory.createInferenceHandGestureService(_tasks);
-
-    _recognizer->configure();
-       _recognizer->prepare();
 }
 
 HandGestureRecognizer::~HandGestureRecognizer()
@@ -45,10 +42,11 @@ HandGestureRecognizer::~HandGestureRecognizer()
 
 }
 
-int HandGestureRecognizer::recognizeGesture(BaseDataType &input)
+string HandGestureRecognizer::recognizeGesture(BaseDataType &input)
 {
-    SINGLEO_LOGD("HandGestureRecognizer->recognizeGesture");
-    return -1;
+    _recognizer->invoke(input);
+    auto result = _recognizer->result();
+    return result._gestureCategory;
     //  _recognizer->invoke(input);
     //  auto &result = _recognizer->result();
 }

diff --git a/services/smart_pointer/src/SmartPointer.cpp b/services/smart_pointer/src/SmartPointer.cpp
index 4baf0e6274fb3fa45d41c6fd0ad55e0e21082240..2a19011a44077e777a1ed15bea1aa910f037c821 100644 (file)
--- a/services/smart_pointer/src/SmartPointer.cpp
+++ b/services/smart_pointer/src/SmartPointer.cpp
@@ -71,7 +71,7 @@ void SmartPointer::internalDataFeederCb(BaseDataType &data, void *user_data)
        memcpy(copied.ptr, image_data.ptr, buffer_size);
 
        if (smart_pointer->_user_cb) {
-               smart_pointer->_user_cb(copied.ptr, copied.width, copied.height, copied.byte_per_pixel, smart_pointer->_user_data);
+               smart_pointer->_user_cb(copied.ptr, copied.width, copied.height, copied.byte_per_pixel, smart_pointer->_user_data, nullptr);
        }
 
        if (smart_pointer->getAsyncManager()->pushInput(copied) != SINGLEO_ERROR_NONE)
@@ -118,7 +118,8 @@ void SmartPointer::perform()
                if (_user_cb) {
                        SINGLEO_LOGD("user callback in perform()");
                        _user_cb(input_data.ptr, input_data.width, input_data.height, input_data.byte_per_pixel, 
-                       static_cast<void *>(&(_pointerPosition.pose._rot_vec)));
+                       static_cast<void *>(&(_pointerPosition.pose._rot_vec)),
+                       static_cast<void *>(&gestureResult));
                }
        }
 

diff --git a/test/services/smartpointer.cpp b/test/services/smartpointer.cpp
index 2f7c7921d0ac899147c2c5e3cb2e9e2eb691c633..81c88e40b1295deb5a91ddf9bc33d4ae0f982e29 100644 (file)
--- a/test/services/smartpointer.cpp
+++ b/test/services/smartpointer.cpp
@@ -53,14 +53,19 @@ void user_callback(void *user_data)
 }
 
 void data_feeder_cb(unsigned char *buffer, unsigned int width, unsigned int height, unsigned int bytes_per_pixel,
-                                  void *user_data)
+                                  void *headPose, void *gesture)
 {
        cv::Mat result(cv::Size(width, height), CV_MAKETYPE(CV_8U, bytes_per_pixel), buffer);
        cv::Mat vizData;
        cv::cvtColor(result, vizData, cv::COLOR_BGR2RGBA);
 
-       Point3f *dof3d = static_cast<Point3f *>(user_data);
+       string *gesture_str = static_cast<string *>(gesture);
+       cv::putText(vizData, *gesture_str, cv::Point(30, 30), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 255, 0, 255), 2);
+
+       Point3f *dof3d = static_cast<Point3f *>(headPose);
+
        cout << "[Pitch, Yaw, Roll]: [ " << dof3d->x << ", " << dof3d->y << ", " << dof3d->z << " ]" << endl;
+       cout << "[Gesture Str]: " << *gesture_str << endl;
        singleo_util_visualizer_2d(vizData, NULL);
 }