From 1dd1e4ff3dbd79c10b7d524672bfba37e4cb893c Mon Sep 17 00:00:00 2001 From: Tae-Young Chung Date: Wed, 5 Jun 2024 17:59:01 +0900 Subject: [PATCH] Support HandGestureRecognizer Change-Id: I78b130e413cacfb04607c8d600321ff620882616 Signed-off-by: Tae-Young Chung --- capi/singleo_native_capi_internal.h | 4 ++-- inference/CMakeLists.txt | 2 +- inference/backends/CMakeLists.txt | 2 +- .../PrivateInferenceHandGestureService.h | 6 ++++-- .../PrivateInferenceHandGestureService.cpp | 21 ++++++++++++++++--- inference/include/HandGestureResult.h | 2 +- packaging/singleo.spec | 2 +- .../include/HandGestureRecognizer.h | 2 +- .../src/HandGestureRecognizer.cpp | 10 ++++----- services/smart_pointer/src/SmartPointer.cpp | 5 +++-- test/services/smartpointer.cpp | 9 ++++++-- 11 files changed, 43 insertions(+), 22 deletions(-) diff --git a/capi/singleo_native_capi_internal.h b/capi/singleo_native_capi_internal.h index a3c350c..3677688 100644 --- a/capi/singleo_native_capi_internal.h +++ b/capi/singleo_native_capi_internal.h @@ -19,7 +19,7 @@ #include "singleo_native_capi.h" typedef void (*singleo_user_cb_t)(unsigned char *buffer, unsigned int width, unsigned int height, - unsigned int bytes_per_pixel, void *user_data); + unsigned int bytes_per_pixel, void *user_data, void *user_data2); /** * @internal @@ -45,7 +45,7 @@ typedef void (*singleo_user_cb_t)(unsigned char *buffer, unsigned int width, uns int singleo_service_register_user_callback(singleo_service_h handle, void (*callback_ptr)(unsigned char *buffer, unsigned int width, unsigned int height, unsigned int bytes_per_pixel, - void *user_data), + void *user_data, void *user_data2), void *user_data); /** diff --git a/inference/CMakeLists.txt b/inference/CMakeLists.txt index 3567f21..13478d6 100644 --- a/inference/CMakeLists.txt +++ b/inference/CMakeLists.txt @@ -7,7 +7,7 @@ IF (${USE_EXTERNAL_INFERENCE_SERVICE}) SET(SINGLEO_INFERENCE_HEADER_DIRECTORY "") SET(SINGLEO_INFERENCE_SERVUCE_FILE "{PROJECT_SOURCE_DIR}/src/InferenceServiceExternal.cpp") ELSE() - SET(SINGLEO_INFERENCE_HEADER_DIRECTORY /usr/include/media backends/mediavision/include backends/private/include) + SET(SINGLEO_INFERENCE_HEADER_DIRECTORY /usr/include/media /usr/include/mediapipe backends/mediavision/include backends/private/include) SET(SINGLEO_INFERENCE_SERVUCE_FILE ${SINGLEO_INFERENCE_SERVUCE_FILE} ${PROJECT_SOURCE_DIR}/src/MvInferenceServiceFactory.cpp diff --git a/inference/backends/CMakeLists.txt b/inference/backends/CMakeLists.txt index 47ad6c1..4f9f783 100644 --- a/inference/backends/CMakeLists.txt +++ b/inference/backends/CMakeLists.txt @@ -12,7 +12,7 @@ FIND_PACKAGE(PkgConfig REQUIRED) PKG_CHECK_MODULES(${PROJECT_NAME}_DEP REQUIRED capi-media-vision opencv facedetection) TARGET_INCLUDE_DIRECTORIES(${PROJECT_NAME} PRIVATE ../include ../../common/include ../../log/include mediavision/include /usr/include/media private/include /usr/include/facedetection /usr/include/mediapipe) -TARGET_LINK_LIBRARIES(${PROJECT_NAME} PRIVATE mv_common singleo_log ${${PROJECT_NAME}_DEP_LIBRARIES} ) +TARGET_LINK_LIBRARIES(${PROJECT_NAME} PRIVATE mv_common singleo_log mediapipe_gesture_recognizer_solution ${${PROJECT_NAME}_DEP_LIBRARIES} ) # Install the library INSTALL(TARGETS ${PROJECT_NAME} DESTINATION ${LIB_INSTALL_DIR}) diff --git a/inference/backends/private/include/PrivateInferenceHandGestureService.h b/inference/backends/private/include/PrivateInferenceHandGestureService.h index 0cbf65f..c06f6e2 100644 --- a/inference/backends/private/include/PrivateInferenceHandGestureService.h +++ b/inference/backends/private/include/PrivateInferenceHandGestureService.h @@ -20,17 +20,19 @@ #include #include #include +#include "mediapipe_hand_solution_api.h" #include "IInferenceHandGestureService.h" + namespace singleo { namespace inference { class PrivateInferenceHandGestureService : public IInferenceHandGestureService { -private: - +private: HandGestureResult _result; + std::unique_ptr _gesture; public: PrivateInferenceHandGestureService(std::vector &tasks); diff --git a/inference/backends/private/src/PrivateInferenceHandGestureService.cpp b/inference/backends/private/src/PrivateInferenceHandGestureService.cpp index 9ed71fd..c682b76 100644 --- a/inference/backends/private/src/PrivateInferenceHandGestureService.cpp +++ b/inference/backends/private/src/PrivateInferenceHandGestureService.cpp @@ -22,9 +22,10 @@ #include #include #include -#include "IHand.h" using namespace std; +using namespace mediapipe_solutions; + #define DETECT_BUFFER_SIZE 0x9000 namespace singleo { @@ -34,7 +35,9 @@ namespace inference PrivateInferenceHandGestureService::PrivateInferenceHandGestureService(std::vector &tasks) { - + SINGLEO_LOGI("PrivateInferenceHandGestureService: ctor"); + _gesture = make_unique(); + SINGLEO_LOGI("PrivateInferenceHandGestureService: ctor-done"); } PrivateInferenceHandGestureService::~PrivateInferenceHandGestureService() @@ -49,12 +52,24 @@ void PrivateInferenceHandGestureService::configure() void PrivateInferenceHandGestureService::prepare() { - + } void PrivateInferenceHandGestureService::invoke(BaseDataType &input) { + Timer timer; + Timer partial; + if (input._data_type != DataType::IMAGE) { + SINGLEO_LOGE("Invalid input type."); + throw invalid_argument("Input type not support."); + } + + SINGLEO_LOGD("PrivateInferenceHandGestureService: invoke"); + ImageDataType &data = dynamic_cast(input); + cv::Mat cvData_(cv::Size(data.width, data.height), CV_MAKE_TYPE(CV_8U, data.byte_per_pixel), data.ptr); + _result._gestureCategory = _gesture->recognize(cvData_); + SINGLEO_LOGD("PrivateInferenceHandGestureService: %s recognized", _result._gestureCategory.c_str()); } HandGestureResult& PrivateInferenceHandGestureService::result() diff --git a/inference/include/HandGestureResult.h b/inference/include/HandGestureResult.h index 272ac5e..aef3e11 100644 --- a/inference/include/HandGestureResult.h +++ b/inference/include/HandGestureResult.h @@ -26,7 +26,7 @@ namespace inference using Points = std::vector; struct HandGestureResult { unsigned int _frame_number {}; - int _gestureId; + std::string _gestureCategory; }; } // inference } // singleo diff --git a/packaging/singleo.spec b/packaging/singleo.spec index 54ff279..a08a3ac 100644 --- a/packaging/singleo.spec +++ b/packaging/singleo.spec @@ -19,7 +19,7 @@ BuildRequires: pkgconfig(grpc++) BuildRequires: pkgconfig(re2) BuildRequires: pkgconfig(facedetection) BuildRequires: mediapipe-devel -# BuildRequires: mediapipe-image-segmentation +BuildRequires: mediapipe-handsolution %define enable_autozoom_api 0 %define enable_smartpointer 1 diff --git a/services/smart_pointer/include/HandGestureRecognizer.h b/services/smart_pointer/include/HandGestureRecognizer.h index f0f6493..f17f3ef 100644 --- a/services/smart_pointer/include/HandGestureRecognizer.h +++ b/services/smart_pointer/include/HandGestureRecognizer.h @@ -37,7 +37,7 @@ private: public: explicit HandGestureRecognizer(input::InputConfigBase &config); ~HandGestureRecognizer(); - int recognizeGesture(BaseDataType &input); + std::string recognizeGesture(BaseDataType &input); }; } // smartpointer diff --git a/services/smart_pointer/src/HandGestureRecognizer.cpp b/services/smart_pointer/src/HandGestureRecognizer.cpp index 5ed6066..6997e8a 100644 --- a/services/smart_pointer/src/HandGestureRecognizer.cpp +++ b/services/smart_pointer/src/HandGestureRecognizer.cpp @@ -35,9 +35,6 @@ HandGestureRecognizer::HandGestureRecognizer(InputConfigBase& config) // MvInferenceServiceFactory factory; PrivateInferenceServiceFactory factory; _recognizer = factory.createInferenceHandGestureService(_tasks); - - _recognizer->configure(); - _recognizer->prepare(); } HandGestureRecognizer::~HandGestureRecognizer() @@ -45,10 +42,11 @@ HandGestureRecognizer::~HandGestureRecognizer() } -int HandGestureRecognizer::recognizeGesture(BaseDataType &input) +string HandGestureRecognizer::recognizeGesture(BaseDataType &input) { - SINGLEO_LOGD("HandGestureRecognizer->recognizeGesture"); - return -1; + _recognizer->invoke(input); + auto result = _recognizer->result(); + return result._gestureCategory; // _recognizer->invoke(input); // auto &result = _recognizer->result(); } diff --git a/services/smart_pointer/src/SmartPointer.cpp b/services/smart_pointer/src/SmartPointer.cpp index 4baf0e6..2a19011 100644 --- a/services/smart_pointer/src/SmartPointer.cpp +++ b/services/smart_pointer/src/SmartPointer.cpp @@ -71,7 +71,7 @@ void SmartPointer::internalDataFeederCb(BaseDataType &data, void *user_data) memcpy(copied.ptr, image_data.ptr, buffer_size); if (smart_pointer->_user_cb) { - smart_pointer->_user_cb(copied.ptr, copied.width, copied.height, copied.byte_per_pixel, smart_pointer->_user_data); + smart_pointer->_user_cb(copied.ptr, copied.width, copied.height, copied.byte_per_pixel, smart_pointer->_user_data, nullptr); } if (smart_pointer->getAsyncManager()->pushInput(copied) != SINGLEO_ERROR_NONE) @@ -118,7 +118,8 @@ void SmartPointer::perform() if (_user_cb) { SINGLEO_LOGD("user callback in perform()"); _user_cb(input_data.ptr, input_data.width, input_data.height, input_data.byte_per_pixel, - static_cast(&(_pointerPosition.pose._rot_vec))); + static_cast(&(_pointerPosition.pose._rot_vec)), + static_cast(&gestureResult)); } } diff --git a/test/services/smartpointer.cpp b/test/services/smartpointer.cpp index 2f7c792..81c88e4 100644 --- a/test/services/smartpointer.cpp +++ b/test/services/smartpointer.cpp @@ -53,14 +53,19 @@ void user_callback(void *user_data) } void data_feeder_cb(unsigned char *buffer, unsigned int width, unsigned int height, unsigned int bytes_per_pixel, - void *user_data) + void *headPose, void *gesture) { cv::Mat result(cv::Size(width, height), CV_MAKETYPE(CV_8U, bytes_per_pixel), buffer); cv::Mat vizData; cv::cvtColor(result, vizData, cv::COLOR_BGR2RGBA); - Point3f *dof3d = static_cast(user_data); + string *gesture_str = static_cast(gesture); + cv::putText(vizData, *gesture_str, cv::Point(30, 30), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 255, 0, 255), 2); + + Point3f *dof3d = static_cast(headPose); + cout << "[Pitch, Yaw, Roll]: [ " << dof3d->x << ", " << dof3d->y << ", " << dof3d->z << " ]" << endl; + cout << "[Gesture Str]: " << *gesture_str << endl; singleo_util_visualizer_2d(vizData, NULL); } -- 2.34.1