*/
int singleo_service_get_result_int(singleo_service_h handle, unsigned int idx, const char *key, unsigned int *value);
+/**
+ * @internal
+ * @brief Gets float type of result corresponding to a given index value.
+ *
+ * @since_tizen 9.0
+ *
+ * @param[in] handle The handle to the service.
+ * @param[in] idx Index value to results.
+ * @param[in] key Key string to a certain member of the result.
+ * @param[out] value Pointer to the float variable to be stored.
+ *
+ * @return 0 on success, otherwise a negative error value
+ * @retval #SINGLEO_SERVICE_ERROR_NONE Successful
+ * @retval #SINGLEO_SERVICE_ERROR_ERROR_INVALID_PARAMETER Invalid parameter
+ * @retval #SINGLEO_SERVICE_ERROR_ERROR_INVALID_OPERATION Invalid operation
+ *
+ * @pre Create a source handle by calling singleo_service_create()
+ * @pre Perform a requested service by calling singleo_service_perform()
+ * @pre Get a number of results by calling singleo_service_get_result_cnt()
+ */
+int singleo_service_get_result_float(singleo_service_h handle, unsigned int idx, const char *key, float *value);
+
/**
* @internal
* @brief Gets string type of result corresponding to a given index value.
#define __SINGLEO_COMMON_TYPES_H__
#include <memory>
+#include <cmath>
#include <cstring>
#include <vector>
#include "SingleoLog.h"
}
};
+struct FPoint {
+ float x {};
+ float y {};
+
+ bool operator==(const FPoint &other) const
+ {
+ const float EPSILON = 1e-5f;
+
+ return (fabs(x - other.x) < EPSILON && fabs(y - other.y) < EPSILON);
+ }
+};
+
enum class DataType { NONE, FILE, IMAGE, RAW };
enum class ImagePixelFormat { NONE, YUV420, RGB888 };
}
};
-enum class ResultType { NONE, OBJECT_DETECTION, FACE_DETECTION, FACE_LANDMARK, IMAGE_CLASSIFICATION, FACE_RECOGNITION, GAZE_ESTIMATION };
+enum class ResultType {
+ NONE,
+ OBJECT_DETECTION,
+ FACE_DETECTION,
+ FACE_LANDMARK,
+ HAND_DETECTION,
+ HAND_LANDMARK,
+ IMAGE_CLASSIFICATION,
+ FACE_RECOGNITION,
+ GAZE_TRACKING
+};
struct BaseResultType {
ResultType _type { ResultType::NONE };
}
};
+struct HdResultType : public BaseResultType {
+ HdResultType() : BaseResultType(ResultType::HAND_DETECTION)
+ {}
+ std::vector<Rect> _rects;
+
+ std::shared_ptr<BaseResultType> clone() override
+ {
+ return std::make_shared<HdResultType>(*this);
+ }
+};
+
+struct HldResultType : public BaseResultType {
+ HldResultType() : BaseResultType(ResultType::HAND_LANDMARK)
+ {}
+ std::vector<Point> _points;
+
+ std::shared_ptr<BaseResultType> clone() override
+ {
+ return std::make_shared<HldResultType>(*this);
+ }
+};
+
+
struct IcResultType : public BaseResultType {
IcResultType() : BaseResultType(ResultType::IMAGE_CLASSIFICATION)
{}
};
struct GazeResultType : public BaseResultType {
- GazeResultType() : BaseResultType(ResultType::GAZE_ESTIMATION)
+ GazeResultType() : BaseResultType(ResultType::GAZE_TRACKING)
{}
-
+ std::vector<Point> _points; // 0: center, 1: gazed
+ std::vector<FPoint> _fpoints;
double _pitch {};
double _yaw {};
- std::vector<Point> _points; // 0: center, 1: gazed
std::shared_ptr<BaseResultType> clone() override
{
}
};
-enum class ServiceType { NONE, AUTO_ZOOM, FOCUS_FINDER };
-
enum class InputFeedType { NONE, CAMERA, SCREEN_CAPTURE };
enum class CameraBackendType { NONE, OPENCV, CAMERA_API, VISION_SOURCE };
SET(SINGLEO_SERVICE_SOURCE_FILES
${SINGLEO_SERVICE_SOURCE_FILES}
+ ${INFERENCE_MEDIAVISION_BACKEND_DIRECTORY}/src/MvGazeTracking.cpp
${INFERENCE_MEDIAVISION_BACKEND_DIRECTORY}/src/MvHandLandmark.cpp
${INFERENCE_MEDIAVISION_BACKEND_DIRECTORY}/src/MvHandDetection.cpp
${INFERENCE_MEDIAVISION_BACKEND_DIRECTORY}/src/MvFaceDetection.cpp
* limitations under the License.
*/
-#ifndef __MV_GAZE_ESTIMATION_H__
-#define __MV_GAZE_ESTIMATION_H__
+#ifndef __MV_GAZE_TRACKING_H__
+#define __MV_GAZE_TRACKING_H__
#include "IInferenceTaskInterface.h"
#include "mv_gaze_tracking.h"
{
private:
mv_gaze_tracking_h _handle {};
- GazeResultType _output_data;
+ GazeResultType _output_data {};
public:
MvGazeTracking();
{
private:
mv_hand_detection_h _handle {};
- FdResultType _output_data {};
+ HdResultType _output_data {};
public:
MvHandDetection();
{
private:
mv_hand_landmark_h _handle {};
- FldResultType _output_data;
+ HldResultType _output_data;
public:
MvHandLandmark();
void MvGazeTracking::configure()
{
- int ret = mv_gaze_tracking_set_model(_handle, "gzt_l2cs_mobilenetv2_224x224.tflite", "gzt_l2cs_mobilenetv2_224x224.json",
- "", "GZE_L2CS_NET_MOBILENETV2");
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw runtime_error("Fail to set gaze tracking model.");
-
- ret = mv_gaze_tracking_configure(_handle);
+ int ret = mv_gaze_tracking_configure(_handle);
if (ret != MEDIA_VISION_ERROR_NONE)
throw runtime_error("Fail to configure gaze tracking.");
}
ret = mv_gaze_tracking_inference(_handle, mv_src);
if (ret != MEDIA_VISION_ERROR_NONE)
- throw runtime_error("Fail to invoke gaze estimation.");
+ throw runtime_error("Fail to invoke gaze tracking.");
} catch (std::runtime_error &e) {
SINGLEO_LOGE("%s", e.what());
}
int ret = mv_gaze_tracking_get_result_count(_handle, &frame_number, &result_cnt);
if (ret != MEDIA_VISION_ERROR_NONE)
- throw runtime_error("Fail to get gaze estimation result count.");
+ throw runtime_error("Fail to get gaze tracking result count.");
+ _output_data = GazeResultType();
_output_data._frame_number = frame_number;
_output_data._is_empty = result_cnt == 0;
_output_data._yaw = yaw;
_output_data._pitch = pitch;
+
+ FPoint point {};
+
+ ret = mv_gaze_tracking_get_pos(_handle, 0, &point.x, &point.y);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw runtime_error("Fail to get gaze tracking result.");
+
+ _output_data._fpoints.push_back(point);
}
return _output_data;
BuildRequires: pkgconfig(re2)
BuildRequires: pkgconfig(libtzplatform-config)
-%define enable_visualizer 0
+%define enable_visualizer 1
%define enable_focusfinder 1
%define use_3rdparty_facedetection 0
-DUSE_INPUT_OPENCV_BACKEND=ON \
-DUSE_AUTOZOOM_API=ON \
-DUSE_FOCUSFINDER_API=%{enable_focusfinder} \
+ -DUSE_AIRGESTURE_API=ON \
-DUSE_INPUT_CAMERA_API_BACKEND=ON \
-DUSE_VISUALIZER=%{enable_visualizer} \
-DUSE_3RDPARTY_FACEDETECTION=%{use_3rdparty_facedetection}
%{_bindir}/test_singleo
%if "%{enable_visualizer}" == "1"
%{_bindir}/test_singleo_on_screen
+%{_bindir}/test_singleo_airgesture
%endif
%{_bindir}/test_singleo_task_manger
INCLUDE(focus_finder/CMakeLists.txt)
ENDIF()
+IF (${USE_AIRGESTURE_API})
+ INCLUDE(air_gesture/CMakeLists.txt)
+ENDIF()
+
ADD_LIBRARY(${PROJECT_NAME} SHARED ${SINGLEO_SERVICE_SOURCE_FILES})
TARGET_INCLUDE_DIRECTORIES(${PROJECT_NAME} PRIVATE include common/include ${ROOT_DIRECTORY}/capi/ ${COMMON_HEADER_LIST} ${INPUT_HEADER_LIST} ${INFERENCE_HEADER_LIST} ${SERVICE_HEADER_LIST} ${LOG_HEADER_LIST})
--- /dev/null
+SET(SINGLEO_SERVICE_SOURCE_FILES
+ ${SINGLEO_SERVICE_SOURCE_FILES}
+ air_gesture/src/TinyTrackerS.cpp
+ air_gesture/src/AirGesture.cpp
+)
+
+LIST(APPEND SERVICE_LIBRARY_LIST ${SERVICE_LIBRARY_LIST})
+LIST(APPEND SERVICE_HEADER_LIST ${SERVICE_HEADER_LIST} ${CMAKE_CURRENT_SOURCE_DIR}/air_gesture/include)
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __AIR_GESTURE_H__
+#define __AIR_GESTURE_H__
+
+#include <opencv2/opencv.hpp>
+#include <mutex>
+
+#include "IService.h"
+#include "SingleoException.h"
+#include "SingleoCommonTypes.h"
+#include "IInputService.h"
+#include "SingleoInferenceTypes.h"
+#include "InputTypes.h"
+#include "IInputObserver.h"
+#include "ServiceDataType.h"
+#include "DataTypes.h"
+#include "AsyncManager.h"
+#include "IPreprocessor.h"
+#include "TaskManager.h"
+#include "IDecoder.h"
+
+namespace singleo
+{
+namespace services
+{
+namespace airgesture
+{
+class AirGesture : public IService, public input::IInputObserver
+{
+private:
+ static bool _registered;
+ std::unique_ptr<TaskManager> _taskManager;
+ std::unique_ptr<singleo::input::IInputService> _input_service;
+ std::queue<std::shared_ptr<BaseDataType> > _inputQ;
+ std::mutex _inputMutex;
+ AirGestureResult _result;
+ std::mutex _result_lock;
+ std::map<std::string, AirGestureResultType> _result_keys = { { "X", AirGestureResultType::X },
+ { "Y", AirGestureResultType::Y } };
+ bool _async_mode { false };
+
+ bool isKeyValid(std::string key);
+ void updateResult(BaseDataType &input_data);
+
+ std::unique_ptr<AsyncManager<ImageDataType, AirGestureResult> > _async_manager;
+ singleo_user_cb_t _user_cb {};
+ void *_user_data {};
+ std::unique_ptr<IPreprocessor> _preprocessor;
+ std::unique_ptr<IDecoder> _decoder;
+ AirGestureMode _mode { AirGestureMode::FACE_TRACKING };
+
+public:
+ AirGesture();
+ virtual ~AirGesture();
+
+ static IService *create()
+ {
+ return new AirGesture();
+ }
+
+ // This function will be called by specific input service internally.
+ // Ps. caller has to provide captured data with concrete class object as data parameter.
+ void inputFeedCb(BaseDataType &data) override;
+ void configure(input::InputConfigBase &config) override;
+ void add_input(BaseDataType &input_data) override;
+ void perform() override;
+ void performAsync() override;
+ void getResultCnt(unsigned int *cnt) override;
+ void getResultInt(unsigned int idx, std::string key, unsigned int *value) override
+ {
+ throw exception::InvalidOperation("Not supported yet.");
+ }
+ void getResultFloat(unsigned int idx, std::string key, float *value) override;
+ void registerUserCallback(singleo_user_cb_t user_cb, void *user_data) override;
+ void unregisterUserCallback() override;
+
+ void runTaskManager(BaseDataType &input_data);
+};
+
+} // airgesture
+} // service
+} // singleo
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __DATA_TYPES_H__
+#define __DATA_TYPES_H__
+
+#include <vector>
+#include "SingleoCommonTypes.h"
+#include "ServiceDataType.h"
+
+namespace singleo
+{
+namespace services
+{
+namespace airgesture
+{
+enum class AirGestureResultType { X, Y };
+
+enum class AirGestureMode { NONE, FACE_TRACKING, GAZE_TRACKING, HAND_TRACKING };
+
+struct AirGestureResult : public ServiceBaseResultType {
+ AirGestureResult() : ServiceBaseResultType(ServiceResultType::AIR_GESTURE)
+ {}
+ bool is_valid { false };
+ unsigned int frame_number {};
+ unsigned int num_of_objects {};
+ float point_x {};
+ float point_y {};
+ std::vector<Rect> rects;
+ std::vector<FPoint> gaze_points;
+ std::vector<Point> landmark_points;
+ std::vector<Rect> handRects;
+ std::vector<Rect> preHandRects;
+ std::vector<Point> hand_landmark_points;
+};
+
+}
+}
+}
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __TINY_TRACKER_S_H__
+#define __TINY_TRACKER_S_H__
+
+#include "IDecoder.h"
+#include "SingleoCommonTypes.h"
+#include "ServiceDataType.h"
+#include "DataTypes.h"
+
+namespace singleo
+{
+namespace services
+{
+namespace airgesture
+{
+class TinyTrackerS : public IDecoder
+{
+private:
+ float _screenWidth { 1600.0f };
+ float _screenHeight { 900.0f };
+ float _x_weight {};
+ float _y_weight {};
+ float _x_intercept {};
+ float _y_intercept {};
+
+ void getCurrentPosition(Rect previous, Rect current, int *x, int *y);
+
+public:
+ TinyTrackerS();
+ virtual ~TinyTrackerS() = default;
+
+ void decode(ServiceBaseResultType &baseResult) override;
+};
+
+}
+}
+}
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2024 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <algorithm>
+#include <map>
+#include "SingleoException.h"
+#include "AirGesture.h"
+#include "InferenceTaskFactory.h"
+#include "SingleoLog.h"
+#include "ImagePreprocessor.h"
+#include "ServiceFactory.h"
+#include "InputCamera.h"
+#include "InputTypes.h"
+#include "InferenceNode.h"
+#include "BridgeNode.h"
+#include "EndpointNode.h"
+#include "ClaheFilter.h"
+#include "TinyTrackerS.h"
+
+using namespace std;
+using namespace singleo::inference;
+using namespace singleo::input;
+using namespace singleo::exception;
+
+namespace singleo
+{
+namespace services
+{
+namespace airgesture
+{
+bool AirGesture::_registered = registerService<AirGesture>("AirGesture");
+
+void BridgeNodeCallback(INode *node)
+{
+ auto callbackNode = dynamic_cast<CallbackNode *>(node);
+ auto &inputBuffer = callbackNode->getInputBuffer();
+ auto imageData = dynamic_pointer_cast<ImageDataType>(inputBuffer->getInputs()[0]);
+ cv::Mat cv_image(cv::Size(imageData->width, imageData->height), CV_MAKETYPE(CV_8U, 3), imageData->ptr);
+ auto outputBuffer = make_shared<SharedBuffer>();
+
+ auto &results = callbackNode->results();
+ for (auto r : results) {
+ if (r->_type != ResultType::FACE_DETECTION)
+ continue;
+
+ auto f_r = dynamic_pointer_cast<FdResultType>(r);
+
+ for (auto rect : f_r->_rects) {
+ if (rect.left < 0)
+ rect.left = 0;
+ if (rect.top < 0)
+ rect.top = 0;
+ if (rect.right >= static_cast<int>(imageData->width))
+ rect.right = imageData->width - 1;
+ if (rect.bottom >= static_cast<int>(imageData->height))
+ rect.bottom = imageData->height - 1;
+
+ cv::Rect roi(rect.left, rect.top, rect.right - rect.left, rect.bottom - rect.top);
+ cv::Mat roi_img = cv_image(roi).clone();
+ ImageDataType faceImage;
+
+ faceImage.width = roi_img.cols;
+ faceImage.height = roi_img.rows;
+ faceImage.byte_per_pixel = roi_img.channels();
+ faceImage.pixel_format = ImagePixelFormat::RGB888;
+ faceImage.is_owned = true;
+
+ size_t faceSize = roi_img.cols * roi_img.rows * roi_img.channels();
+
+ faceImage.ptr = new unsigned char[faceSize];
+ memcpy(faceImage.ptr, roi_img.data, faceSize);
+
+ outputBuffer->addInput(make_shared<ImageDataType>(faceImage));
+ break;
+ }
+ }
+
+ // Face detection failed so do not go forward.
+ if (outputBuffer->getInputs().size() == 0)
+ return;
+
+ callbackNode->setOutputBuffer(outputBuffer);
+}
+
+AirGesture::AirGesture()
+{
+ // In default, we will use Inference service factory for Mediavision to use Mediavision framework
+ // for inference service. TODO. introduce meta config file approach later.
+ auto factory = InferenceTaskFactory::instance().create("MvInferenceTaskFactory");
+
+ _taskManager = make_unique<TaskManager>();
+
+ _taskManager->requestNewNode(NodeType::STARTPOINT, "startpoint");
+
+ auto hand_detection_node = _taskManager->requestNewNode(NodeType::INFERENCE, "hand_detection");
+ dynamic_cast<InferenceNode *>(hand_detection_node)->setInferenceTask(factory->createHandDetection());
+
+ auto face_detection_node = _taskManager->requestNewNode(NodeType::INFERENCE, "face_detection");
+ dynamic_cast<InferenceNode *>(face_detection_node)->setInferenceTask(factory->createFaceDetection());
+
+ auto bridge_node_a = _taskManager->requestNewNode(NodeType::BRIDGE, "bridge_node_a");
+ dynamic_cast<BridgeNode *>(bridge_node_a)->setCb(BridgeNodeCallback);
+
+ auto gaze_tracking_node = _taskManager->requestNewNode(NodeType::INFERENCE, "gaze_tracking");
+ dynamic_cast<InferenceNode *>(gaze_tracking_node)->setInferenceTask(factory->createGazeTracking());
+
+ _taskManager->requestNewNode(NodeType::ENDPOINT, "endpoint");
+
+ _preprocessor = make_unique<ImagePreprocessor>();
+ _decoder = make_unique<TinyTrackerS>();
+
+ _taskManager->resetPipeline();
+
+ _taskManager->addEdge("startpoint", "hand_detection");
+ _taskManager->addEdge("startpoint", "face_detection");
+ _taskManager->addEdge("hand_detection", "endpoint");
+ _taskManager->addEdge("face_detection", "bridge_node_a");
+ _taskManager->addEdge("bridge_node_a", "gaze_tracking");
+ _taskManager->addEdge("face_detection", "endpoint");
+ _taskManager->addEdge("gaze_tracking", "endpoint");
+
+ _taskManager->setBypass("hand_detection");
+
+ _taskManager->verify();
+}
+
+void AirGesture::configure(InputConfigBase &config)
+{
+ // Create InputCamera service if input service type is CAMERA.
+ if (config._input_feed_type == InputFeedType::CAMERA) {
+ CameraConfig cameraConfig;
+
+ try {
+ cameraConfig = dynamic_cast<CameraConfig &>(config);
+ } catch (bad_cast &e) {
+ SINGLEO_LOGE("Camera input service requires CameraConfig.");
+ throw InvalidOperation("Camera input service requires CameraConfig.");
+ }
+
+ _input_service = make_unique<InputCamera>(cameraConfig.backend_type);
+ _input_service->registerObserver(this);
+ _input_service->configure(cameraConfig);
+
+ SINGLEO_LOGD("Camera input service has been initialized.");
+ }
+}
+
+AirGesture::~AirGesture()
+{
+ if (_async_mode) {
+ _input_service->streamOff();
+ _async_manager->destroy();
+ }
+
+ _taskManager->clear();
+}
+
+void AirGesture::inputFeedCb(BaseDataType &data)
+{
+ _preprocessor->addInput(data);
+
+ auto originalImg = dynamic_pointer_cast<ImageDataType>(_preprocessor->getOutput().clone());
+ auto preprocessedImg = dynamic_cast<ImageDataType &>(_preprocessor->getOutput());
+
+ if (_user_cb) {
+ cv::Mat imgCv(cv::Size(preprocessedImg.width, preprocessedImg.height), CV_MAKETYPE(CV_8U, 3),
+ preprocessedImg.ptr);
+
+ _result_lock.lock();
+
+ if (_result.is_valid) {
+ auto rects = _result.rects;
+ auto landmarks = _result.landmark_points;
+ auto handRects = _result.handRects;
+ auto handLandmarks = _result.hand_landmark_points;
+
+ _result_lock.unlock();
+
+ int delta_x {}, delta_y {};
+
+ for (unsigned int idx = 0; idx < rects.size(); ++idx) {
+ cv::rectangle(imgCv, cv::Point(rects[idx].left, rects[idx].top),
+ cv::Point(rects[idx].right, rects[idx].bottom), cv::Scalar(255, 0, 0), 3);
+
+ delta_x = rects[idx].left;
+ delta_y = rects[idx].top;
+ }
+
+ for (auto &point : landmarks) {
+ // delta_x and delta_y are a base position where circle will be displayed.
+ // point.x and point.y starts at 0 so move the base position on screen with delta_x and delta_y.
+ int x = point.x + delta_x;
+ int y = point.y + delta_y;
+ cv::circle(imgCv, cv::Point(x, y), 2, cv::Scalar(255, 0, 0), 3);
+ }
+
+ for (unsigned int idx = 0; idx < handRects.size(); ++idx) {
+ cv::rectangle(imgCv, cv::Point(handRects[idx].left, handRects[idx].top),
+ cv::Point(handRects[idx].right, handRects[idx].bottom), cv::Scalar(255, 0, 0), 3);
+
+ delta_x = handRects[idx].left;
+ delta_y = handRects[idx].top;
+ }
+
+ for (auto &point : handLandmarks) {
+ // delta_x and delta_y are a base position where circle will be displayed.
+ // point.x and point.y starts at 0 so move the base position on screen with delta_x and delta_y.
+ int x = point.x + delta_x;
+ int y = point.y + delta_y;
+
+ cv::circle(imgCv, cv::Point(x, y), 2, cv::Scalar(255, 0, 0), 3);
+ }
+
+ _user_cb(preprocessedImg.ptr, preprocessedImg.width, preprocessedImg.height, preprocessedImg.byte_per_pixel,
+ _user_data);
+
+ _result_lock.lock();
+ _result.is_valid = false;
+ _result_lock.unlock();
+ }
+
+ _result_lock.unlock();
+ }
+
+ _preprocessor->invoke();
+
+ // Set is_owned to false to prevent from releasing copied image data
+ // because the owner of copied is AirGesture so the copied will be released
+ // in performAsync function.
+ // Ps. if is_owned is true, copied image data can be released while task manager is being performed.
+ originalImg->is_owned = false;
+
+ // Make sure to release copied buffer if incoming queue isn't empty so skipped pushing the buffer.
+ // If empty then push the buffer to the incoming queue. This buffer will be released at the end of runTaskManager function.
+ if (_async_manager->pushInput(*originalImg) != SINGLEO_ERROR_NONE)
+ delete originalImg->ptr;
+}
+
+bool AirGesture::isKeyValid(std::string key)
+{
+ auto it = _result_keys.find(key);
+ if (it == _result_keys.end())
+ return false;
+
+ return true;
+}
+
+void AirGesture::add_input(BaseDataType &input_data)
+{
+ if (_inputQ.size() > 0) {
+ SINGLEO_LOGE("Only one input data is allowed.");
+ throw InvalidOperation("Only one input data is allowed.");
+ }
+
+ if (input_data._data_type != DataType::IMAGE && input_data._data_type != DataType::FILE) {
+ SINGLEO_LOGE("Only IMAGE and FILE types are allowed.");
+ throw InvalidOperation("Only IMAGE and FILE types are allowed.");
+ }
+
+ std::lock_guard<std::mutex> lock(_inputMutex);
+
+ _inputQ.push(input_data.clone());
+}
+
+void AirGesture::runTaskManager(BaseDataType &input_data)
+{
+ _taskManager->addInput(input_data);
+ _taskManager->run();
+
+ updateResult(input_data);
+}
+
+void AirGesture::perform()
+{
+ ImagePreprocessor preprocessor;
+
+ // If input service is not set, input data comes from user.
+ // In this case, get input data from _inputs.
+ if (!_input_service) {
+ std::lock_guard<std::mutex> lock(_inputMutex);
+
+ _preprocessor->addInput(*_inputQ.front());
+ _inputQ.pop();
+ } else {
+ ImageDataType input_data;
+
+ _input_service->capture(input_data);
+ _preprocessor->addInput(input_data);
+ }
+
+ runTaskManager(_preprocessor->getOutput());
+}
+
+void AirGesture::performAsync()
+{
+ if (!_input_service) {
+ SINGLEO_LOGE("This API is valid only the case that input feed service is used.");
+ throw InvalidOperation("Invalid API request.");
+ }
+
+ _input_service->streamOn();
+
+ _async_mode = true;
+ _async_manager = make_unique<AsyncManager<ImageDataType, AirGestureResult> >();
+ _async_manager->registerInvokeCb(this, [this](IService *service, BaseDataType &data) {
+ auto air_gesture = static_cast<AirGesture *>(service);
+
+ air_gesture->runTaskManager(data);
+
+ auto imageData = dynamic_cast<ImageDataType &>(data);
+
+ // A given imageData - allocated by inputFeedCb function - has been used so release it here.
+ if (!imageData.is_owned)
+ delete imageData.ptr;
+ });
+}
+
+void AirGesture::updateResult(BaseDataType &in_data)
+{
+ AirGestureResult airgesture_result;
+
+ for (auto &output : _taskManager->output()) {
+ if (output->_is_empty)
+ continue;
+
+ if (output->_type == ResultType::FACE_DETECTION) {
+ auto &rects = dynamic_cast<FdResultType &>(*output)._rects;
+
+ airgesture_result.rects = rects;
+ } else if (output->_type == ResultType::FACE_LANDMARK) {
+ auto &points = dynamic_cast<FldResultType &>(*output)._points;
+
+ airgesture_result.landmark_points = points;
+ } else if (output->_type == ResultType::GAZE_TRACKING) {
+ auto &points = dynamic_cast<GazeResultType &>(*output)._fpoints;
+
+ airgesture_result.gaze_points = points;
+ } else if (output->_type == ResultType::HAND_DETECTION) {
+ auto &rects = dynamic_cast<HdResultType &>(*output)._rects;
+
+ if (_result.preHandRects.empty())
+ airgesture_result.preHandRects = rects;
+ else
+ airgesture_result.preHandRects = _result.preHandRects;
+
+ airgesture_result.handRects = rects;
+ } else if (output->_type == ResultType::HAND_LANDMARK) {
+ auto &hand_landmark_points = dynamic_cast<HldResultType &>(*output)._points;
+
+ airgesture_result.hand_landmark_points = hand_landmark_points;
+ }
+ }
+
+ if (_mode == AirGestureMode::FACE_TRACKING) {
+ airgesture_result.is_valid = true;
+ airgesture_result.num_of_objects = 1;
+ SINGLEO_LOGD("### FACE TRACKING MODE.");
+
+ if (!airgesture_result.rects.empty()) {
+ _mode = AirGestureMode::GAZE_TRACKING;
+ _taskManager->setBypass("face_detection", true, false);
+ _taskManager->setBypass("hand_detection", false, false);
+ }
+ } else if (_mode == AirGestureMode::GAZE_TRACKING) {
+ airgesture_result.is_valid = true;
+ airgesture_result.num_of_objects = 1;
+ SINGLEO_LOGD("### GAZE TRACKING MODE.");
+
+ if (!airgesture_result.handRects.empty()) {
+ _mode = AirGestureMode::HAND_TRACKING;
+ _taskManager->setBypass("face_detection", true, false);
+ _taskManager->setBypass("gaze_tracking", true, false);
+ }
+ } else if (_mode == AirGestureMode::HAND_TRACKING) {
+ airgesture_result.is_valid = true;
+ airgesture_result.num_of_objects = 1;
+ SINGLEO_LOGD("### HAND_TRACKING MODE.");
+
+ if (airgesture_result.handRects.empty()) {
+ _mode = AirGestureMode::FACE_TRACKING;
+ _taskManager->setBypass("hand_detection", true, false);
+ _taskManager->setBypass("face_detection", false, false);
+ _taskManager->setBypass("gaze_tracking", false, false);
+ }
+ }
+
+ if (_decoder)
+ _decoder->decode(airgesture_result);
+
+ if (_async_mode) {
+ _async_manager->pushOutput(airgesture_result);
+ } else {
+ _result_lock.lock();
+ _result = airgesture_result;
+ _result_lock.unlock();
+ }
+}
+
+void AirGesture::getResultCnt(unsigned int *cnt)
+{
+ if (_async_mode) {
+ auto result = _async_manager->popOutput();
+ std::unique_lock<std::mutex> lock(_result_lock);
+
+ _result = result;
+ *cnt = static_cast<unsigned int>(_result.num_of_objects);
+ return;
+ }
+
+ std::unique_lock<std::mutex> lock(_result_lock);
+
+ *cnt = static_cast<unsigned int>(_result.num_of_objects);
+}
+
+void AirGesture::getResultFloat(unsigned int idx, std::string key, float *value)
+{
+ transform(key.begin(), key.end(), key.begin(), ::toupper);
+
+ SINGLEO_LOGD("given key = %s", key.c_str());
+
+ if (!isKeyValid(key))
+ throw InvalidParameter("A given result key is inavlid.");
+
+ std::unique_lock<std::mutex> lock(_result_lock);
+
+ switch (_result_keys[key]) {
+ case AirGestureResultType::X:
+ *value = _result.point_x;
+ break;
+ case AirGestureResultType::Y:
+ *value = _result.point_y;
+ break;
+ }
+}
+
+void AirGesture::registerUserCallback(singleo_user_cb_t user_cb, void *user_data)
+{
+ _user_cb = user_cb;
+ _user_data = user_data;
+}
+
+void AirGesture::unregisterUserCallback()
+{
+ _user_cb = nullptr;
+ _user_data = nullptr;
+}
+
+}
+}
+}
--- /dev/null
+/**
+ * Copyright (c) 2024 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "SingleoException.h"
+#include "SingleoLog.h"
+#include "TinyTrackerS.h"
+
+using namespace std;
+using namespace singleo::exception;
+
+namespace singleo
+{
+namespace services
+{
+namespace airgesture
+{
+TinyTrackerS::TinyTrackerS()
+{
+ // The output result of TinyTracker model corresponds to -10cm to 10cm,
+ // so it is necessary to convert the gaze position to the coordinates on the screen.
+ // For this purpose, we can obtain a linear equation that passes through two points (-10, 0)
+ // and (10, screen width) for the horizontal resolution of the screen and (-10, 0)
+ // and (10, screen height) for the vertical resolution.
+ // This allows us to map the output results onto the screen's coordinate system.
+ // Let's find the slope a(weight) and intercept b(intercept)
+ // a = (y2 - y1) / (x2 - x1)
+ _x_weight = _screenWidth / 20.0f;
+ _y_weight = _screenHeight / 20.0f;
+
+ // 0 = a * -10 + b ==> b = a * 10
+ _x_intercept = _x_weight * 10.0f;
+ _y_intercept = _y_weight * 10.0f;
+}
+
+void TinyTrackerS::getCurrentPosition(Rect previous, Rect current, int *x, int *y)
+{
+ // Get center position.
+ int center_x = (previous.left + previous.right) / 2;
+ int center_y = (previous.top + previous.bottom) / 2;
+ int center_x_new = (current.left + current.right) / 2;
+ int center_y_new = (current.top + current.bottom) / 2;
+
+ *x = center_x_new - center_x;
+ *y = center_y_new - center_y;
+}
+
+void TinyTrackerS::decode(ServiceBaseResultType &baseResult)
+{
+ if (baseResult._result_type != ServiceResultType::AIR_GESTURE) {
+ SINGLEO_LOGE("invalid result data type.");
+ throw InvalidParameter("invalid result data type.");
+ }
+
+ auto &result = dynamic_cast<AirGestureResult &>(baseResult);
+ if (!result.is_valid)
+ return;
+
+ if (result.gaze_points.empty())
+ return;
+
+ int delta_x = 0, delta_y = 0;
+
+ if (!result.preHandRects.empty() && !result.handRects.empty())
+ getCurrentPosition(result.preHandRects[0], result.handRects[0], &delta_x, &delta_y);
+
+ result.point_x = _x_weight * result.gaze_points[0].x + _x_intercept;
+ result.point_y = _screenHeight - (_y_weight * result.gaze_points[0].y + _y_intercept);
+ result.point_x -= delta_x;
+ result.point_y += delta_y;
+}
+
+}
+}
+}
\ No newline at end of file
#define __AUTO_ZOOM_H__
#include "IService.h"
+#include "SingleoException.h"
#include "SingleoCommonTypes.h"
#include "IInputService.h"
#include "SingleoInferenceTypes.h"
void performAsync() override;
void getResultCnt(unsigned int *cnt) override;
void getResultInt(unsigned int idx, std::string key, unsigned int *value) override;
+ void getResultFloat(unsigned int idx, std::string key, float *value) override
+ {
+ throw exception::InvalidOperation("Not support yet.");
+ }
void registerUserCallback(singleo_user_cb_t user_cb, void *user_data) override;
void unregisterUserCallback() override;
~Context()
{}
- ServiceType _service_type {};
bool _async_mode { false };
IService *_service_handle {};
};
--- /dev/null
+/**
+ * Copyright (c) 2024 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __IDECODER_H__
+#define __IDECODER_H__
+
+#include "ServiceDataType.h"
+
+namespace singleo
+{
+namespace services
+{
+class IDecoder
+{
+public:
+ virtual void decode(ServiceBaseResultType &baseResult) = 0;
+};
+
+}
+}
+
+#endif
\ No newline at end of file
virtual void performAsync() = 0;
virtual void getResultCnt(unsigned int *cnt) = 0;
virtual void getResultInt(unsigned int idx, std::string key, unsigned int *value) = 0;
+ virtual void getResultFloat(unsigned int idx, std::string key, float *value) = 0;
virtual void registerUserCallback(singleo_user_cb_t user_cb, void *user_data) = 0;
virtual void unregisterUserCallback() = 0;
};
{
// TODO
-enum class ServiceResultType { NONE, AUTO_ZOOM, FOCUS_FINDER };
+enum class ServiceResultType { NONE, AUTO_ZOOM, FOCUS_FINDER, AIR_GESTURE };
struct ServiceBaseResultType {
ServiceResultType _result_type { ServiceResultType::NONE };
void ImagePreprocessor::convertToCv(ImageDataType &image_data)
{
cv::Mat cv_image(cv::Size(image_data.width, image_data.height), CV_MAKETYPE(CV_8U, 3), image_data.ptr);
+ cv::cvtColor(cv_image, cv_image, cv::COLOR_BGR2RGB);
_cv_image = cv_image;
}
void performAsync() override;
void getResultCnt(unsigned int *cnt) override;
void getResultInt(unsigned int idx, std::string key, unsigned int *value) override;
+ void getResultFloat(unsigned int idx, std::string key, float *value) override
+ {
+ throw exception::InvalidOperation("Not supported yet.");
+ }
void registerUserCallback(singleo_user_cb_t user_cb, void *user_data) override;
void unregisterUserCallback() override;
};
return SINGLEO_ERROR_NONE;
}
+int singleo_service_get_result_float(singleo_service_h handle, unsigned int idx, const char *key, float *value)
+{
+ try {
+ auto context = static_cast<Context *>(handle);
+ context->_service_handle->getResultFloat(idx, key, value);
+ } catch (const BaseException &e) {
+ SINGLEO_LOGE("%s", e.what());
+ return e.getError();
+ }
+
+ return SINGLEO_ERROR_NONE;
+}
+
int singleo_service_register_user_callback(singleo_service_h handle, singleo_user_cb_t user_cb, void *user_data)
{
try {
INSTALL(TARGETS test_singleo_on_screen DESTINATION ${CMAKE_INSTALL_BINDIR})
ENDIF()
+SET(TEST_SOURCE_AIRGESTURE_LIST test_airgesture_on_screen.cpp)
+
+ADD_EXECUTABLE(test_singleo_airgesture ${TEST_SOURCE_AIRGESTURE_LIST})
+TARGET_COMPILE_DEFINITIONS(test_singleo_airgesture PRIVATE -DTEST_RES_PATH="${TEST_RES_PATH}")
+TARGET_INCLUDE_DIRECTORIES(test_singleo_airgesture PRIVATE ../../capi/ ../../common/include ../../log/include ../../visualizer/include)
+TARGET_LINK_LIBRARIES(test_singleo_airgesture
+ gtest gtest_main pthread singleo_service singleo_visualizer opencv_imgcodecs)
+
+INSTALL(TARGETS test_singleo_airgesture DESTINATION ${CMAKE_INSTALL_BINDIR})
+
SET(TEST_SOURCE_TASK_MANAGER_LIST test_task_manager.cpp)
ADD_EXECUTABLE(test_singleo_task_manger ${TEST_SOURCE_TASK_MANAGER_LIST})
--- /dev/null
+/**
+ * Copyright (c) 2024 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <iostream>
+#include <thread>
+#include <opencv2/opencv.hpp>
+
+#include <mutex>
+#include "gtest/gtest.h"
+#include "singleo_native_capi_internal.h"
+#include "singleo_util_visualizer_2d.h"
+#include "singleo_error.h"
+#include "SingleoCommonTypes.h"
+
+using namespace testing;
+using namespace singleo;
+using namespace std;
+
+struct Context {
+ std::mutex _mutex;
+ singleo_service_h handle {};
+ vector<Point> points;
+};
+
+void airgesture_callback(void *user_data)
+{
+ Context *context = static_cast<Context *>(user_data);
+ singleo_service_h handle = static_cast<singleo_service_h>(context->handle);
+ bool is_loop_exit = false;
+ unsigned long frame_number = 0;
+
+ while (!is_loop_exit) {
+ unsigned int cnt;
+
+ int ret = singleo_service_get_result_cnt(handle, &cnt);
+ if (ret != SINGLEO_ERROR_NONE)
+ break;
+
+ cout << "cnt = " << cnt << " frame number = " << frame_number << endl;
+
+ for (unsigned int idx = 0; idx < cnt; ++idx) {
+ float x {}, y {};
+
+ ret = singleo_service_get_result_float(handle, idx, "X", &x);
+ ASSERT_EQ(ret, SINGLEO_ERROR_NONE);
+
+ ret = singleo_service_get_result_float(handle, idx, "Y", &y);
+ ASSERT_EQ(ret, SINGLEO_ERROR_NONE);
+
+ cout << "x = " << x << " y = " << y << endl;
+
+ Point point = { static_cast<int>(x), static_cast<int>(y) };
+
+ std::unique_lock<std::mutex> lock(context->_mutex);
+ context->points.push_back(point);
+ }
+
+ if (++frame_number > 5000 && cnt > 0)
+ is_loop_exit = true;
+ }
+}
+
+void user_callback(unsigned char *buffer, unsigned int width, unsigned int height, unsigned int bytes_per_pixel,
+ void *user_data)
+{
+ Context *context = static_cast<Context *>(user_data);
+ cv::Mat result(cv::Size(width, height), CV_MAKETYPE(CV_8U, 3), buffer);
+
+ cv::Mat resized, flipped;
+
+ cv::flip(result, flipped, 1);
+ cv::resize(flipped, resized, cv::Size(1600, 900));
+
+ std::unique_lock<std::mutex> lock(context->_mutex);
+
+ if (!context->points.empty()) {
+ cv::circle(resized, cv::Point(context->points[0].x, context->points[0].y), 3, cv::Scalar(0, 255, 0), 5);
+ singleo_util_visualizer_2d(resized, NULL);
+
+ context->points.clear();
+ }
+}
+
+TEST(AirGestureAsyncOnScreenTest, InferenceRequestWithCameraInputFeedShouldBeOk)
+{
+ Context context {};
+
+ int ret = singleo_service_create("service=AirGesture, input_feed=camera, camera_id=0, fps=30, async=1",
+ &context.handle);
+ ASSERT_EQ(ret, SINGLEO_ERROR_NONE);
+
+ ret = singleo_service_register_user_callback(context.handle, user_callback, &context);
+ ASSERT_EQ(ret, SINGLEO_ERROR_NONE);
+
+ ret = singleo_service_perform(context.handle);
+ ASSERT_EQ(ret, SINGLEO_ERROR_NONE);
+
+ unique_ptr<thread> thread_handle = make_unique<thread>(&airgesture_callback, static_cast<void *>(&context));
+
+ thread_handle->join();
+
+ ret = singleo_service_unregister_user_callback(context.handle);
+ ASSERT_EQ(ret, SINGLEO_ERROR_NONE);
+
+ ret = singleo_service_destroy(context.handle);
+ ASSERT_EQ(ret, SINGLEO_ERROR_NONE);
+}
#define __SINGLEO_UTIL_RENDER_2D_H__
#include <opencv2/opencv.hpp>
-#include <mv_common.h>
#include <cmath>
#define pixfmt_fourcc(a, b, c, d) ((uint32_t)(a) | ((uint32_t)(b) << 8) | ((uint32_t)(c) << 16) | ((uint32_t)(d) << 24))
projects / platform / core / api / singleo.git / commitdiff