From: Tae-Young Chung Date: Tue, 14 May 2024 07:34:09 +0000 (+0900) Subject: Refact IInferenceFaceService, GazeEstimator, HeadPoseEstimator, FaceShapeModelManager X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=ec7b5547d540e8369d7066e5f8abd92536beea76;p=platform%2Fcore%2Fapi%2Fsingleo.git Refact IInferenceFaceService, GazeEstimator, HeadPoseEstimator, FaceShapeModelManager Change them to use FACE_COMPONENT enumeration. FaceShapeModelManager provides the given face components and IInferenceFaceService has to infer the corresponding components from given images. Change-Id: I3cd639897b7ca3d2ce1cce89277ca47064ecd1fe Signed-off-by: Tae-Young Chung --- diff --git a/inference/backends/mediavision/include/MvInferenceFaceService.h b/inference/backends/mediavision/include/MvInferenceFaceService.h index f332596..fc67d89 100644 --- a/inference/backends/mediavision/include/MvInferenceFaceService.h +++ b/inference/backends/mediavision/include/MvInferenceFaceService.h @@ -36,7 +36,7 @@ private: public: MvInferenceFaceService(std::vector &tasks); virtual ~MvInferenceFaceService() {}; - void configure() override; + void configure(const std::vector &landmarkIndices) override; void prepare() override; void invoke(BaseDataType &input) override; FaceResult &result() override; diff --git a/inference/backends/mediavision/src/MvInferenceFaceService.cpp b/inference/backends/mediavision/src/MvInferenceFaceService.cpp index f4d777f..d539593 100644 --- a/inference/backends/mediavision/src/MvInferenceFaceService.cpp +++ b/inference/backends/mediavision/src/MvInferenceFaceService.cpp @@ -56,7 +56,7 @@ MvInferenceFaceService::MvInferenceFaceService(std::vector } } -void MvInferenceFaceService::configure() +void MvInferenceFaceService::configure(const vector &landmarkIndices) { } diff --git a/inference/backends/private/include/Kalman2d.h b/inference/backends/private/include/Kalman2d.h index d9060c9..7aac13d 100644 --- a/inference/backends/private/include/Kalman2d.h +++ b/inference/backends/private/include/Kalman2d.h @@ -40,4 +40,4 @@ public: }; } } -#endif \ No newline at end of file +#endif \ No newline at end of file diff --git a/inference/backends/private/include/PrivateInferenceFaceService.h b/inference/backends/private/include/PrivateInferenceFaceService.h index 24bb348..6122ec3 100644 --- a/inference/backends/private/include/PrivateInferenceFaceService.h +++ b/inference/backends/private/include/PrivateInferenceFaceService.h @@ -58,7 +58,7 @@ private: public: PrivateInferenceFaceService(std::vector &tasks); virtual ~PrivateInferenceFaceService(); - void configure() override; + void configure(const std::vector &landmarkIndices) override; void prepare() override; void invoke(BaseDataType &input) override; FaceResult &result() override; diff --git a/inference/backends/private/src/PrivateInferenceFaceService.cpp b/inference/backends/private/src/PrivateInferenceFaceService.cpp index 2ef6fc9..edaa90a 100644 --- a/inference/backends/private/src/PrivateInferenceFaceService.cpp +++ b/inference/backends/private/src/PrivateInferenceFaceService.cpp @@ -88,32 +88,6 @@ PrivateInferenceFaceService::PrivateInferenceFaceService(std::vector(); } break; default: @@ -135,9 +109,14 @@ PrivateInferenceFaceService::~PrivateInferenceFaceService() } } -void PrivateInferenceFaceService::configure() +void PrivateInferenceFaceService::configure(const vector &landmarkIndices) { - + if (!landmarkIndices.empty() && isLandmarkDetectTask) { + for (auto &idx : landmarkIndices) { + _landmarkMappingIndex.push_back(idx); + _filterLandmarks.push_back(make_unique()); + } + } } void PrivateInferenceFaceService::prepare() diff --git a/inference/include/FaceShapeComponents.h b/inference/include/FaceShapeComponents.h new file mode 100644 index 0000000..df253de --- /dev/null +++ b/inference/include/FaceShapeComponents.h @@ -0,0 +1,33 @@ +/** + * Copyright (c) 2024 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +#ifndef __FACE_SHAPE_COMPONENTS_H__ +#define __FACE_SHAPE_COMPONENTS_H__ + +namespace singleo +{ +namespace inference +{ +enum class FACE_COMPONENTS { + NOSE_RIDGE, + NOSE_BASE, + LEFT_EYE_CORNERS, + RIGHT_EYE_CORNERS, +}; +} // inference +} // singleo + +#endif \ No newline at end of file diff --git a/inference/include/IInferenceFaceService.h b/inference/include/IInferenceFaceService.h index 744f262..4badd43 100644 --- a/inference/include/IInferenceFaceService.h +++ b/inference/include/IInferenceFaceService.h @@ -19,6 +19,7 @@ #include "FaceResult.h" #include "SingleoInferenceTypes.h" +#include "FaceShapeComponents.h" namespace singleo { @@ -29,7 +30,7 @@ class IInferenceFaceService public: virtual ~IInferenceFaceService() {}; - virtual void configure() = 0; + virtual void configure(const std::vector &landmarkIndices) = 0; virtual void prepare() = 0; virtual void invoke(BaseDataType &input) = 0; virtual FaceResult &result() = 0; diff --git a/services/smart_pointer/include/FaceShapeModelManager.h b/services/smart_pointer/include/FaceShapeModelManager.h index 578b525..c64bd20 100644 --- a/services/smart_pointer/include/FaceShapeModelManager.h +++ b/services/smart_pointer/include/FaceShapeModelManager.h @@ -19,8 +19,9 @@ #include #include -#include - +#include +#include "PoseVector.h" +#include "FaceShapeComponents.h" namespace singleo { namespace services @@ -35,11 +36,13 @@ private: void loadModelFromFile(const std::string &model_path); template T ToNumber(const std::string &text); + std::map> _faceShapeIndices; public: FaceShapeModelManager(const std::string &model_path); ~FaceShapeModelManager(); const std::vector &getFaceShape(); + const std::vector& getFaceShapeIndices(singleo::inference::FACE_COMPONENTS component); }; } // smartpointer } // services diff --git a/services/smart_pointer/include/GazeEstimator.h b/services/smart_pointer/include/GazeEstimator.h index aa98600..8c90a6e 100644 --- a/services/smart_pointer/include/GazeEstimator.h +++ b/services/smart_pointer/include/GazeEstimator.h @@ -38,6 +38,10 @@ private: std::unique_ptr _head_pose_estimator; std::vector _tasks { inference::TaskType::FACE_DETECTION, inference::TaskType::FACE_LANDMARK_DETECTION }; + const std::vector _faceComponents { singleo::inference::FACE_COMPONENTS::NOSE_RIDGE, + singleo::inference::FACE_COMPONENTS::NOSE_BASE, + singleo::inference::FACE_COMPONENTS::LEFT_EYE_CORNERS, + singleo::inference::FACE_COMPONENTS::RIGHT_EYE_CORNERS }; PoseVector _headPose; public: explicit GazeEstimator(input::InputConfigBase &config); diff --git a/services/smart_pointer/include/HeadPoseEstimator.h b/services/smart_pointer/include/HeadPoseEstimator.h index 489724b..bb507e3 100644 --- a/services/smart_pointer/include/HeadPoseEstimator.h +++ b/services/smart_pointer/include/HeadPoseEstimator.h @@ -25,6 +25,7 @@ #include #include "PoseVector.h" #include "SingleoCommonTypes.h" +#include "FaceShapeComponents.h" namespace singleo { @@ -37,6 +38,7 @@ class HeadPoseEstimator private: std::vector _landmarks_2d; std::vector _landmarks_3d; + std::vector _faceComponentIndices; cv::Mat _camera_matrix; cv::Mat _camera_dist_coeff; @@ -48,14 +50,13 @@ private: std::vector _pose_axes_3d; std::vector _pose_axes; - bool _isInit; - public: - HeadPoseEstimator(); + HeadPoseEstimator(const std::vector &components); ~HeadPoseEstimator(); PoseVector estimate(const std::vector &landmark2d); std::vector &getPoseAxes(); + std::vector &getFaceComponentsIndieces(); }; } // smartpointer } // services diff --git a/services/smart_pointer/src/FaceShapeModelManager.cpp b/services/smart_pointer/src/FaceShapeModelManager.cpp index 1c96429..bc9b748 100644 --- a/services/smart_pointer/src/FaceShapeModelManager.cpp +++ b/services/smart_pointer/src/FaceShapeModelManager.cpp @@ -21,6 +21,7 @@ #include "SingleoLog.h" using namespace std; +using namespace singleo::inference; namespace singleo { @@ -35,6 +36,10 @@ FaceShapeModelManager::FaceShapeModelManager(const std::string &model_path) throw std::invalid_argument("Invalid face shape model path"); loadModelFromFile(model_path); + _faceShapeIndices.insert(make_pair>(FACE_COMPONENTS::NOSE_RIDGE, vector{27, 28, 29, 30})); + _faceShapeIndices.insert(make_pair>(FACE_COMPONENTS::NOSE_BASE, vector{31, 32, 33, 34, 35})); + _faceShapeIndices.insert(make_pair>(FACE_COMPONENTS::LEFT_EYE_CORNERS, vector{36, 39})); + _faceShapeIndices.insert(make_pair>(FACE_COMPONENTS::RIGHT_EYE_CORNERS, vector{42, 45})); } FaceShapeModelManager::~FaceShapeModelManager() @@ -84,6 +89,12 @@ const vector &FaceShapeModelManager::getFaceShape() return _faceShape; } +const vector &FaceShapeModelManager::getFaceShapeIndices(FACE_COMPONENTS component) +{ + auto indices = _faceShapeIndices.find(component); + return indices->second; +} + template float FaceShapeModelManager::ToNumber(const string &text); } // smartpointer } // services diff --git a/services/smart_pointer/src/GazeEstimator.cpp b/services/smart_pointer/src/GazeEstimator.cpp index dd27f55..71afeea 100644 --- a/services/smart_pointer/src/GazeEstimator.cpp +++ b/services/smart_pointer/src/GazeEstimator.cpp @@ -33,15 +33,18 @@ namespace smartpointer { GazeEstimator::GazeEstimator(InputConfigBase& config) { + _head_pose_estimator = make_unique(_faceComponents); + _headPose.reset(); + + // MvInferenceServiceFactory factory; PrivateInferenceServiceFactory factory; _face_estimator = factory.createInferenceFaceService(_tasks); - _face_estimator->configure(); + _face_estimator->configure(_head_pose_estimator->getFaceComponentsIndieces()); _face_estimator->prepare(); - _head_pose_estimator = make_unique(); - _headPose.reset(); + } GazeEstimator::~GazeEstimator() diff --git a/services/smart_pointer/src/HeadPoseEstimator.cpp b/services/smart_pointer/src/HeadPoseEstimator.cpp index 41d60c2..754a04a 100644 --- a/services/smart_pointer/src/HeadPoseEstimator.cpp +++ b/services/smart_pointer/src/HeadPoseEstimator.cpp @@ -23,6 +23,7 @@ using namespace std; using namespace cv; +using namespace singleo::inference; namespace singleo { @@ -30,23 +31,26 @@ namespace services { namespace smartpointer { -HeadPoseEstimator::HeadPoseEstimator() +HeadPoseEstimator::HeadPoseEstimator(const vector &components) { FaceShapeModelManager faceShapeModelmgr("/usr/share/singleo/pdm.txt"); const vector faceShape = faceShapeModelmgr.getFaceShape(); - // for (auto &point : faceShape) - // _landmarks_3d.push_back(cv::Point3f(point.x, point.y, point.z)); - vector landmarkMappingIndex = {27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 39, 42, 45}; - for (auto &idx : landmarkMappingIndex) { - _landmarks_3d.push_back(cv::Point3f(faceShape[idx].x, faceShape[idx].y, faceShape[idx].z)); + // std::vector + for (auto &component : components){ + for (auto &index : faceShapeModelmgr.getFaceShapeIndices(component)) + { + _landmarks_3d.push_back(cv::Point3f(faceShape[index].x, faceShape[index].y, faceShape[index].z)); + _faceComponentIndices.push_back(index); + } } - // 2nd landmark is the nose's tip - _pose_axes_3d.push_back(_landmarks_3d[6]); - _pose_axes_3d.push_back(cv::Point3f(_landmarks_3d[6].x - 50, _landmarks_3d[6].y, _landmarks_3d[6].z)); - _pose_axes_3d.push_back(cv::Point3f(_landmarks_3d[6].x, _landmarks_3d[6].y - 50, _landmarks_3d[6].z)); - _pose_axes_3d.push_back(cv::Point3f(_landmarks_3d[6].x, _landmarks_3d[6].y, _landmarks_3d[6].z -50)); + // Nose base landmarks are 31, 32, 33, 34, 35 and 33 is the nose tip + int noseTipIndex = faceShapeModelmgr.getFaceShapeIndices(FACE_COMPONENTS::NOSE_BASE)[2]; + _pose_axes_3d.push_back(cv::Point3f(faceShape[noseTipIndex].x, faceShape[noseTipIndex].y, faceShape[noseTipIndex].z)); + _pose_axes_3d.push_back(cv::Point3f(faceShape[noseTipIndex].x - 50, faceShape[noseTipIndex].y, faceShape[noseTipIndex].z)); + _pose_axes_3d.push_back(cv::Point3f(faceShape[noseTipIndex].x, faceShape[noseTipIndex].y - 50, faceShape[noseTipIndex].z)); + _pose_axes_3d.push_back(cv::Point3f(faceShape[noseTipIndex].x, faceShape[noseTipIndex].y, faceShape[noseTipIndex].z -50)); _pose_axes.resize(_pose_axes_3d.size()); _landmarks_2d.resize(_landmarks_3d.size()); @@ -82,8 +86,6 @@ HeadPoseEstimator::HeadPoseEstimator() SINGLEO_LOGI("%.3f", _camera_dist_coeff.at(r, c)); } } - - _isInit = false; } HeadPoseEstimator::~HeadPoseEstimator() @@ -102,15 +104,11 @@ PoseVector HeadPoseEstimator::estimate(const vector &landmark2d) } SINGLEO_LOGI("Get 2d landmark takes %d ms", timer.check_ms()); - if (true/*!_isInit*/) { - timer.reset(); - // init - cv::solvePnPRansac(_landmarks_3d, _landmarks_2d, - _camera_matrix, _camera_dist_coeff, - _rotation_vector, _translation_vector, false, 100, 8.0F, 0.99, cv::noArray(), cv::SOLVEPNP_EPNP); - SINGLEO_LOGI("1st solvePnP takes %d ms", timer.check_ms()); - _isInit = true; - } + timer.reset(); + cv::solvePnPRansac(_landmarks_3d, _landmarks_2d, + _camera_matrix, _camera_dist_coeff, + _rotation_vector, _translation_vector, false, 100, 8.0F, 0.99, cv::noArray(), cv::SOLVEPNP_EPNP); + SINGLEO_LOGI("1st solvePnP takes %d ms", timer.check_ms()); timer.reset(); // estimate pose @@ -137,6 +135,11 @@ vector &HeadPoseEstimator::getPoseAxes() return _pose_axes; } +std::vector &HeadPoseEstimator::getFaceComponentsIndieces() +{ + return _faceComponentIndices; +} + } // smartpointer } // services } // singleo