Refact IInferenceFaceService, GazeEstimator, HeadPoseEstimator, FaceShapeModelManager

Change them to use FACE_COMPONENT enumeration. FaceShapeModelManager
provides the given face components and IInferenceFaceService has to
infer the corresponding components from given images.

Change-Id: I3cd639897b7ca3d2ce1cce89277ca47064ecd1fe
Signed-off-by: Tae-Young Chung <ty83.chung@samsung.com>

diff --git a/inference/backends/mediavision/include/MvInferenceFaceService.h b/inference/backends/mediavision/include/MvInferenceFaceService.h
index f332596d933c2c2381183f04d9e37bc5eb756ae2..fc67d89192146b463702174b4c528e65ebc02a6c 100644 (file)
--- a/inference/backends/mediavision/include/MvInferenceFaceService.h
+++ b/inference/backends/mediavision/include/MvInferenceFaceService.h
@@ -36,7 +36,7 @@ private:
 public:
        MvInferenceFaceService(std::vector<inference::TaskType> &tasks);
        virtual ~MvInferenceFaceService() {};
-       void configure() override;
+       void configure(const std::vector<int> &landmarkIndices) override;
        void prepare() override;
        void invoke(BaseDataType &input) override;
        FaceResult &result() override;

diff --git a/inference/backends/mediavision/src/MvInferenceFaceService.cpp b/inference/backends/mediavision/src/MvInferenceFaceService.cpp
index f4d777fc8647f483f685e086074fe6ad20f53661..d5395936bcb67c6f2f32117611d10378c2be1f61 100644 (file)
--- a/inference/backends/mediavision/src/MvInferenceFaceService.cpp
+++ b/inference/backends/mediavision/src/MvInferenceFaceService.cpp
@@ -56,7 +56,7 @@ MvInferenceFaceService::MvInferenceFaceService(std::vector<inference::TaskType>
     }
 }
 
-void MvInferenceFaceService::configure()
+void MvInferenceFaceService::configure(const vector<int> &landmarkIndices)
 {
 
 }

diff --git a/inference/backends/private/include/Kalman2d.h b/inference/backends/private/include/Kalman2d.h
index d9060c9aa0c8671c4d8e014b102b7803cc1531d0..7aac13db1aab91106c80cc76cab190ec6aff73e8 100644 (file)
--- a/inference/backends/private/include/Kalman2d.h
+++ b/inference/backends/private/include/Kalman2d.h
@@ -40,4 +40,4 @@ public:
 };
 }
 }
-#endif 
\ No newline at end of file
+#endif
\ No newline at end of file

diff --git a/inference/backends/private/include/PrivateInferenceFaceService.h b/inference/backends/private/include/PrivateInferenceFaceService.h
index 24bb3483a79a3663f6c64beabce1d0bd56cde718..6122ec3eda4098fa700a2bd546cd321f45f659fc 100644 (file)
--- a/inference/backends/private/include/PrivateInferenceFaceService.h
+++ b/inference/backends/private/include/PrivateInferenceFaceService.h
@@ -58,7 +58,7 @@ private:
 public:
        PrivateInferenceFaceService(std::vector<inference::TaskType> &tasks);
        virtual ~PrivateInferenceFaceService();
-       void configure() override;
+       void configure(const std::vector<int> &landmarkIndices) override;
        void prepare() override;
        void invoke(BaseDataType &input) override;
        FaceResult &result() override;

diff --git a/inference/backends/private/src/PrivateInferenceFaceService.cpp b/inference/backends/private/src/PrivateInferenceFaceService.cpp
index 2ef6fc9e9c47abde84de20b8c987a30be409927e..edaa90ac0c829b268846618c774ee24f66f4e1cd 100644 (file)
--- a/inference/backends/private/src/PrivateInferenceFaceService.cpp
+++ b/inference/backends/private/src/PrivateInferenceFaceService.cpp
@@ -88,32 +88,6 @@ PrivateInferenceFaceService::PrivateInferenceFaceService(std::vector<inference::
             ret = mv_facial_landmark_prepare(_handle);
             if (ret != MEDIA_VISION_ERROR_NONE)
                 throw runtime_error("Fail to prepare face landmark detection.");
-
-            _landmarkMappingIndex.resize(13);
-            // nose ridge
-            _landmarkMappingIndex[0] = 27;
-            _landmarkMappingIndex[1] = 28;
-            _landmarkMappingIndex[2] = 29;
-            _landmarkMappingIndex[3] = 30;
-
-            // nose base
-            _landmarkMappingIndex[4] = 31;
-            _landmarkMappingIndex[5] = 32;
-            _landmarkMappingIndex[6] = 33;
-            _landmarkMappingIndex[7] = 34;
-            _landmarkMappingIndex[8] = 35;
-
-            // left eye
-            _landmarkMappingIndex[9] = 36;
-            _landmarkMappingIndex[10] = 39;
-
-            // right eye
-            _landmarkMappingIndex[11] = 42;
-            _landmarkMappingIndex[12] = 45;
-
-            _filterLandmarks.resize(13);
-            for (auto &filt : _filterLandmarks)
-                filt = make_unique<Kalman2D>();
         }
             break;
         default:
@@ -135,9 +109,14 @@ PrivateInferenceFaceService::~PrivateInferenceFaceService()
     }
 }
 
-void PrivateInferenceFaceService::configure()
+void PrivateInferenceFaceService::configure(const vector<int> &landmarkIndices)
 {
-
+        if (!landmarkIndices.empty() && isLandmarkDetectTask) {
+            for (auto &idx : landmarkIndices) {
+                _landmarkMappingIndex.push_back(idx);
+                _filterLandmarks.push_back(make_unique<Kalman2D>());
+            }
+        }
 }
 
 void PrivateInferenceFaceService::prepare()

diff --git a/inference/include/FaceShapeComponents.h b/inference/include/FaceShapeComponents.h
new file mode 100644 (file)
index 0000000..df253de
--- /dev/null
+++ b/inference/include/FaceShapeComponents.h
@@ -0,0 +1,33 @@
+/**
+ * Copyright (c) 2024 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+#ifndef __FACE_SHAPE_COMPONENTS_H__
+#define __FACE_SHAPE_COMPONENTS_H__
+
+namespace singleo
+{
+namespace inference
+{
+enum class FACE_COMPONENTS {
+    NOSE_RIDGE,
+    NOSE_BASE,
+    LEFT_EYE_CORNERS,
+    RIGHT_EYE_CORNERS,
+};
+} // inference
+} // singleo
+
+#endif
\ No newline at end of file

diff --git a/inference/include/IInferenceFaceService.h b/inference/include/IInferenceFaceService.h
index 744f26226c8456739599f9b75bcb3b07a5848f14..4badd432889f01ce5cd40185557dd84a1f75163e 100644 (file)
--- a/inference/include/IInferenceFaceService.h
+++ b/inference/include/IInferenceFaceService.h
@@ -19,6 +19,7 @@
 
 #include "FaceResult.h"
 #include "SingleoInferenceTypes.h"
+#include "FaceShapeComponents.h"
 
 namespace singleo
 {
@@ -29,7 +30,7 @@ class IInferenceFaceService
 public:
        virtual ~IInferenceFaceService() {};
 
-       virtual void configure() = 0;
+       virtual void configure(const std::vector<int> &landmarkIndices) = 0;
        virtual void prepare() = 0;
        virtual void invoke(BaseDataType &input) = 0;
        virtual FaceResult &result() = 0;

diff --git a/services/smart_pointer/include/FaceShapeModelManager.h b/services/smart_pointer/include/FaceShapeModelManager.h
index 578b525766e315521948678e23fafda23266db64..c64bd20400d88aceeed17563e4bfc1c476052cf4 100644 (file)
--- a/services/smart_pointer/include/FaceShapeModelManager.h
+++ b/services/smart_pointer/include/FaceShapeModelManager.h
@@ -19,8 +19,9 @@
 
 #include <memory>
 #include <vector>
-#include <PoseVector.h>
-
+#include <map>
+#include "PoseVector.h"
+#include "FaceShapeComponents.h"
 namespace singleo
 {
 namespace services
@@ -35,11 +36,13 @@ private:
     void loadModelFromFile(const std::string &model_path);
     template <typename T>
     T ToNumber(const std::string &text);
+    std::map<singleo::inference::FACE_COMPONENTS, std::vector<int>> _faceShapeIndices;
 
 public:
     FaceShapeModelManager(const std::string &model_path);
     ~FaceShapeModelManager();
     const std::vector<Point3f> &getFaceShape();
+    const std::vector<int>& getFaceShapeIndices(singleo::inference::FACE_COMPONENTS component);
 };
 } // smartpointer
 } // services 

diff --git a/services/smart_pointer/include/GazeEstimator.h b/services/smart_pointer/include/GazeEstimator.h
index aa986004b2aff3d4ed6ef0d848bd514cb11f59a2..8c90a6e076113dfdf7ff22c88e77de71ee7acc24 100644 (file)
--- a/services/smart_pointer/include/GazeEstimator.h
+++ b/services/smart_pointer/include/GazeEstimator.h
@@ -38,6 +38,10 @@ private:
     std::unique_ptr<HeadPoseEstimator> _head_pose_estimator;
     std::vector<inference::TaskType> _tasks { inference::TaskType::FACE_DETECTION, inference::TaskType::FACE_LANDMARK_DETECTION };
 
+    const std::vector<singleo::inference::FACE_COMPONENTS> _faceComponents { singleo::inference::FACE_COMPONENTS::NOSE_RIDGE,
+                                                                        singleo::inference::FACE_COMPONENTS::NOSE_BASE,
+                                                                        singleo::inference::FACE_COMPONENTS::LEFT_EYE_CORNERS,
+                                                                        singleo::inference::FACE_COMPONENTS::RIGHT_EYE_CORNERS };
     PoseVector _headPose;
 public:
     explicit GazeEstimator(input::InputConfigBase &config);

diff --git a/services/smart_pointer/include/HeadPoseEstimator.h b/services/smart_pointer/include/HeadPoseEstimator.h
index 489724b6f0dd9c19c8638cb447b314f467407ac3..bb507e3f741ef311a572a2eb18a0e0c5273cc008 100644 (file)
--- a/services/smart_pointer/include/HeadPoseEstimator.h
+++ b/services/smart_pointer/include/HeadPoseEstimator.h
@@ -25,6 +25,7 @@
 #include <opencv2/calib3d.hpp>
 #include "PoseVector.h"
 #include "SingleoCommonTypes.h"
+#include "FaceShapeComponents.h"
 
 namespace singleo
 {
@@ -37,6 +38,7 @@ class HeadPoseEstimator
 private:
     std::vector<cv::Point2f> _landmarks_2d;
     std::vector<cv::Point3f> _landmarks_3d;
+    std::vector<int> _faceComponentIndices;
 
     cv::Mat _camera_matrix;
     cv::Mat _camera_dist_coeff;
@@ -48,14 +50,13 @@ private:
     std::vector<cv::Point3f> _pose_axes_3d;
     std::vector<singleo::services::smartpointer::Point2f> _pose_axes;
 
-    bool _isInit;
-
 public:
-    HeadPoseEstimator();
+    HeadPoseEstimator(const std::vector<singleo::inference::FACE_COMPONENTS> &components);
     ~HeadPoseEstimator();
 
     PoseVector estimate(const std::vector<singleo::Point> &landmark2d);
     std::vector<singleo::services::smartpointer::Point2f> &getPoseAxes();
+    std::vector<int> &getFaceComponentsIndieces();
 };
 } // smartpointer
 } // services 

diff --git a/services/smart_pointer/src/FaceShapeModelManager.cpp b/services/smart_pointer/src/FaceShapeModelManager.cpp
index 1c964291763c6323ba9e7067eaea8eb2493443df..bc9b7481b1b83787a0863265c9e6baf0b00c0523 100644 (file)
--- a/services/smart_pointer/src/FaceShapeModelManager.cpp
+++ b/services/smart_pointer/src/FaceShapeModelManager.cpp
@@ -21,6 +21,7 @@
 #include "SingleoLog.h"
 
 using namespace std;
+using namespace singleo::inference;
 
 namespace singleo
 {
@@ -35,6 +36,10 @@ FaceShapeModelManager::FaceShapeModelManager(const std::string &model_path)
         throw std::invalid_argument("Invalid face shape model path");
 
     loadModelFromFile(model_path);
+    _faceShapeIndices.insert(make_pair<FACE_COMPONENTS, vector<int>>(FACE_COMPONENTS::NOSE_RIDGE, vector<int>{27, 28, 29, 30}));
+    _faceShapeIndices.insert(make_pair<FACE_COMPONENTS, vector<int>>(FACE_COMPONENTS::NOSE_BASE, vector<int>{31, 32, 33, 34, 35}));
+    _faceShapeIndices.insert(make_pair<FACE_COMPONENTS, vector<int>>(FACE_COMPONENTS::LEFT_EYE_CORNERS, vector<int>{36, 39}));
+    _faceShapeIndices.insert(make_pair<FACE_COMPONENTS, vector<int>>(FACE_COMPONENTS::RIGHT_EYE_CORNERS, vector<int>{42, 45}));
 }
 
 FaceShapeModelManager::~FaceShapeModelManager()
@@ -84,6 +89,12 @@ const vector<Point3f> &FaceShapeModelManager::getFaceShape()
     return _faceShape;
 }
 
+const vector<int> &FaceShapeModelManager::getFaceShapeIndices(FACE_COMPONENTS component)
+{
+    auto indices = _faceShapeIndices.find(component);
+    return indices->second;
+}
+
 template float FaceShapeModelManager::ToNumber(const string &text);
 } // smartpointer
 } // services

diff --git a/services/smart_pointer/src/GazeEstimator.cpp b/services/smart_pointer/src/GazeEstimator.cpp
index dd27f558b42ff3a99cdbd2bc482e1187adcb9aa2..71afeea99f916d7efaaf49f3fbe1494e9793d236 100644 (file)
--- a/services/smart_pointer/src/GazeEstimator.cpp
+++ b/services/smart_pointer/src/GazeEstimator.cpp
@@ -33,15 +33,18 @@ namespace smartpointer
 {
 GazeEstimator::GazeEstimator(InputConfigBase& config)
 {
+    _head_pose_estimator = make_unique<HeadPoseEstimator>(_faceComponents);
+    _headPose.reset();
+
+
     // MvInferenceServiceFactory factory;
     PrivateInferenceServiceFactory factory;
     _face_estimator = factory.createInferenceFaceService(_tasks);
 
-    _face_estimator->configure();
+    _face_estimator->configure(_head_pose_estimator->getFaceComponentsIndieces());
        _face_estimator->prepare();
 
-    _head_pose_estimator = make_unique<HeadPoseEstimator>();
-    _headPose.reset();
+
 }
 
 GazeEstimator::~GazeEstimator()

diff --git a/services/smart_pointer/src/HeadPoseEstimator.cpp b/services/smart_pointer/src/HeadPoseEstimator.cpp
index 41d60c283b1239c63a79415d6d07596dddd2979f..754a04a9dcf56f431297116032fa30dda68577cd 100644 (file)
--- a/services/smart_pointer/src/HeadPoseEstimator.cpp
+++ b/services/smart_pointer/src/HeadPoseEstimator.cpp
@@ -23,6 +23,7 @@
 
 using namespace std;
 using namespace cv;
+using namespace singleo::inference;
 
 namespace singleo
 {
@@ -30,23 +31,26 @@ namespace services
 {
 namespace smartpointer
 {
-HeadPoseEstimator::HeadPoseEstimator()
+HeadPoseEstimator::HeadPoseEstimator(const vector<FACE_COMPONENTS> &components)
 {
     FaceShapeModelManager faceShapeModelmgr("/usr/share/singleo/pdm.txt");
     const vector<Point3f> faceShape = faceShapeModelmgr.getFaceShape();
 
-    // for (auto &point : faceShape)
-    //     _landmarks_3d.push_back(cv::Point3f(point.x, point.y, point.z));
-    vector<int> landmarkMappingIndex = {27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 39, 42, 45};
-    for (auto &idx : landmarkMappingIndex) {
-        _landmarks_3d.push_back(cv::Point3f(faceShape[idx].x, faceShape[idx].y, faceShape[idx].z));
+    // std::vector
+    for (auto &component : components){
+        for (auto &index : faceShapeModelmgr.getFaceShapeIndices(component))
+        {
+            _landmarks_3d.push_back(cv::Point3f(faceShape[index].x, faceShape[index].y, faceShape[index].z));
+            _faceComponentIndices.push_back(index);
+        }
     }
 
-    // 2nd landmark is the nose's tip
-    _pose_axes_3d.push_back(_landmarks_3d[6]);
-    _pose_axes_3d.push_back(cv::Point3f(_landmarks_3d[6].x - 50, _landmarks_3d[6].y, _landmarks_3d[6].z));
-    _pose_axes_3d.push_back(cv::Point3f(_landmarks_3d[6].x, _landmarks_3d[6].y - 50, _landmarks_3d[6].z));
-    _pose_axes_3d.push_back(cv::Point3f(_landmarks_3d[6].x, _landmarks_3d[6].y, _landmarks_3d[6].z -50));
+    // Nose base landmarks are 31, 32, 33, 34, 35 and 33 is the nose tip
+    int noseTipIndex = faceShapeModelmgr.getFaceShapeIndices(FACE_COMPONENTS::NOSE_BASE)[2];
+    _pose_axes_3d.push_back(cv::Point3f(faceShape[noseTipIndex].x, faceShape[noseTipIndex].y, faceShape[noseTipIndex].z));
+    _pose_axes_3d.push_back(cv::Point3f(faceShape[noseTipIndex].x - 50, faceShape[noseTipIndex].y, faceShape[noseTipIndex].z));
+    _pose_axes_3d.push_back(cv::Point3f(faceShape[noseTipIndex].x, faceShape[noseTipIndex].y - 50, faceShape[noseTipIndex].z));
+    _pose_axes_3d.push_back(cv::Point3f(faceShape[noseTipIndex].x, faceShape[noseTipIndex].y, faceShape[noseTipIndex].z -50));
     _pose_axes.resize(_pose_axes_3d.size());
 
     _landmarks_2d.resize(_landmarks_3d.size());
@@ -82,8 +86,6 @@ HeadPoseEstimator::HeadPoseEstimator()
             SINGLEO_LOGI("%.3f", _camera_dist_coeff.at<float>(r, c));
         }
     }
-
-    _isInit = false;
 }
 
 HeadPoseEstimator::~HeadPoseEstimator()
@@ -102,15 +104,11 @@ PoseVector HeadPoseEstimator::estimate(const vector<Point> &landmark2d)
     }
     SINGLEO_LOGI("Get 2d landmark takes %d ms", timer.check_ms());
 
-    if (true/*!_isInit*/) {
-        timer.reset();
-        // init
-        cv::solvePnPRansac(_landmarks_3d, _landmarks_2d,
-                    _camera_matrix, _camera_dist_coeff,
-                    _rotation_vector, _translation_vector, false, 100, 8.0F, 0.99, cv::noArray(), cv::SOLVEPNP_EPNP);
-        SINGLEO_LOGI("1st solvePnP takes %d ms", timer.check_ms());
-        _isInit = true;
-    }
+    timer.reset();
+    cv::solvePnPRansac(_landmarks_3d, _landmarks_2d,
+                _camera_matrix, _camera_dist_coeff,
+                _rotation_vector, _translation_vector, false, 100, 8.0F, 0.99, cv::noArray(), cv::SOLVEPNP_EPNP);
+    SINGLEO_LOGI("1st solvePnP takes %d ms", timer.check_ms());
 
     timer.reset();
     // estimate pose
@@ -137,6 +135,11 @@ vector<Point2f> &HeadPoseEstimator::getPoseAxes()
     return _pose_axes;
 }
 
+std::vector<int> &HeadPoseEstimator::getFaceComponentsIndieces()
+{
+    return _faceComponentIndices;
+}
+
 } // smartpointer
 } // services
 } // singleo