mv_machine_learning: introduce config file for external plugin

[Issue type] : new feature

Introduce a config file for image segmentation external plugin by adding
selfie_segmentation_plugin.json file, parsing plugin information from
the config file, and use the information.

With this patch, external plugin relevant code has no dependency of
the config file for image segmentation task group. So this patch
checks if external plugin should be used or not based on the plugin flag
value which is read from the plugin config file after reading the plugin file
name from the plugin config file.

Change-Id: I126ca1706ce021257c187f76ff54615c3d0be743
Signed-off-by: Inki Dae <inki.dae@samsung.com>

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5c904ce..b141e0b 100644 (file)
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -251,6 +251,7 @@ if (${ENABLE_ML_IMAGE_SEGMENTATION})
     )
     install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${fw_name}-image-segmentation.pc DESTINATION ${LIB_INSTALL_DIR}/pkgconfig)
     install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/mv_machine_learning/image_segmentation/meta/selfie_segmentation.json DESTINATION ${CMAKE_INSTALL_DATADIR}/${fw_name})
+       install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/mv_machine_learning/image_segmentation/meta/selfie_segmentation_plugin.json DESTINATION ${CMAKE_INSTALL_DATADIR}/${fw_name})
        list(APPEND TOTAL_REQUIRED ${PC_NAME})
        list(APPEND TOTAL_LDFLAGS ${PC_LDFLAGS})
 endif()

diff --git a/mv_machine_learning/common/include/machine_learning_config.h b/mv_machine_learning/common/include/machine_learning_config.h
index c10d46e..edf0b79 100644 (file)
--- a/mv_machine_learning/common/include/machine_learning_config.h
+++ b/mv_machine_learning/common/include/machine_learning_config.h
@@ -36,6 +36,8 @@ private:
        std::string _modelMetaFilePath;
        std::string _modelDefaultPath;
        std::string _modelLabelFilePath;
+       std::string _pluginFileName;
+       bool _usePlugin {};
        int _backendType {};
        int _targetDeviceType {};
        double _confidence_threshold {};
@@ -46,17 +48,20 @@ public:
 
        void setUserModel(const std::string &model_file, const std::string &meta_file, const std::string &label_file);
        void parseConfigFile(const std::string &configFilePath);
+       void parsePluginConfigFile(const std::string &pluginConfigFilePath);
        void parseMetaFile();
        void setBackendType(int backend_type);
        void setTargetDeviceType(int device_type);
        const std::string &getDefaultModelName() const;
        const std::string &getModelFilePath() const;
        const std::string &getLabelFilePath() const;
+       const std::string &getPluginFileName() const;
        MetaMap &getInputMetaMap() const;
        MetaMap &getOutputMetaMap() const;
        double getConfidenceThreshold() const;
        int getBackendType() const;
        int getTargetDeviceType() const;
+       bool isPluginUsed() const;
        void loadMetaFile(std::unique_ptr<MetaParser> parser);
 };
 

diff --git a/mv_machine_learning/common/include/machine_learning_key.h b/mv_machine_learning/common/include/machine_learning_key.h
index 6add66c..9283258 100644 (file)
--- a/mv_machine_learning/common/include/machine_learning_key.h
+++ b/mv_machine_learning/common/include/machine_learning_key.h
@@ -86,4 +86,16 @@
  */
 #define MV_TARGET_DEVICE_TYPE "TARGET_DEVICE_TYPE"
 
+/**
+ * @brief Defines #MV_PLUGIN_NAME
+ *        to set plugin library name for using external plugin.
+ */
+#define MV_PLUGIN_NAME "PLUGIN_NAME"
+
+/**
+ * @brief Defines #MV_USE_PLUGIN
+ *        to indicate whether extern plugin is used or not for a given task type.
+ */
+#define MV_USE_PLUGIN "USE_PLUGIN"
+
 #endif /* __MACHINE_LEARNING_KEY_H__ */

diff --git a/mv_machine_learning/common/src/machine_learning_config.cpp b/mv_machine_learning/common/src/machine_learning_config.cpp
index 50906ee..bc7ff99 100644 (file)
--- a/mv_machine_learning/common/src/machine_learning_config.cpp
+++ b/mv_machine_learning/common/src/machine_learning_config.cpp
@@ -55,6 +55,11 @@ const std::string &MachineLearningConfig::getLabelFilePath() const
        return _modelLabelFilePath;
 }
 
+const std::string &MachineLearningConfig::getPluginFileName() const
+{
+       return _pluginFileName;
+}
+
 MetaMap &MachineLearningConfig::getInputMetaMap() const
 {
        return _parser->getInputMetaMap();
@@ -80,6 +85,11 @@ int MachineLearningConfig::getTargetDeviceType() const
        return _targetDeviceType;
 }
 
+bool MachineLearningConfig::isPluginUsed() const
+{
+       return _usePlugin;
+}
+
 void MachineLearningConfig::setUserModel(const string &model_file, const string &meta_file, const string &label_file)
 {
        if (!model_file.empty())
@@ -149,6 +159,23 @@ void MachineLearningConfig::parseConfigFile(const std::string &configFilePath)
                LOGW("threshold value doesn't exist.");
 }
 
+void MachineLearningConfig::parsePluginConfigFile(const std::string &pluginConfigFilePath)
+{
+       auto config = make_unique<EngineConfig>(MV_CONFIG_PATH + pluginConfigFilePath);
+
+       int ret = config->getStringAttribute(MV_PLUGIN_NAME, &_pluginFileName);
+       if (ret != MEDIA_VISION_ERROR_NONE)
+               throw InvalidOperation("Fail to get model plugin file name");
+
+       ret = config->getStringAttribute(MV_DEFAULT_MODEL_NAME, &_defaultModelName);
+       if (ret != MEDIA_VISION_ERROR_NONE)
+               throw InvalidOperation("Fail to get default model name.");
+
+       ret = config->getBooleanAttribute(MV_USE_PLUGIN, &_usePlugin);
+       if (ret != MEDIA_VISION_ERROR_NONE)
+               throw InvalidOperation("Fail to get use_plugin value");
+}
+
 void MachineLearningConfig::loadMetaFile(unique_ptr<MetaParser> parser)
 {
        _parser = move(parser);

diff --git a/mv_machine_learning/image_segmentation/include/image_segmentation_external.h b/mv_machine_learning/image_segmentation/include/image_segmentation_external.h
index 30b9f89..4897796 100644 (file)
--- a/mv_machine_learning/image_segmentation/include/image_segmentation_external.h
+++ b/mv_machine_learning/image_segmentation/include/image_segmentation_external.h
@@ -39,7 +39,7 @@ private:
        ImageSegmentationResult _current_result;
 
 public:
-       explicit ImageSegmentationExternal(ImageSegmentationTaskType task_type);
+       ImageSegmentationExternal(ImageSegmentationTaskType task_type, const char *plugin_name);
        virtual ~ImageSegmentationExternal();
 
        void preDestroy() override;

diff --git a/mv_machine_learning/image_segmentation/include/image_segmentation_type.h b/mv_machine_learning/image_segmentation/include/image_segmentation_type.h
index 0029f64..3d9493e 100644 (file)
--- a/mv_machine_learning/image_segmentation/include/image_segmentation_type.h
+++ b/mv_machine_learning/image_segmentation/include/image_segmentation_type.h
@@ -44,8 +44,7 @@ struct ImageSegmentationResult {
 
 enum class ImageSegmentationTaskType {
        IMAGE_SEGMENTATION_TASK_NONE = 0,
-       SELFIE_SEGMENTATION_PLUGIN,
-
+       SELFIE_SEGMENTATION,
        // TODO
 };
 

diff --git a/mv_machine_learning/image_segmentation/include/selfie_segmentation_adapter.h b/mv_machine_learning/image_segmentation/include/selfie_segmentation_adapter.h
index 86c797c..3984d5a 100644 (file)
--- a/mv_machine_learning/image_segmentation/include/selfie_segmentation_adapter.h
+++ b/mv_machine_learning/image_segmentation/include/selfie_segmentation_adapter.h
@@ -35,8 +35,9 @@ private:
        std::shared_ptr<MachineLearningConfig> _config;
        T _source;
        const std::string _config_file_name = "selfie_segmentation.json";
+       const std::string _plugin_config_file_name = "selfie_segmentation_plugin.json";
 
-       void create(const std::string &model_name);
+       void create(std::string model_name = "");
        template<typename U> void create(ImageSegmentationTaskType task_type);
        ImageSegmentationTaskType convertToTaskType(std::string model_name);
 

diff --git a/mv_machine_learning/image_segmentation/meta/selfie_segmentation.json b/mv_machine_learning/image_segmentation/meta/selfie_segmentation.json
index 61287bf..90e74ca 100644 (file)
--- a/mv_machine_learning/image_segmentation/meta/selfie_segmentation.json
+++ b/mv_machine_learning/image_segmentation/meta/selfie_segmentation.json
@@ -12,12 +12,17 @@
             "value" : ""
         },
         {
+            "name"  : "MODEL_LABEL_FILE_NAME",
+            "type"  : "string",
+            "value" : ""
+        },
+        {
             "name"  : "DEFAULT_MODEL_NAME",
             "type"  : "string",
-            "value" : "SELFIE_SEGMENTATION_PLUGIN"
+            "value" : "SELFIE_SEGMENTATION"
         },
         {
-            "name"  : "META_FILE_NAME",
+            "name"  : "MODEL_META_FILE_NAME",
             "type"  : "string",
             "value" : ""
         },

diff --git a/mv_machine_learning/image_segmentation/meta/selfie_segmentation_plugin.json b/mv_machine_learning/image_segmentation/meta/selfie_segmentation_plugin.json
new file mode 100644 (file)
index 0000000..46a871c
--- /dev/null
+++ b/mv_machine_learning/image_segmentation/meta/selfie_segmentation_plugin.json
@@ -0,0 +1,20 @@
+{
+    "attributes":
+    [
+        {
+            "name" : "PLUGIN_NAME",
+            "type" : "string",
+            "value" : "libimage_segmentation_plugin.so"
+        },
+        {
+            "name"  : "DEFAULT_MODEL_NAME",
+            "type"  : "string",
+            "value" : "SELFIE_SEGMENTATION"
+        },
+        {
+            "name"  : "USE_PLUGIN",
+            "type"  : "boolean",
+            "value" : true
+        }
+    ]
+}

diff --git a/mv_machine_learning/image_segmentation/src/image_segmentation_external.cpp b/mv_machine_learning/image_segmentation/src/image_segmentation_external.cpp
index d88ec01..0e1a8d7 100644 (file)
--- a/mv_machine_learning/image_segmentation/src/image_segmentation_external.cpp
+++ b/mv_machine_learning/image_segmentation/src/image_segmentation_external.cpp
@@ -32,15 +32,12 @@ namespace machine_learning
 typedef IImageSegmentation *create_t(ImageSegmentationTaskType task_type);
 typedef void destroy_t(IImageSegmentation *);
 
-ImageSegmentationExternal::ImageSegmentationExternal(ImageSegmentationTaskType task_type)
+ImageSegmentationExternal::ImageSegmentationExternal(ImageSegmentationTaskType task_type, const char *plugin_name)
                : _plugin_handle(), _image_segmentation_plugin()
 {
-       // TODO. plugin module name will be given by configuration file later.
-       const char *module_name = "libimage_segmentation_plugin.so";
-
        // Load external plugin library.
-       LOGI("lib: %s", module_name);
-       _plugin_handle = dlopen(module_name, RTLD_NOW);
+       LOGI("lib: %s", plugin_name);
+       _plugin_handle = dlopen(plugin_name, RTLD_NOW);
 
        if (!_plugin_handle)
                throw InvalidOperation("Fail to open plugin library.");

diff --git a/mv_machine_learning/image_segmentation/src/selfie_segmentation_adapter.cpp b/mv_machine_learning/image_segmentation/src/selfie_segmentation_adapter.cpp
index 2c97801..5a178a2 100644 (file)
--- a/mv_machine_learning/image_segmentation/src/selfie_segmentation_adapter.cpp
+++ b/mv_machine_learning/image_segmentation/src/selfie_segmentation_adapter.cpp
@@ -31,7 +31,12 @@ namespace machine_learning
 template<typename T, typename V> ImageSegmentationAdapter<T, V>::ImageSegmentationAdapter() : _source()
 {
        _config = make_shared<MachineLearningConfig>();
-       _config->parseConfigFile(_config_file_name);
+
+       // If the model type needs external plugin then bypass to load the meta file and just create the external plugin.
+       // In this case, external plugin will use its own meta file approach regardless of Mediavision's one.
+       _config->parsePluginConfigFile(_plugin_config_file_name);
+       if (!_config->isPluginUsed())
+               _config->parseConfigFile(_config_file_name);
 
        create(_config->getDefaultModelName());
 }
@@ -45,19 +50,23 @@ template<typename T, typename V>
 template<typename U>
 void ImageSegmentationAdapter<T, V>::create(ImageSegmentationTaskType task_type)
 {
-       switch (task_type) {
-       case ImageSegmentationTaskType::SELFIE_SEGMENTATION_PLUGIN:
-               _selfie_segmentation = make_unique<ImageSegmentationExternal>(task_type);
-               break;
-       default:
-               throw InvalidOperation("Invalid image segmentation task type.");
-       }
-       // TODO.
+       // TODO. add switch-case statement here for Mediavision own task types.
 }
 
-template<typename T, typename V> void ImageSegmentationAdapter<T, V>::create(const string &model_name)
+template<typename T, typename V> void ImageSegmentationAdapter<T, V>::create(std::string model_name)
 {
-       ImageSegmentationTaskType task_type = convertToTaskType(model_name);
+       if (model_name.empty())
+               model_name = _config->getDefaultModelName();
+
+       auto task_type = convertToTaskType(model_name);
+
+       if (_config->isPluginUsed()) {
+               auto plugin_name = _config->getPluginFileName();
+
+               _selfie_segmentation = make_unique<ImageSegmentationExternal>(task_type, plugin_name.c_str());
+               return;
+       }
+
        _config->loadMetaFile(make_unique<ImageSegmentationParser>(static_cast<int>(task_type)));
        mv_inference_data_type_e dataType = _config->getInputMetaMap().begin()->second->dataType;
 
@@ -81,8 +90,8 @@ ImageSegmentationTaskType ImageSegmentationAdapter<T, V>::convertToTaskType(stri
 
        transform(model_name.begin(), model_name.end(), model_name.begin(), ::toupper);
 
-       if (model_name == "SELFIE_SEGMENTATION_PLUGIN")
-               return ImageSegmentationTaskType::SELFIE_SEGMENTATION_PLUGIN;
+       if (model_name == "SELFIE_SEGMENTATION")
+               return ImageSegmentationTaskType::SELFIE_SEGMENTATION;
 
        throw InvalidParameter("Invalid selfie segmentation model name.");
 }

diff --git a/packaging/capi-media-vision.spec b/packaging/capi-media-vision.spec
index 24af792..fea0e32 100644 (file)
--- a/packaging/capi-media-vision.spec
+++ b/packaging/capi-media-vision.spec
@@ -421,6 +421,7 @@ find . -name '*.gcno' -not -path "./test/*" -not -path "./mv_machine_learning/*"
 %endif
 %if "%{enable_ml_image_segmentation}" == "1"
 %{_datadir}/%{name}/selfie_segmentation.json
+%{_datadir}/%{name}/selfie_segmentation_plugin.json
 %{_libdir}/libmv_image_segmentation.so
 %endif