target_include_directories(${PROJECT_NAME} PUBLIC ${${PROJECT_NAME}_DEP_INCLUDE_DIRS})
install(TARGETS ${PROJECT_NAME} DESTINATION ${LIB_INSTALL_DIR})
install(
- FILES ${PROJECT_SOURCE_DIR}/include/machine_learning_exception.h
- ${PROJECT_SOURCE_DIR}/include/machine_learning_config.h
- ${PROJECT_SOURCE_DIR}/include/MachineLearningType.h
- ${PROJECT_SOURCE_DIR}/meta/include/machine_learning_preprocess.h
+ FILES ${PROJECT_SOURCE_DIR}/include/MvMlException.h
+ ${PROJECT_SOURCE_DIR}/include/MvMlConfig.h
+ ${PROJECT_SOURCE_DIR}/include/mv_ml_types.h
+ ${PROJECT_SOURCE_DIR}/meta/include/MvMlPreprocess.h
DESTINATION include/media
)
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __ASYNC_MANAGER_H__
+#define __ASYNC_MANAGER_H__
+
+#include <queue>
+#include <thread>
+#include <mutex>
+#include <condition_variable>
+#include <atomic>
+#include <functional>
+#include <algorithm>
+#include <chrono>
+
+#include <mv_common.h>
+#include <mv_inference_type.h>
+#include "MvMlException.h"
+
+#ifdef LOG_TAG
+#undef LOG_TAG
+#endif
+
+#define LOG_TAG "TIZEN_MEDIA_VISION"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T> struct AsyncInputQueue {
+ unsigned long frame_number {};
+ std::vector<std::vector<T> > inputs;
+};
+
+template<typename T, typename R> class AsyncManager
+{
+private:
+ using CallbackType = std::function<void()>;
+
+ const std::chrono::seconds _INPUT_QUEUE_WAIT_TIME_SEC { 1 };
+ const std::chrono::seconds _OUTPUT_QUEUE_WAIT_TIME_SEC { 3 };
+ std::queue<AsyncInputQueue<T> > _incoming_queue;
+ std::queue<R> _outgoing_queue;
+ std::mutex _incoming_queue_mutex;
+ std::mutex _outgoing_queue_mutex;
+ std::unique_ptr<std::thread> _thread_handle;
+ std::atomic<bool> _exit_thread { false };
+ std::condition_variable _outgoing_queue_event;
+ std::condition_variable _incoming_queue_event;
+ unsigned long _input_frame_number {};
+ CallbackType _callback;
+
+ void pushToInput(AsyncInputQueue<T> &inputQueue)
+ {
+ std::lock_guard<std::mutex> lock(_incoming_queue_mutex);
+
+ _incoming_queue.push(inputQueue);
+ _incoming_queue_event.notify_one();
+ }
+ bool isInputQueueEmpty()
+ {
+ std::lock_guard<std::mutex> lock(_incoming_queue_mutex);
+
+ return _incoming_queue.empty();
+ }
+ R popFromOutput()
+ {
+ std::lock_guard<std::mutex> lock(_outgoing_queue_mutex);
+ R output = _outgoing_queue.front();
+
+ _outgoing_queue.pop();
+
+ return output;
+ }
+ void inferenceThreadLoop()
+ {
+ // If user called destroy API then this thread loop will be terminated.
+ while (!_exit_thread) {
+ // Wait for input data.
+ auto inputExist = waitforInputQueue();
+ if (!inputExist) {
+ LOGW("Input queue is empty");
+ break;
+ }
+
+ // Exit this thread loop if user called destroy API while in thread loop.
+ if (_exit_thread)
+ break;
+
+ // What the callback function has to do is to,
+ // 1. Get input data from input queue.
+ // 2. Run inference.
+ // 3. Put output data to output queue.
+ _callback();
+ }
+
+ // waitforOutputQueue function could wait for the notify event after while loop is exited.
+ // So make sure to call notify_one() here.
+ _outgoing_queue_event.notify_one();
+ }
+ void invoke()
+ {
+ if (!_thread_handle)
+ _thread_handle = std::make_unique<std::thread>(&AsyncManager::inferenceThreadLoop, this);
+ }
+ bool waitforInputQueue()
+ {
+ std::unique_lock<std::mutex> lock(_incoming_queue_mutex);
+
+ auto result = _incoming_queue_event.wait_for(lock, _INPUT_QUEUE_WAIT_TIME_SEC,
+ [this] { return !_incoming_queue.empty(); });
+ if (!result) {
+ LOGW("Waiting for input queue has been timed out.");
+ return false;
+ }
+
+ return true;
+ }
+ bool isOutputQueueEmpty()
+ {
+ std::lock_guard<std::mutex> lock(_outgoing_queue_mutex);
+
+ return _outgoing_queue.empty();
+ }
+ void waitforOutputQueue()
+ {
+ std::unique_lock<std::mutex> lock(_outgoing_queue_mutex);
+
+ if (!_outgoing_queue_event.wait_for(lock, _OUTPUT_QUEUE_WAIT_TIME_SEC, [this] {
+ if (_exit_thread)
+ throw exception::InvalidOperation("thread is exited already.");
+
+ return !_outgoing_queue.empty();
+ })) {
+ throw exception::InvalidOperation("Waiting for output queue has been timed out.");
+ }
+ }
+
+public:
+ AsyncManager(const CallbackType &cb) : _callback(cb)
+ {}
+ virtual ~AsyncManager() = default;
+
+ bool isWorking()
+ {
+ if (_exit_thread)
+ return false;
+
+ return true;
+ }
+ void stop()
+ {
+ // Make sure to wait for the completion of all inference requests in the incoming queue.
+ _exit_thread = true;
+
+ _thread_handle->join();
+ _thread_handle = nullptr;
+
+ std::lock_guard<std::mutex> lock(_outgoing_queue_mutex);
+ std::queue<R> empty;
+
+ swap(_outgoing_queue, empty);
+ }
+ AsyncInputQueue<T> popFromInput()
+ {
+ std::lock_guard<std::mutex> lock(_incoming_queue_mutex);
+ AsyncInputQueue<T> inputQueue = _incoming_queue.front();
+
+ _incoming_queue.pop();
+
+ return inputQueue;
+ }
+ void pushToOutput(R &output)
+ {
+ std::lock_guard<std::mutex> lock(_outgoing_queue_mutex);
+
+ _outgoing_queue.push(output);
+ _outgoing_queue_event.notify_one();
+ }
+ void push(std::vector<std::vector<T> > &inputs)
+ {
+ _input_frame_number++;
+
+ if (!isInputQueueEmpty()) {
+ LOGD("input frame number(%ld) has been skipped.", _input_frame_number);
+ return;
+ }
+
+ AsyncInputQueue<T> in_queue = { _input_frame_number, inputs };
+
+ pushToInput(in_queue);
+
+ LOGD("Pushed : input frame number = %lu", in_queue.frame_number);
+
+ invoke();
+ }
+ R pop()
+ {
+ waitforOutputQueue();
+
+ return popFromOutput();
+ }
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __CONTEXT_H__
+#define __CONTEXT_H__
+
+#include <map>
+#include "ITask.h"
+
+namespace mediavision
+{
+namespace common
+{
+class Context
+{
+public:
+ Context()
+ {}
+ ~Context()
+ {}
+
+ std::map<std::string, ITask *> __tasks;
+};
+} // namespace
+} // namespace
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __ITASK_H__
+#define __ITASK_H__
+
+#include "mv_ml_types.h"
+#include <string>
+
+namespace mediavision
+{
+namespace common
+{
+class ITask
+{
+public:
+ virtual ~ITask() {};
+ virtual void setModelInfo(const std::string &model_file, const std::string &meta_file,
+ const std::string &label_file, const std::string &model_name = "") = 0;
+ virtual void setEngineInfo(const std::string &engine_type, const std::string &device_type) = 0;
+ virtual void configure() = 0;
+ virtual unsigned int getNumberOfEngines() = 0;
+ virtual const std::string &getEngineType(unsigned int engine_index) = 0;
+ virtual unsigned int getNumberOfDevices(const std::string &engine_type) = 0;
+ virtual const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) = 0;
+ virtual void prepare() = 0;
+ virtual void perform(mediavision::machine_learning::InputBaseType &input) = 0;
+ virtual void performAsync(mediavision::machine_learning::InputBaseType &input) = 0;
+ virtual mediavision::machine_learning::OutputBaseType &getOutput() = 0;
+ virtual mediavision::machine_learning::OutputBaseType &getOutputCache() = 0;
+};
+} // namespace
+} // namespace
+
+#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __MACHINE_LEARNING_NATIVE_H__
-#define __MACHINE_LEARNING_NATIVE_H__
-
-#include <memory>
-#include <string>
-
-#include "context.h"
-#include "itask.h"
-#include "mv_private.h"
-
-#include "MachineLearningType.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-void *machine_learning_native_create();
-void machine_learning_native_add(void *handle, const std::string &task_name, mediavision::common::ITask *task);
-void machine_learning_native_destory(void *handle);
-void machine_learning_native_configure(void *handle, const std::string &task_name);
-void machine_learning_native_prepare(void *handle, const std::string &task_name);
-void machine_learning_native_inference(void *handle, const std::string &task_name, InputBaseType &input);
-void machine_learning_native_inference_async(void *handle, const std::string &task_name, InputBaseType &input);
-OutputBaseType &machine_learning_native_get_result(void *handle, const std::string &task_name);
-OutputBaseType &machine_learning_native_get_result_cache(void *handle, const std::string &task_name);
-void machine_learning_native_set_model(void *handle, const std::string &task_name, const char *model_file,
- const char *meta_file, const char *label_file, const char *model_name = "");
-void machine_learning_native_set_engine(void *handle, const std::string &task_name, const char *backend_type,
- const char *device_type);
-void machine_learning_native_get_engine_count(void *handle, const std::string &task_name, unsigned int *engine_count);
-void machine_learning_native_get_engine_type(void *handle, const std::string &task_name,
- const unsigned int engine_index, char **engine_type);
-void machine_learning_native_get_device_count(void *handle, const std::string &task_name, const char *engine_type,
- unsigned int *device_count);
-void machine_learning_native_get_device_type(void *handle, const std::string &task_name, const char *engine_type,
- const unsigned int device_index, char **device_type);
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __MACHINE_LEARNING_TYPE_H__
-#define __MACHINE_LEARNING_TYPE_H__
-
-#include <mv_common.h>
-#include <mv_inference_type.h>
-
-namespace mediavision
-{
-namespace machine_learning
-{
-struct InputBaseType {
- mv_source_h inference_src {};
- InputBaseType(mv_source_h src = NULL) : inference_src(src)
- {}
- virtual ~InputBaseType()
- {}
-};
-
-struct OutputBaseType {
- unsigned long frame_number {};
- virtual ~OutputBaseType()
- {}
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __MV_ML_CONFIG_H__
+#define __MV_ML_CONFIG_H__
+
+#include <mv_common.h>
+#include "mv_private.h"
+#include "EngineConfig.h"
+
+#include "MetaParser.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+class Config
+{
+private:
+ std::unique_ptr<MetaParser> _parser;
+ std::string _defaultModelName;
+ std::string _modelFilePath;
+ std::string _modelMetaFilePath;
+ std::string _modelDefaultPath;
+ std::string _modelLabelFilePath;
+ std::string _pluginFileName;
+ bool _usePlugin {};
+ int _backendType {};
+ int _targetDeviceType {};
+ double _confidence_threshold {};
+
+public:
+ Config();
+ virtual ~Config() = default;
+
+ void setUserModel(const std::string &model_file, const std::string &meta_file, const std::string &label_file);
+ void parseConfigFile(const std::string &configFilePath);
+ void parsePluginConfigFile(const std::string &pluginConfigFilePath);
+ void parseMetaFile();
+ void setBackendType(int backend_type);
+ void setTargetDeviceType(int device_type);
+ const std::string &getDefaultModelName() const;
+ const std::string &getModelFilePath() const;
+ const std::string &getLabelFilePath() const;
+ const std::string &getPluginFileName() const;
+ MetaMap &getInputMetaMap() const;
+ MetaMap &getOutputMetaMap() const;
+ double getConfidenceThreshold() const;
+ int getBackendType() const;
+ int getTargetDeviceType() const;
+ bool isPluginUsed() const;
+ void loadMetaFile(std::unique_ptr<MetaParser> parser);
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __MV_ML_EXCEPTION_H__
+#define __MV_ML_EXCEPTION_H__
+
+#include <exception>
+#include <string>
+
+#include <mv_common.h>
+
+namespace mediavision
+{
+namespace machine_learning
+{
+namespace exception
+{
+class BaseException : public std::exception
+{
+private:
+ std::string _msg;
+ int _errorType;
+
+public:
+ BaseException(std::string msg, int errorType) : _msg(msg), _errorType(errorType)
+ {}
+ ~BaseException() override = default;
+
+ const char *what() const noexcept override
+ {
+ return _msg.c_str();
+ }
+ int getError() const
+ {
+ return _errorType;
+ }
+};
+
+class InvalidParameter : public BaseException
+{
+public:
+ InvalidParameter(std::string msg) : BaseException("Invalid parameter: " + msg, MEDIA_VISION_ERROR_INVALID_PARAMETER)
+ {}
+ ~InvalidParameter() override = default;
+};
+
+class InvalidOperation : public BaseException
+{
+public:
+ InvalidOperation(std::string msg, int errorType = MEDIA_VISION_ERROR_INVALID_OPERATION)
+ : BaseException("Invalid operation: " + msg, errorType)
+ {}
+ ~InvalidOperation() final = default;
+};
+
+class OutOfMemory : public BaseException
+{
+public:
+ OutOfMemory(std::string msg) : BaseException("Out of memory: " + msg, MEDIA_VISION_ERROR_OUT_OF_MEMORY)
+ {}
+ ~OutOfMemory() final = default;
+};
+
+class NoData : public BaseException
+{
+public:
+ NoData(std::string msg) : BaseException("No Data: " + msg, MEDIA_VISION_ERROR_NO_DATA)
+ {}
+ ~NoData() final = default;
+};
+
+}; // Exception
+}; // MachineLearning
+}; // Mediavision
+
+#endif // __MV_ML_EXCEPTION_H__
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __ASYNC_MANAGER_H__
-#define __ASYNC_MANAGER_H__
-
-#include <queue>
-#include <thread>
-#include <mutex>
-#include <condition_variable>
-#include <atomic>
-#include <functional>
-#include <algorithm>
-#include <chrono>
-
-#include <mv_common.h>
-#include <mv_inference_type.h>
-#include "machine_learning_exception.h"
-
-#ifdef LOG_TAG
-#undef LOG_TAG
-#endif
-
-#define LOG_TAG "TIZEN_MEDIA_VISION"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T> struct AsyncInputQueue {
- unsigned long frame_number {};
- std::vector<std::vector<T> > inputs;
-};
-
-template<typename T, typename R> class AsyncManager
-{
-private:
- using CallbackType = std::function<void()>;
-
- const std::chrono::seconds _INPUT_QUEUE_WAIT_TIME_SEC { 1 };
- const std::chrono::seconds _OUTPUT_QUEUE_WAIT_TIME_SEC { 3 };
- std::queue<AsyncInputQueue<T> > _incoming_queue;
- std::queue<R> _outgoing_queue;
- std::mutex _incoming_queue_mutex;
- std::mutex _outgoing_queue_mutex;
- std::unique_ptr<std::thread> _thread_handle;
- std::atomic<bool> _exit_thread { false };
- std::condition_variable _outgoing_queue_event;
- std::condition_variable _incoming_queue_event;
- unsigned long _input_frame_number {};
- CallbackType _callback;
-
- void pushToInput(AsyncInputQueue<T> &inputQueue)
- {
- std::lock_guard<std::mutex> lock(_incoming_queue_mutex);
-
- _incoming_queue.push(inputQueue);
- _incoming_queue_event.notify_one();
- }
- bool isInputQueueEmpty()
- {
- std::lock_guard<std::mutex> lock(_incoming_queue_mutex);
-
- return _incoming_queue.empty();
- }
- R popFromOutput()
- {
- std::lock_guard<std::mutex> lock(_outgoing_queue_mutex);
- R output = _outgoing_queue.front();
-
- _outgoing_queue.pop();
-
- return output;
- }
- void inferenceThreadLoop()
- {
- // If user called destroy API then this thread loop will be terminated.
- while (!_exit_thread) {
- // Wait for input data.
- auto inputExist = waitforInputQueue();
- if (!inputExist) {
- LOGW("Input queue is empty");
- break;
- }
-
- // Exit this thread loop if user called destroy API while in thread loop.
- if (_exit_thread)
- break;
-
- // What the callback function has to do is to,
- // 1. Get input data from input queue.
- // 2. Run inference.
- // 3. Put output data to output queue.
- _callback();
- }
-
- // waitforOutputQueue function could wait for the notify event after while loop is exited.
- // So make sure to call notify_one() here.
- _outgoing_queue_event.notify_one();
- }
- void invoke()
- {
- if (!_thread_handle)
- _thread_handle = std::make_unique<std::thread>(&AsyncManager::inferenceThreadLoop, this);
- }
- bool waitforInputQueue()
- {
- std::unique_lock<std::mutex> lock(_incoming_queue_mutex);
-
- auto result = _incoming_queue_event.wait_for(lock, _INPUT_QUEUE_WAIT_TIME_SEC,
- [this] { return !_incoming_queue.empty(); });
- if (!result) {
- LOGW("Waiting for input queue has been timed out.");
- return false;
- }
-
- return true;
- }
- bool isOutputQueueEmpty()
- {
- std::lock_guard<std::mutex> lock(_outgoing_queue_mutex);
-
- return _outgoing_queue.empty();
- }
- void waitforOutputQueue()
- {
- std::unique_lock<std::mutex> lock(_outgoing_queue_mutex);
-
- if (!_outgoing_queue_event.wait_for(lock, _OUTPUT_QUEUE_WAIT_TIME_SEC, [this] {
- if (_exit_thread)
- throw exception::InvalidOperation("thread is exited already.");
-
- return !_outgoing_queue.empty();
- })) {
- throw exception::InvalidOperation("Waiting for output queue has been timed out.");
- }
- }
-
-public:
- AsyncManager(const CallbackType &cb) : _callback(cb)
- {}
- virtual ~AsyncManager() = default;
-
- bool isWorking()
- {
- if (_exit_thread)
- return false;
-
- return true;
- }
- void stop()
- {
- // Make sure to wait for the completion of all inference requests in the incoming queue.
- _exit_thread = true;
-
- _thread_handle->join();
- _thread_handle = nullptr;
-
- std::lock_guard<std::mutex> lock(_outgoing_queue_mutex);
- std::queue<R> empty;
-
- swap(_outgoing_queue, empty);
- }
- AsyncInputQueue<T> popFromInput()
- {
- std::lock_guard<std::mutex> lock(_incoming_queue_mutex);
- AsyncInputQueue<T> inputQueue = _incoming_queue.front();
-
- _incoming_queue.pop();
-
- return inputQueue;
- }
- void pushToOutput(R &output)
- {
- std::lock_guard<std::mutex> lock(_outgoing_queue_mutex);
-
- _outgoing_queue.push(output);
- _outgoing_queue_event.notify_one();
- }
- void push(std::vector<std::vector<T> > &inputs)
- {
- _input_frame_number++;
-
- if (!isInputQueueEmpty()) {
- LOGD("input frame number(%ld) has been skipped.", _input_frame_number);
- return;
- }
-
- AsyncInputQueue<T> in_queue = { _input_frame_number, inputs };
-
- pushToInput(in_queue);
-
- LOGD("Pushed : input frame number = %lu", in_queue.frame_number);
-
- invoke();
- }
- R pop()
- {
- waitforOutputQueue();
-
- return popFromOutput();
- }
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __COMMON_H__
+#define __COMMON_H__
+
+#include <string>
+#include <vector>
+#include <map>
+
+#include "mv_inference_type.h"
+
+namespace mediavision
+{
+namespace common
+{
+static std::map<std::string, int> gBackendTypeTable = {
+ { "OPENCV", MV_INFERENCE_BACKEND_OPENCV }, { "TFLITE", MV_INFERENCE_BACKEND_TFLITE },
+ { "ARMNN", MV_INFERENCE_BACKEND_ARMNN }, { "ONE", MV_INFERENCE_BACKEND_ONE },
+ { "NNTRAINER", MV_INFERENCE_BACKEND_NNTRAINER }, { "SNPE", MV_INFERENCE_BACKEND_SNPE }
+};
+
+static std::map<std::string, int> gDeviceTypeTable = { { "CPU", MV_INFERENCE_TARGET_DEVICE_CPU },
+ { "GPU", MV_INFERENCE_TARGET_DEVICE_GPU },
+ { "NPU", MV_INFERENCE_TARGET_DEVICE_CUSTOM } };
+
+static int GetBackendType(std::string backend_type)
+{
+ auto item = gBackendTypeTable.find(backend_type);
+ if (item != gBackendTypeTable.end())
+ return item->second;
+
+ return MEDIA_VISION_ERROR_INVALID_PARAMETER;
+}
+
+static int GetDeviceType(std::string device_type)
+{
+ auto item = gDeviceTypeTable.find(device_type);
+ if (item != gDeviceTypeTable.end())
+ return item->second;
+
+ return MEDIA_VISION_ERROR_INVALID_PARAMETER;
+}
+
+} // namespace
+} // namespace
+
+#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __CONTEXT_H__
-#define __CONTEXT_H__
-
-#include <map>
-#include "itask.h"
-
-namespace mediavision
-{
-namespace common
-{
-class Context
-{
-public:
- Context()
- {}
- ~Context()
- {}
-
- std::map<std::string, ITask *> __tasks;
-};
-} // namespace
-} // namespace
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __ITASK_H__
-#define __ITASK_H__
-
-#include "MachineLearningType.h"
-#include <string>
-
-namespace mediavision
-{
-namespace common
-{
-class ITask
-{
-public:
- virtual ~ITask() {};
- virtual void setModelInfo(const std::string &model_file, const std::string &meta_file,
- const std::string &label_file, const std::string &model_name = "") = 0;
- virtual void setEngineInfo(const std::string &engine_type, const std::string &device_type) = 0;
- virtual void configure() = 0;
- virtual unsigned int getNumberOfEngines() = 0;
- virtual const std::string &getEngineType(unsigned int engine_index) = 0;
- virtual unsigned int getNumberOfDevices(const std::string &engine_type) = 0;
- virtual const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) = 0;
- virtual void prepare() = 0;
- virtual void perform(mediavision::machine_learning::InputBaseType &input) = 0;
- virtual void performAsync(mediavision::machine_learning::InputBaseType &input) = 0;
- virtual mediavision::machine_learning::OutputBaseType &getOutput() = 0;
- virtual mediavision::machine_learning::OutputBaseType &getOutputCache() = 0;
-};
-} // namespace
-} // namespace
-
-#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __KEYS_H__
+#define __KEYS_H__
+
+/**
+ * @brief Defines #MV_MODEL_DEFAULT_PATH
+ * to set the model default path.
+ *
+ * @since_tizen 8.0
+ */
+#define MV_MODEL_DEFAULT_PATH "MODEL_DEFAULT_PATH"
+
+/**
+ * @brief Defines #MV_MODEL_FILE_NAME
+ * to set the model file path.
+ *
+ * @since_tizen 8.0
+ */
+#define MV_MODEL_FILE_NAME "MODEL_FILE_NAME"
+
+/**
+ * @brief Defines #MV_DEFAULT_MODEL_NAME
+ * to set the default model name.
+ *
+ * @since_tizen 8.0
+ */
+#define MV_DEFAULT_MODEL_NAME "DEFAULT_MODEL_NAME"
+
+/**
+ * @brief Defines #MV_MODEL_META_FILE_NAME to set inference
+ * models's metadata file attribute of the engine configuration.
+ * @details The file includes inference model's metadata such as input and output
+ * node names, input tensor's width and height,
+ * mean and standard deviation values for pre-processing.
+ *
+ * @since_tizen 8.0
+ */
+#define MV_MODEL_META_FILE_NAME "MODEL_META_FILE_NAME"
+
+/**
+ * @brief Defines #MV_MODEL_LABEL_FILE_NAME to set inference
+ * models's label file attribute of the engine configuration.
+ * @details The file includes inference model's label file.
+ *
+ * @since_tizen 8.0
+ */
+#define MV_MODEL_LABEL_FILE_NAME "MODEL_LABEL_FILE_NAME"
+
+/**
+ * @brief Defines #MV_MODEL_CONFIDENCE_THRESHOLD to set inference
+ * models's confidence threshold value of the engine configuration.
+ * @details The file includes inference model's confidence threshold value.
+ *
+ * @since_tizen 8.0
+ */
+#define MV_MODEL_CONFIDENCE_THRESHOLD "CONFIDENCE_THRESHOLD"
+
+/**
+ * @brief Defines #MV_BACKEND_TYPE
+ * to set inference backend engine type. In default, tensorflow lite is used.
+ *
+ * @since_tizen 8.0
+ */
+#define MV_BACKEND_TYPE "BACKEND_TYPE"
+
+/**
+ * @brief Defines #MV_TARGET_DEVICE_TYPE
+ * to set inference target device type. In default, CPU device is used.
+ *
+ * @since_tizen 8.0
+ */
+#define MV_TARGET_DEVICE_TYPE "TARGET_DEVICE_TYPE"
+
+/**
+ * @brief Defines #MV_PLUGIN_NAME
+ * to set plugin library name for using external plugin.
+ */
+#define MV_PLUGIN_NAME "PLUGIN_NAME"
+
+/**
+ * @brief Defines #MV_USE_PLUGIN
+ * to indicate whether extern plugin is used or not for a given task type.
+ */
+#define MV_USE_PLUGIN "USE_PLUGIN"
+
+#endif /* __KEYS_H__ */
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __MACHINE_LEARNING_CONFIG_H__
-#define __MACHINE_LEARNING_CONFIG_H__
-
-#include <mv_common.h>
-#include "mv_private.h"
-#include "EngineConfig.h"
-
-#include "MetaParser.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-class MachineLearningConfig
-{
-private:
- std::unique_ptr<MetaParser> _parser;
- std::string _defaultModelName;
- std::string _modelFilePath;
- std::string _modelMetaFilePath;
- std::string _modelDefaultPath;
- std::string _modelLabelFilePath;
- std::string _pluginFileName;
- bool _usePlugin {};
- int _backendType {};
- int _targetDeviceType {};
- double _confidence_threshold {};
-
-public:
- MachineLearningConfig();
- virtual ~MachineLearningConfig() = default;
-
- void setUserModel(const std::string &model_file, const std::string &meta_file, const std::string &label_file);
- void parseConfigFile(const std::string &configFilePath);
- void parsePluginConfigFile(const std::string &pluginConfigFilePath);
- void parseMetaFile();
- void setBackendType(int backend_type);
- void setTargetDeviceType(int device_type);
- const std::string &getDefaultModelName() const;
- const std::string &getModelFilePath() const;
- const std::string &getLabelFilePath() const;
- const std::string &getPluginFileName() const;
- MetaMap &getInputMetaMap() const;
- MetaMap &getOutputMetaMap() const;
- double getConfidenceThreshold() const;
- int getBackendType() const;
- int getTargetDeviceType() const;
- bool isPluginUsed() const;
- void loadMetaFile(std::unique_ptr<MetaParser> parser);
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __MACHINE_LEARNING_EXCEPTION_H__
-#define __MACHINE_LEARNING_EXCEPTION_H__
-
-#include <exception>
-#include <string>
-
-#include <mv_common.h>
-
-namespace mediavision
-{
-namespace machine_learning
-{
-namespace exception
-{
-class BaseException : public std::exception
-{
-private:
- std::string _msg;
- int _errorType;
-
-public:
- BaseException(std::string msg, int errorType) : _msg(msg), _errorType(errorType)
- {}
- ~BaseException() override = default;
-
- const char *what() const noexcept override
- {
- return _msg.c_str();
- }
- int getError() const
- {
- return _errorType;
- }
-};
-
-class InvalidParameter : public BaseException
-{
-public:
- InvalidParameter(std::string msg) : BaseException("Invalid parameter: " + msg, MEDIA_VISION_ERROR_INVALID_PARAMETER)
- {}
- ~InvalidParameter() override = default;
-};
-
-class InvalidOperation : public BaseException
-{
-public:
- InvalidOperation(std::string msg, int errorType = MEDIA_VISION_ERROR_INVALID_OPERATION)
- : BaseException("Invalid operation: " + msg, errorType)
- {}
- ~InvalidOperation() final = default;
-};
-
-class OutOfMemory : public BaseException
-{
-public:
- OutOfMemory(std::string msg) : BaseException("Out of memory: " + msg, MEDIA_VISION_ERROR_OUT_OF_MEMORY)
- {}
- ~OutOfMemory() final = default;
-};
-
-class NoData : public BaseException
-{
-public:
- NoData(std::string msg) : BaseException("No Data: " + msg, MEDIA_VISION_ERROR_NO_DATA)
- {}
- ~NoData() final = default;
-};
-
-}; // Exception
-}; // MachineLearning
-}; // Mediavision
-
-#endif // __MACHINE_LEARNING_EXCEPTION_H__
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __MACHINE_LEARNING_KEY_H__
-#define __MACHINE_LEARNING_KEY_H__
-
-/**
- * @brief Defines #MV_MODEL_DEFAULT_PATH
- * to set the model default path.
- *
- * @since_tizen 8.0
- */
-#define MV_MODEL_DEFAULT_PATH "MODEL_DEFAULT_PATH"
-
-/**
- * @brief Defines #MV_MODEL_FILE_NAME
- * to set the model file path.
- *
- * @since_tizen 8.0
- */
-#define MV_MODEL_FILE_NAME "MODEL_FILE_NAME"
-
-/**
- * @brief Defines #MV_DEFAULT_MODEL_NAME
- * to set the default model name.
- *
- * @since_tizen 8.0
- */
-#define MV_DEFAULT_MODEL_NAME "DEFAULT_MODEL_NAME"
-
-/**
- * @brief Defines #MV_MODEL_META_FILE_NAME to set inference
- * models's metadata file attribute of the engine configuration.
- * @details The file includes inference model's metadata such as input and output
- * node names, input tensor's width and height,
- * mean and standard deviation values for pre-processing.
- *
- * @since_tizen 8.0
- */
-#define MV_MODEL_META_FILE_NAME "MODEL_META_FILE_NAME"
-
-/**
- * @brief Defines #MV_MODEL_LABEL_FILE_NAME to set inference
- * models's label file attribute of the engine configuration.
- * @details The file includes inference model's label file.
- *
- * @since_tizen 8.0
- */
-#define MV_MODEL_LABEL_FILE_NAME "MODEL_LABEL_FILE_NAME"
-
-/**
- * @brief Defines #MV_MODEL_CONFIDENCE_THRESHOLD to set inference
- * models's confidence threshold value of the engine configuration.
- * @details The file includes inference model's confidence threshold value.
- *
- * @since_tizen 8.0
- */
-#define MV_MODEL_CONFIDENCE_THRESHOLD "CONFIDENCE_THRESHOLD"
-
-/**
- * @brief Defines #MV_BACKEND_TYPE
- * to set inference backend engine type. In default, tensorflow lite is used.
- *
- * @since_tizen 8.0
- */
-#define MV_BACKEND_TYPE "BACKEND_TYPE"
-
-/**
- * @brief Defines #MV_TARGET_DEVICE_TYPE
- * to set inference target device type. In default, CPU device is used.
- *
- * @since_tizen 8.0
- */
-#define MV_TARGET_DEVICE_TYPE "TARGET_DEVICE_TYPE"
-
-/**
- * @brief Defines #MV_PLUGIN_NAME
- * to set plugin library name for using external plugin.
- */
-#define MV_PLUGIN_NAME "PLUGIN_NAME"
-
-/**
- * @brief Defines #MV_USE_PLUGIN
- * to indicate whether extern plugin is used or not for a given task type.
- */
-#define MV_USE_PLUGIN "USE_PLUGIN"
-
-#endif /* __MACHINE_LEARNING_KEY_H__ */
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __MV_MACHINE_LEARNING_COMMON_H__
-#define __MV_MACHINE_LEARNING_COMMON_H__
-
-#include <string>
-#include <vector>
-#include <map>
-
-#include "mv_inference_type.h"
-
-namespace mediavision
-{
-namespace common
-{
-static std::map<std::string, int> gBackendTypeTable = {
- { "OPENCV", MV_INFERENCE_BACKEND_OPENCV }, { "TFLITE", MV_INFERENCE_BACKEND_TFLITE },
- { "ARMNN", MV_INFERENCE_BACKEND_ARMNN }, { "ONE", MV_INFERENCE_BACKEND_ONE },
- { "NNTRAINER", MV_INFERENCE_BACKEND_NNTRAINER }, { "SNPE", MV_INFERENCE_BACKEND_SNPE }
-};
-
-static std::map<std::string, int> gDeviceTypeTable = { { "CPU", MV_INFERENCE_TARGET_DEVICE_CPU },
- { "GPU", MV_INFERENCE_TARGET_DEVICE_GPU },
- { "NPU", MV_INFERENCE_TARGET_DEVICE_CUSTOM } };
-
-static int GetBackendType(std::string backend_type)
-{
- auto item = gBackendTypeTable.find(backend_type);
- if (item != gBackendTypeTable.end())
- return item->second;
-
- return MEDIA_VISION_ERROR_INVALID_PARAMETER;
-}
-
-static int GetDeviceType(std::string device_type)
-{
- auto item = gDeviceTypeTable.find(device_type);
- if (item != gDeviceTypeTable.end())
- return item->second;
-
- return MEDIA_VISION_ERROR_INVALID_PARAMETER;
-}
-
-} // namespace
-} // namespace
-
-#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __MV_ML_TYPE_H__
+#define __MV_ML_TYPE_H__
+
+#include <mv_common.h>
+#include <mv_inference_type.h>
+
+namespace mediavision
+{
+namespace machine_learning
+{
+struct InputBaseType {
+ mv_source_h inference_src {};
+ InputBaseType(mv_source_h src = NULL) : inference_src(src)
+ {}
+ virtual ~InputBaseType()
+ {}
+};
+
+struct OutputBaseType {
+ unsigned long frame_number {};
+ virtual ~OutputBaseType()
+ {}
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __NATIVE_CAPI_H__
+#define __NATIVE_CAPI_H__
+
+#include <memory>
+#include <string>
+
+#include "Context.h"
+#include "ITask.h"
+#include "mv_private.h"
+#include "mv_ml_types.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+void *machine_learning_native_create();
+void machine_learning_native_add(void *handle, const std::string &task_name, mediavision::common::ITask *task);
+void machine_learning_native_destory(void *handle);
+void machine_learning_native_configure(void *handle, const std::string &task_name);
+void machine_learning_native_prepare(void *handle, const std::string &task_name);
+void machine_learning_native_inference(void *handle, const std::string &task_name, InputBaseType &input);
+void machine_learning_native_inference_async(void *handle, const std::string &task_name, InputBaseType &input);
+OutputBaseType &machine_learning_native_get_result(void *handle, const std::string &task_name);
+OutputBaseType &machine_learning_native_get_result_cache(void *handle, const std::string &task_name);
+void machine_learning_native_set_model(void *handle, const std::string &task_name, const char *model_file,
+ const char *meta_file, const char *label_file, const char *model_name = "");
+void machine_learning_native_set_engine(void *handle, const std::string &task_name, const char *backend_type,
+ const char *device_type);
+void machine_learning_native_get_engine_count(void *handle, const std::string &task_name, unsigned int *engine_count);
+void machine_learning_native_get_engine_type(void *handle, const std::string &task_name,
+ const unsigned int engine_index, char **engine_type);
+void machine_learning_native_get_device_count(void *handle, const std::string &task_name, const char *engine_type,
+ unsigned int *device_count);
+void machine_learning_native_get_device_type(void *handle, const std::string &task_name, const char *engine_type,
+ const unsigned int device_index, char **device_type);
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __MV_ML_PREPROCESS_H__
+#define __MV_ML_PREPROCESS_H__
+
+#include <string>
+#include <map>
+#include <memory>
+#include <vector>
+
+#include <inference_engine_type.h>
+#include <mv_inference_type.h>
+
+#include "mv_common.h"
+
+#include <opencv2/core.hpp>
+#include <opencv2/imgproc.hpp>
+
+/**
+ * @file MvMlPreprocess.h
+ * @brief This file contains the Preprocess class definition which
+ * provides Preprocess before running inference.
+ */
+
+namespace mediavision
+{
+namespace machine_learning
+{
+struct PreprocessConfig {
+ bool skip_csc {}; /**< It indicates whether color space conversion operation should be skipped or not. */
+ mv_colorspace_e output_format {}; /**< The pixel format of output tensor to be converted. */
+ mv_inference_data_type_e output_data_type {}; /**< The data type of output tensor to be converted. */
+ int output_channel {}; /** The channel size of output tensor te be converted. */
+ int output_width {}; /** The width size of output tensor to be converted. */
+ int output_height {}; /** The height size of output tensor to be converted. */
+ bool normalize {}; /**< It indicates whether normalization to input data should be performed or not. */
+ std::vector<double> mean;
+ std::vector<double> std;
+ bool quantize {}; /**< It indicates whether quantization to input data should be performed or not. */
+ std::vector<double> scale;
+ std::vector<double> zeropoint;
+};
+
+class Preprocess
+{
+public:
+ Preprocess()
+ {}
+ ~Preprocess() = default;
+
+ void setConfig(const PreprocessConfig &config);
+ template<typename T> void run(mv_source_h &mv_src, std::vector<T> &inputVector);
+
+ std::vector<unsigned int> &getImageWidth()
+ {
+ return _vImageWidth;
+ }
+ std::vector<unsigned int> &getImageHeight()
+ {
+ return _vImageHeight;
+ }
+
+private:
+ std::vector<unsigned int> _vImageWidth;
+ std::vector<unsigned int> _vImageHeight;
+ PreprocessConfig _config;
+
+ int convertToCv(int given_type, int ch);
+ void colorConvert(cv::Mat &source, cv::Mat &dest, int sType, int dType);
+ void convertToCvSource(std::vector<mv_source_h> &mv_srcs, std::vector<cv::Mat> &cv_srcs);
+ void normalize(cv::Mat &source, cv::Mat &dest, const std::vector<double> &mean, const std::vector<double> &std);
+ void quantize(cv::Mat &source, cv::Mat &dest, const std::vector<double> &scale,
+ const std::vector<double> &zeropoint);
+};
+
+} /* machine_learning */
+} /* mediavision */
+
+#endif /* __MV_ML_PREPROCESS_H__ */
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __COMMON_H__
-#define __COMMON_H__
-
-#include <string>
-#include <mv_common.h>
-#include <mv_private.h>
-#include <json-glib/json-glib.h>
-#include <inference_engine_type.h>
-#include <dlog.h>
-#include "machine_learning_exception.h"
-#include "mv_inference_type.h"
-#include "types.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T, typename U> T GetSupportedType(JsonObject *in_obj, std::string key, U &in_map)
-{
- auto supportedType = in_map.find(json_object_get_string_member(in_obj, key.c_str()));
- if (supportedType == in_map.end()) {
- throw mediavision::machine_learning::exception::InvalidParameter("invalid type.");
- }
-
- LOGI("%s: %d:%s", key.c_str(), static_cast<int>(supportedType->second), supportedType->first.c_str());
-
- return supportedType->second;
-}
-
-}
-}
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __MACHINE_LEARNING_PREPROCESS_H__
-#define __MACHINE_LEARNING_PREPROCESS_H__
-
-#include <string>
-#include <map>
-#include <memory>
-#include <vector>
-
-#include <inference_engine_type.h>
-#include <mv_inference_type.h>
-
-#include "mv_common.h"
-
-#include <opencv2/core.hpp>
-#include <opencv2/imgproc.hpp>
-
-/**
- * @file machine_learning_preprocess.h
- * @brief This file contains the Preprocess class definition which
- * provides Preprocess before running inference.
- */
-
-namespace mediavision
-{
-namespace machine_learning
-{
-struct PreprocessConfig {
- bool skip_csc {}; /**< It indicates whether color space conversion operation should be skipped or not. */
- mv_colorspace_e output_format {}; /**< The pixel format of output tensor to be converted. */
- mv_inference_data_type_e output_data_type {}; /**< The data type of output tensor to be converted. */
- int output_channel {}; /** The channel size of output tensor te be converted. */
- int output_width {}; /** The width size of output tensor to be converted. */
- int output_height {}; /** The height size of output tensor to be converted. */
- bool normalize {}; /**< It indicates whether normalization to input data should be performed or not. */
- std::vector<double> mean;
- std::vector<double> std;
- bool quantize {}; /**< It indicates whether quantization to input data should be performed or not. */
- std::vector<double> scale;
- std::vector<double> zeropoint;
-};
-
-class Preprocess
-{
-public:
- Preprocess()
- {}
- ~Preprocess() = default;
-
- void setConfig(const PreprocessConfig &config);
- template<typename T> void run(mv_source_h &mv_src, std::vector<T> &inputVector);
-
- std::vector<unsigned int> &getImageWidth()
- {
- return _vImageWidth;
- }
- std::vector<unsigned int> &getImageHeight()
- {
- return _vImageHeight;
- }
-
-private:
- std::vector<unsigned int> _vImageWidth;
- std::vector<unsigned int> _vImageHeight;
- PreprocessConfig _config;
-
- int convertToCv(int given_type, int ch);
- void colorConvert(cv::Mat &source, cv::Mat &dest, int sType, int dType);
- void convertToCvSource(std::vector<mv_source_h> &mv_srcs, std::vector<cv::Mat> &cv_srcs);
- void normalize(cv::Mat &source, cv::Mat &dest, const std::vector<double> &mean, const std::vector<double> &std);
- void quantize(cv::Mat &source, cv::Mat &dest, const std::vector<double> &scale,
- const std::vector<double> &zeropoint);
-};
-
-} /* machine_learning */
-} /* mediavision */
-
-#endif /* __MACHINE_LEARNING_PREPROCESS_H__ */
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __COMMON_H__
+#define __COMMON_H__
+
+#include <string>
+#include <mv_common.h>
+#include <mv_private.h>
+#include <json-glib/json-glib.h>
+#include <inference_engine_type.h>
+#include <dlog.h>
+#include "MvMlException.h"
+#include "mv_inference_type.h"
+#include "types.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T, typename U> T GetSupportedType(JsonObject *in_obj, std::string key, U &in_map)
+{
+ auto supportedType = in_map.find(json_object_get_string_member(in_obj, key.c_str()));
+ if (supportedType == in_map.end()) {
+ throw mediavision::machine_learning::exception::InvalidParameter("invalid type.");
+ }
+
+ LOGI("%s: %d:%s", key.c_str(), static_cast<int>(supportedType->second), supportedType->first.c_str());
+
+ return supportedType->second;
+}
+
+}
+}
+
+#endif
\ No newline at end of file
#include <fstream>
#include <algorithm>
-#include "machine_learning_exception.h"
+#include "MvMlException.h"
#include "MetaParser.h"
-#include "common.h"
+#include "meta_common.h"
using namespace std;
using namespace mediavision::machine_learning::exception;
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <unistd.h>
+#include <fstream>
+#include <string>
+#include <queue>
+#include <algorithm>
+#include "mv_private.h"
+#include "MvMlPreprocess.h"
+#include "MvMlException.h"
+
+constexpr int colorConvertTable[][12] = {
+ { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+ { 0, -1, 0, 0, 0, 0, 0, 0, 0, cv::COLOR_GRAY2BGR565, cv::COLOR_GRAY2RGB, cv::COLOR_GRAY2RGBA },
+ { 0, cv::COLOR_YUV2GRAY_I420, -1, 0, 0, 0, 0, 0, 0, 0, cv::COLOR_RGBA2GRAY, cv::COLOR_YUV2RGBA_I420 },
+ { 0, cv::COLOR_YUV2GRAY_NV12, 0, -1, 0, 0, 0, 0, 0, 0, cv::COLOR_YUV2RGB_NV12, cv::COLOR_YUV2RGBA_NV12 },
+ { 0, cv::COLOR_YUV2GRAY_YV12, 0, 0, -1, 0, 0, 0, 0, 0, cv::COLOR_YUV2RGB_YV12, cv::COLOR_YUV2RGBA_YV12 },
+ { 0, cv::COLOR_YUV2GRAY_NV21, 0, 0, 0, -1, 0, 0, 0, 0, cv::COLOR_YUV2RGB_NV21, cv::COLOR_YUV2RGBA_NV21 },
+ { 0, cv::COLOR_YUV2GRAY_YUYV, 0, 0, 0, 0, -1, 0, 0, 0, cv::COLOR_YUV2RGB_YUYV, cv::COLOR_YUV2RGBA_YUYV },
+ { 0, cv::COLOR_YUV2GRAY_UYVY, 0, 0, 0, 0, 0, -1, 0, 0, cv::COLOR_YUV2BGR_UYVY, cv::COLOR_YUV2BGRA_UYVY },
+ { 0, cv::COLOR_YUV2GRAY_Y422, 0, 0, 0, 0, 0, 0, -1, 0, cv::COLOR_YUV2RGB_Y422, cv::COLOR_YUV2RGBA_Y422 },
+ { 0, cv::COLOR_BGR5652GRAY, 0, 0, 0, 0, 0, 0, 0, -1, cv::COLOR_BGR5652BGR, cv::COLOR_BGR5652BGRA },
+ { 0, cv::COLOR_RGB2GRAY, 0, 0, 0, 0, 0, 0, 0, 0, -1, cv::COLOR_RGB2RGBA },
+ { 0, cv::COLOR_RGBA2GRAY, 0, 0, 0, 0, 0, 0, 0, cv::COLOR_BGRA2BGR565, cv::COLOR_RGBA2RGB, -1 }
+};
+
+using namespace std;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+void Preprocess::colorConvert(cv::Mat &source, cv::Mat &dest, int sType, int dType)
+{
+ LOGI("ENTER");
+
+ auto conversionColor = static_cast<int>(colorConvertTable[sType][dType]);
+ if (conversionColor == -1) { /* Don't need conversion */
+ dest = source;
+ } else if (conversionColor > 0) {
+ /* Class for representation the given image as cv::Mat before conversion */
+ cv::cvtColor(source, dest, conversionColor);
+ } else {
+ throw InvalidOperation("Fail to ColorConvert");
+ }
+
+ LOGI("LEAVE");
+}
+
+void Preprocess::normalize(cv::Mat &source, cv::Mat &dest, const vector<double> &mean, const vector<double> &std)
+{
+ LOGI("ENTER");
+
+ try {
+ cv::subtract(source, cv::Scalar(mean[0], mean[1], mean[2]), dest);
+ source = dest;
+ cv::divide(source, cv::Scalar(std[0], std[1], std[2]), dest);
+ } catch (cv::Exception &e) {
+ throw InvalidOperation("Fail to substract/divide");
+ }
+
+ LOGI("LEAVE");
+}
+
+void Preprocess::quantize(cv::Mat &source, cv::Mat &dest, const vector<double> &scale, const vector<double> &zeropoint)
+{
+ LOGI("ENTER");
+
+ try {
+ cv::subtract(source, cv::Scalar(zeropoint[0], zeropoint[1], zeropoint[2]), dest);
+ source = dest;
+ cv::multiply(source, cv::Scalar(scale[0], scale[1], scale[2]), dest);
+ } catch (cv::Exception &e) {
+ throw InvalidOperation("Fail to subtract/multiply");
+ }
+
+ LOGI("LEAVE");
+}
+
+int Preprocess::convertToCv(int given_type, int ch)
+{
+ int type = 0;
+
+ switch (given_type) {
+ case INFERENCE_TENSOR_DATA_TYPE_UINT8:
+ case MV_INFERENCE_DATA_UINT8:
+ LOGI("Type is %d ch with UINT8", ch);
+ type = ch == 1 ? CV_8UC1 : CV_8UC3;
+ break;
+ case INFERENCE_TENSOR_DATA_TYPE_FLOAT32:
+ case MV_INFERENCE_DATA_FLOAT32:
+ LOGI("Type is %d ch with FLOAT32", ch);
+ type = ch == 1 ? CV_32FC1 : CV_32FC3;
+ break;
+ default:
+ LOGI("unknown data type so FLOAT32 data type will be used in default");
+ type = ch == 1 ? CV_32FC1 : CV_32FC3;
+ break;
+ }
+
+ return type;
+}
+
+void Preprocess::convertToCvSource(vector<mv_source_h> &mv_srcs, vector<cv::Mat> &cv_srcs)
+{
+ for (auto &mv_src : mv_srcs) {
+ mv_colorspace_e colorspace = MEDIA_VISION_COLORSPACE_INVALID;
+ unsigned int width = 0, height = 0;
+ unsigned int bufferSize = 0;
+ unsigned char *buffer = NULL;
+
+ if (mv_source_get_width(mv_src, &width) != MEDIA_VISION_ERROR_NONE ||
+ mv_source_get_height(mv_src, &height) != MEDIA_VISION_ERROR_NONE ||
+ mv_source_get_colorspace(mv_src, &colorspace) != MEDIA_VISION_ERROR_NONE ||
+ mv_source_get_buffer(mv_src, &buffer, &bufferSize))
+ throw InvalidParameter("Invalid Parameter.");
+
+ _vImageWidth.push_back(width);
+ _vImageHeight.push_back(height);
+
+ // TODO. Let's support various color spaces.
+
+ if (colorspace != MEDIA_VISION_COLORSPACE_RGB888)
+ throw InvalidOperation("Not Supported format.");
+
+ /* convert mv_source to cv::Mat */
+ cv::Mat cvSource = cv::Mat(cv::Size(width, height), CV_MAKETYPE(CV_8U, 3), buffer).clone();
+
+ cv_srcs.push_back(cvSource);
+ LOGI("Size: w:%u, h:%u", cvSource.size().width, cvSource.size().height);
+ }
+}
+
+void Preprocess::setConfig(const PreprocessConfig &config)
+{
+ _config = config;
+}
+
+template<typename T> void Preprocess::run(mv_source_h &mv_src, vector<T> &inputVector)
+{
+ LOGI("ENTER");
+
+ vector<cv::Mat> oriCvSources;
+ vector<mv_source_h> mv_srcs = { mv_src };
+
+ _vImageWidth.clear();
+ _vImageHeight.clear();
+ convertToCvSource(mv_srcs, oriCvSources);
+
+ inputVector.resize(_config.output_height * _config.output_width * _config.output_channel);
+
+ int data_type = convertToCv(_config.output_data_type, _config.output_channel);
+ // dest is a wrapper of the buffer.
+ cv::Mat dest(cv::Size(_config.output_width, _config.output_height), data_type, inputVector.data());
+ cv::Mat cvSource, cvDest;
+
+ // cvSource has new allocation with dest.size()
+ cv::resize(oriCvSources[0], cvSource, dest.size());
+
+ if (_config.skip_csc) {
+ cvSource.convertTo(dest, dest.type());
+ } else {
+ mv_colorspace_e colorspace = MEDIA_VISION_COLORSPACE_INVALID;
+ int ret = mv_source_get_colorspace(mv_srcs[0], &colorspace);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get color space.");
+
+ // cvDest is allocated if colorspace is not RGB888, and
+ // cvDest shares the data with cvSource if the colorspace is RGB888.
+ colorConvert(cvSource, cvDest, colorspace, _config.output_format);
+ cvDest.convertTo(dest, dest.type());
+ }
+
+ if (_config.normalize)
+ normalize(dest, dest, _config.mean, _config.std);
+
+ if (_config.quantize)
+ quantize(dest, dest, _config.scale, _config.zeropoint);
+
+ LOGI("LEAVE");
+}
+
+template void Preprocess::run<float>(mv_source_h &mv_src, vector<float> &inputVector);
+template void Preprocess::run<unsigned char>(mv_source_h &mv_src, vector<unsigned char> &inputVector);
+
+} /* machine_learning */
+} /* mediavision */
#include <string>
#include <queue>
#include <algorithm>
-#include "machine_learning_exception.h"
+#include "MvMlException.h"
#include "PostprocessParser.h"
-#include "common.h"
+#include "meta_common.h"
using namespace std;
using namespace mediavision::machine_learning::exception;
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <unistd.h>
-#include <fstream>
-#include <string>
-#include <queue>
-#include <algorithm>
-#include "mv_private.h"
-#include "machine_learning_preprocess.h"
-#include "machine_learning_exception.h"
-
-constexpr int colorConvertTable[][12] = {
- { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
- { 0, -1, 0, 0, 0, 0, 0, 0, 0, cv::COLOR_GRAY2BGR565, cv::COLOR_GRAY2RGB, cv::COLOR_GRAY2RGBA },
- { 0, cv::COLOR_YUV2GRAY_I420, -1, 0, 0, 0, 0, 0, 0, 0, cv::COLOR_RGBA2GRAY, cv::COLOR_YUV2RGBA_I420 },
- { 0, cv::COLOR_YUV2GRAY_NV12, 0, -1, 0, 0, 0, 0, 0, 0, cv::COLOR_YUV2RGB_NV12, cv::COLOR_YUV2RGBA_NV12 },
- { 0, cv::COLOR_YUV2GRAY_YV12, 0, 0, -1, 0, 0, 0, 0, 0, cv::COLOR_YUV2RGB_YV12, cv::COLOR_YUV2RGBA_YV12 },
- { 0, cv::COLOR_YUV2GRAY_NV21, 0, 0, 0, -1, 0, 0, 0, 0, cv::COLOR_YUV2RGB_NV21, cv::COLOR_YUV2RGBA_NV21 },
- { 0, cv::COLOR_YUV2GRAY_YUYV, 0, 0, 0, 0, -1, 0, 0, 0, cv::COLOR_YUV2RGB_YUYV, cv::COLOR_YUV2RGBA_YUYV },
- { 0, cv::COLOR_YUV2GRAY_UYVY, 0, 0, 0, 0, 0, -1, 0, 0, cv::COLOR_YUV2BGR_UYVY, cv::COLOR_YUV2BGRA_UYVY },
- { 0, cv::COLOR_YUV2GRAY_Y422, 0, 0, 0, 0, 0, 0, -1, 0, cv::COLOR_YUV2RGB_Y422, cv::COLOR_YUV2RGBA_Y422 },
- { 0, cv::COLOR_BGR5652GRAY, 0, 0, 0, 0, 0, 0, 0, -1, cv::COLOR_BGR5652BGR, cv::COLOR_BGR5652BGRA },
- { 0, cv::COLOR_RGB2GRAY, 0, 0, 0, 0, 0, 0, 0, 0, -1, cv::COLOR_RGB2RGBA },
- { 0, cv::COLOR_RGBA2GRAY, 0, 0, 0, 0, 0, 0, 0, cv::COLOR_BGRA2BGR565, cv::COLOR_RGBA2RGB, -1 }
-};
-
-using namespace std;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-void Preprocess::colorConvert(cv::Mat &source, cv::Mat &dest, int sType, int dType)
-{
- LOGI("ENTER");
-
- auto conversionColor = static_cast<int>(colorConvertTable[sType][dType]);
- if (conversionColor == -1) { /* Don't need conversion */
- dest = source;
- } else if (conversionColor > 0) {
- /* Class for representation the given image as cv::Mat before conversion */
- cv::cvtColor(source, dest, conversionColor);
- } else {
- throw InvalidOperation("Fail to ColorConvert");
- }
-
- LOGI("LEAVE");
-}
-
-void Preprocess::normalize(cv::Mat &source, cv::Mat &dest, const vector<double> &mean, const vector<double> &std)
-{
- LOGI("ENTER");
-
- try {
- cv::subtract(source, cv::Scalar(mean[0], mean[1], mean[2]), dest);
- source = dest;
- cv::divide(source, cv::Scalar(std[0], std[1], std[2]), dest);
- } catch (cv::Exception &e) {
- throw InvalidOperation("Fail to substract/divide");
- }
-
- LOGI("LEAVE");
-}
-
-void Preprocess::quantize(cv::Mat &source, cv::Mat &dest, const vector<double> &scale, const vector<double> &zeropoint)
-{
- LOGI("ENTER");
-
- try {
- cv::subtract(source, cv::Scalar(zeropoint[0], zeropoint[1], zeropoint[2]), dest);
- source = dest;
- cv::multiply(source, cv::Scalar(scale[0], scale[1], scale[2]), dest);
- } catch (cv::Exception &e) {
- throw InvalidOperation("Fail to subtract/multiply");
- }
-
- LOGI("LEAVE");
-}
-
-int Preprocess::convertToCv(int given_type, int ch)
-{
- int type = 0;
-
- switch (given_type) {
- case INFERENCE_TENSOR_DATA_TYPE_UINT8:
- case MV_INFERENCE_DATA_UINT8:
- LOGI("Type is %d ch with UINT8", ch);
- type = ch == 1 ? CV_8UC1 : CV_8UC3;
- break;
- case INFERENCE_TENSOR_DATA_TYPE_FLOAT32:
- case MV_INFERENCE_DATA_FLOAT32:
- LOGI("Type is %d ch with FLOAT32", ch);
- type = ch == 1 ? CV_32FC1 : CV_32FC3;
- break;
- default:
- LOGI("unknown data type so FLOAT32 data type will be used in default");
- type = ch == 1 ? CV_32FC1 : CV_32FC3;
- break;
- }
-
- return type;
-}
-
-void Preprocess::convertToCvSource(vector<mv_source_h> &mv_srcs, vector<cv::Mat> &cv_srcs)
-{
- for (auto &mv_src : mv_srcs) {
- mv_colorspace_e colorspace = MEDIA_VISION_COLORSPACE_INVALID;
- unsigned int width = 0, height = 0;
- unsigned int bufferSize = 0;
- unsigned char *buffer = NULL;
-
- if (mv_source_get_width(mv_src, &width) != MEDIA_VISION_ERROR_NONE ||
- mv_source_get_height(mv_src, &height) != MEDIA_VISION_ERROR_NONE ||
- mv_source_get_colorspace(mv_src, &colorspace) != MEDIA_VISION_ERROR_NONE ||
- mv_source_get_buffer(mv_src, &buffer, &bufferSize))
- throw InvalidParameter("Invalid Parameter.");
-
- _vImageWidth.push_back(width);
- _vImageHeight.push_back(height);
-
- // TODO. Let's support various color spaces.
-
- if (colorspace != MEDIA_VISION_COLORSPACE_RGB888)
- throw InvalidOperation("Not Supported format.");
-
- /* convert mv_source to cv::Mat */
- cv::Mat cvSource = cv::Mat(cv::Size(width, height), CV_MAKETYPE(CV_8U, 3), buffer).clone();
-
- cv_srcs.push_back(cvSource);
- LOGI("Size: w:%u, h:%u", cvSource.size().width, cvSource.size().height);
- }
-}
-
-void Preprocess::setConfig(const PreprocessConfig &config)
-{
- _config = config;
-}
-
-template<typename T> void Preprocess::run(mv_source_h &mv_src, vector<T> &inputVector)
-{
- LOGI("ENTER");
-
- vector<cv::Mat> oriCvSources;
- vector<mv_source_h> mv_srcs = { mv_src };
-
- _vImageWidth.clear();
- _vImageHeight.clear();
- convertToCvSource(mv_srcs, oriCvSources);
-
- inputVector.resize(_config.output_height * _config.output_width * _config.output_channel);
-
- int data_type = convertToCv(_config.output_data_type, _config.output_channel);
- // dest is a wrapper of the buffer.
- cv::Mat dest(cv::Size(_config.output_width, _config.output_height), data_type, inputVector.data());
- cv::Mat cvSource, cvDest;
-
- // cvSource has new allocation with dest.size()
- cv::resize(oriCvSources[0], cvSource, dest.size());
-
- if (_config.skip_csc) {
- cvSource.convertTo(dest, dest.type());
- } else {
- mv_colorspace_e colorspace = MEDIA_VISION_COLORSPACE_INVALID;
- int ret = mv_source_get_colorspace(mv_srcs[0], &colorspace);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get color space.");
-
- // cvDest is allocated if colorspace is not RGB888, and
- // cvDest shares the data with cvSource if the colorspace is RGB888.
- colorConvert(cvSource, cvDest, colorspace, _config.output_format);
- cvDest.convertTo(dest, dest.type());
- }
-
- if (_config.normalize)
- normalize(dest, dest, _config.mean, _config.std);
-
- if (_config.quantize)
- quantize(dest, dest, _config.scale, _config.zeropoint);
-
- LOGI("LEAVE");
-}
-
-template void Preprocess::run<float>(mv_source_h &mv_src, vector<float> &inputVector);
-template void Preprocess::run<unsigned char>(mv_source_h &mv_src, vector<unsigned char> &inputVector);
-
-} /* machine_learning */
-} /* mediavision */
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "mv_private.h"
-
-#include "MachineLearningNative.h"
-#include "machine_learning_exception.h"
-
-using namespace std;
-using namespace mediavision::common;
-using namespace mediavision::machine_learning;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-inline ITask *get_task(void *handle, const std::string &name)
-{
- auto context = static_cast<Context *>(handle);
-
- return context->__tasks.at(name);
-}
-
-void *machine_learning_native_create()
-{
- return static_cast<void *>(new Context());
-}
-
-void machine_learning_native_add(void *handle, const string &task_name, ITask *task)
-{
- auto context = static_cast<Context *>(handle);
-
- context->__tasks.insert(make_pair(task_name, task));
-}
-
-void machine_learning_native_destory(void *handle)
-{
- auto context = static_cast<Context *>(handle);
-
- for (auto &m : context->__tasks)
- delete m.second;
-
- delete context;
-}
-
-void machine_learning_native_configure(void *handle, const string &task_name)
-{
- auto task = get_task(handle, task_name);
-
- task->configure();
-}
-
-void machine_learning_native_prepare(void *handle, const string &task_name)
-{
- auto task = get_task(handle, task_name);
-
- task->prepare();
-}
-
-void machine_learning_native_inference(void *handle, const string &task_name, InputBaseType &input)
-{
- auto task = get_task(handle, task_name);
-
- task->perform(input);
-}
-
-void machine_learning_native_inference_async(void *handle, const string &task_name, InputBaseType &input)
-{
- auto task = get_task(handle, task_name);
-
- task->performAsync(input);
-}
-
-OutputBaseType &machine_learning_native_get_result(void *handle, const string &task_name)
-{
- auto task = get_task(handle, task_name);
-
- return task->getOutput();
-}
-
-OutputBaseType &machine_learning_native_get_result_cache(void *handle, const string &task_name)
-{
- auto task = get_task(handle, task_name);
-
- return task->getOutputCache();
-}
-
-void machine_learning_native_set_model(void *handle, const string &task_name, const char *model_file,
- const char *meta_file, const char *label_file, const char *model_name)
-{
- auto task = get_task(handle, task_name);
-
- task->setModelInfo(model_file, meta_file, label_file, model_name);
-}
-
-void machine_learning_native_set_engine(void *handle, const string &task_name, const char *backend_type,
- const char *device_type)
-{
- auto task = get_task(handle, task_name);
-
- task->setEngineInfo(backend_type, device_type);
-}
-
-void machine_learning_native_get_engine_count(void *handle, const string &task_name, unsigned int *engine_count)
-{
- auto task = get_task(handle, task_name);
-
- *engine_count = task->getNumberOfEngines();
-}
-
-void machine_learning_native_get_engine_type(void *handle, const string &task_name, const unsigned int engine_index,
- char **engine_type)
-{
- auto task = get_task(handle, task_name);
-
- *engine_type = (char *) task->getEngineType(engine_index).c_str();
-}
-
-void machine_learning_native_get_device_count(void *handle, const string &task_name, const char *engine_type,
- unsigned int *device_count)
-{
- auto task = get_task(handle, task_name);
-
- *device_count = task->getNumberOfDevices(engine_type);
-}
-
-void machine_learning_native_get_device_type(void *handle, const string &task_name, const char *engine_type,
- const unsigned int device_index, char **device_type)
-{
- auto task = get_task(handle, task_name);
-
- *device_type = (char *) task->getDeviceType(engine_type, device_index).c_str();
-}
-
-}
-}
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "MvMlException.h"
+#include "MvMlConfig.h"
+#include "keys.h"
+
+using namespace std;
+using namespace MediaVision::Common;
+using namespace mediavision::machine_learning;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+Config::Config()
+{}
+
+void Config::setBackendType(int backend_type)
+{
+ _backendType = backend_type;
+}
+
+void Config::setTargetDeviceType(int device_type)
+{
+ _targetDeviceType = device_type;
+}
+
+const std::string &Config::getDefaultModelName() const
+{
+ return _defaultModelName;
+}
+
+const std::string &Config::getModelFilePath() const
+{
+ return _modelFilePath;
+}
+
+const std::string &Config::getLabelFilePath() const
+{
+ return _modelLabelFilePath;
+}
+
+const std::string &Config::getPluginFileName() const
+{
+ return _pluginFileName;
+}
+
+MetaMap &Config::getInputMetaMap() const
+{
+ return _parser->getInputMetaMap();
+}
+
+MetaMap &Config::getOutputMetaMap() const
+{
+ return _parser->getOutputMetaMap();
+}
+
+double Config::getConfidenceThreshold() const
+{
+ return _confidence_threshold;
+}
+
+int Config::getBackendType() const
+{
+ return _backendType;
+}
+
+int Config::getTargetDeviceType() const
+{
+ return _targetDeviceType;
+}
+
+bool Config::isPluginUsed() const
+{
+ return _usePlugin;
+}
+
+void Config::setUserModel(const string &model_file, const string &meta_file, const string &label_file)
+{
+ if (!model_file.empty())
+ _modelFilePath = _modelDefaultPath + model_file;
+ if (!meta_file.empty())
+ _modelMetaFilePath = _modelDefaultPath + meta_file;
+ if (!label_file.empty())
+ _modelLabelFilePath = _modelDefaultPath + label_file;
+}
+
+static bool IsJsonFile(const string &fileName)
+{
+ return (!fileName.substr(fileName.find_last_of(".") + 1).compare("json"));
+}
+
+void Config::parseConfigFile(const std::string &configFilePath)
+{
+ auto config = make_unique<EngineConfig>(MV_CONFIG_PATH + configFilePath);
+
+ int ret = config->getStringAttribute(MV_DEFAULT_MODEL_NAME, &_defaultModelName);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get default model name.");
+
+ if (_backendType == MV_INFERENCE_BACKEND_NONE) {
+ ret = config->getIntegerAttribute(MV_BACKEND_TYPE, &_backendType);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get backend engine type.");
+ }
+
+ if (_targetDeviceType == MV_INFERENCE_TARGET_DEVICE_NONE) {
+ ret = config->getIntegerAttribute(MV_TARGET_DEVICE_TYPE, &_targetDeviceType);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get target device type.");
+ }
+
+ ret = config->getStringAttribute(MV_MODEL_DEFAULT_PATH, &_modelDefaultPath);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get model default path");
+
+ ret = config->getStringAttribute(MV_MODEL_FILE_NAME, &_modelFilePath);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get model file path");
+
+ _modelFilePath = _modelDefaultPath + _modelFilePath;
+
+ ret = config->getStringAttribute(MV_MODEL_META_FILE_NAME, &_modelMetaFilePath);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get model meta file path");
+
+ if (_modelMetaFilePath.empty())
+ throw InvalidOperation("Model meta file doesn't exist.");
+
+ if (!IsJsonFile(_modelMetaFilePath))
+ throw InvalidOperation("Model meta file should be json");
+
+ _modelMetaFilePath = _modelDefaultPath + _modelMetaFilePath;
+ LOGI("meta file path = %s", _modelMetaFilePath.c_str());
+
+ ret = config->getStringAttribute(MV_MODEL_LABEL_FILE_NAME, &_modelLabelFilePath);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get model label file path");
+
+ _modelLabelFilePath = _modelDefaultPath + _modelLabelFilePath;
+
+ ret = config->getDoubleAttribute(MV_MODEL_CONFIDENCE_THRESHOLD, &_confidence_threshold);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ LOGW("threshold value doesn't exist.");
+}
+
+void Config::parsePluginConfigFile(const std::string &pluginConfigFilePath)
+{
+ auto config = make_unique<EngineConfig>(MV_CONFIG_PATH + pluginConfigFilePath);
+
+ int ret = config->getStringAttribute(MV_PLUGIN_NAME, &_pluginFileName);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get model plugin file name");
+
+ ret = config->getStringAttribute(MV_DEFAULT_MODEL_NAME, &_defaultModelName);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get default model name.");
+
+ ret = config->getBooleanAttribute(MV_USE_PLUGIN, &_usePlugin);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get use_plugin value");
+}
+
+void Config::loadMetaFile(unique_ptr<MetaParser> parser)
+{
+ _parser = move(parser);
+ _parser->load(_modelMetaFilePath);
+}
+
+}
+}
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "machine_learning_exception.h"
-#include "machine_learning_config.h"
-#include "machine_learning_key.h"
-
-using namespace std;
-using namespace MediaVision::Common;
-using namespace mediavision::machine_learning;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-MachineLearningConfig::MachineLearningConfig()
-{}
-
-void MachineLearningConfig::setBackendType(int backend_type)
-{
- _backendType = backend_type;
-}
-
-void MachineLearningConfig::setTargetDeviceType(int device_type)
-{
- _targetDeviceType = device_type;
-}
-
-const std::string &MachineLearningConfig::getDefaultModelName() const
-{
- return _defaultModelName;
-}
-
-const std::string &MachineLearningConfig::getModelFilePath() const
-{
- return _modelFilePath;
-}
-
-const std::string &MachineLearningConfig::getLabelFilePath() const
-{
- return _modelLabelFilePath;
-}
-
-const std::string &MachineLearningConfig::getPluginFileName() const
-{
- return _pluginFileName;
-}
-
-MetaMap &MachineLearningConfig::getInputMetaMap() const
-{
- return _parser->getInputMetaMap();
-}
-
-MetaMap &MachineLearningConfig::getOutputMetaMap() const
-{
- return _parser->getOutputMetaMap();
-}
-
-double MachineLearningConfig::getConfidenceThreshold() const
-{
- return _confidence_threshold;
-}
-
-int MachineLearningConfig::getBackendType() const
-{
- return _backendType;
-}
-
-int MachineLearningConfig::getTargetDeviceType() const
-{
- return _targetDeviceType;
-}
-
-bool MachineLearningConfig::isPluginUsed() const
-{
- return _usePlugin;
-}
-
-void MachineLearningConfig::setUserModel(const string &model_file, const string &meta_file, const string &label_file)
-{
- if (!model_file.empty())
- _modelFilePath = _modelDefaultPath + model_file;
- if (!meta_file.empty())
- _modelMetaFilePath = _modelDefaultPath + meta_file;
- if (!label_file.empty())
- _modelLabelFilePath = _modelDefaultPath + label_file;
-}
-
-static bool IsJsonFile(const string &fileName)
-{
- return (!fileName.substr(fileName.find_last_of(".") + 1).compare("json"));
-}
-
-void MachineLearningConfig::parseConfigFile(const std::string &configFilePath)
-{
- auto config = make_unique<EngineConfig>(MV_CONFIG_PATH + configFilePath);
-
- int ret = config->getStringAttribute(MV_DEFAULT_MODEL_NAME, &_defaultModelName);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get default model name.");
-
- if (_backendType == MV_INFERENCE_BACKEND_NONE) {
- ret = config->getIntegerAttribute(MV_BACKEND_TYPE, &_backendType);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get backend engine type.");
- }
-
- if (_targetDeviceType == MV_INFERENCE_TARGET_DEVICE_NONE) {
- ret = config->getIntegerAttribute(MV_TARGET_DEVICE_TYPE, &_targetDeviceType);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get target device type.");
- }
-
- ret = config->getStringAttribute(MV_MODEL_DEFAULT_PATH, &_modelDefaultPath);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get model default path");
-
- ret = config->getStringAttribute(MV_MODEL_FILE_NAME, &_modelFilePath);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get model file path");
-
- _modelFilePath = _modelDefaultPath + _modelFilePath;
-
- ret = config->getStringAttribute(MV_MODEL_META_FILE_NAME, &_modelMetaFilePath);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get model meta file path");
-
- if (_modelMetaFilePath.empty())
- throw InvalidOperation("Model meta file doesn't exist.");
-
- if (!IsJsonFile(_modelMetaFilePath))
- throw InvalidOperation("Model meta file should be json");
-
- _modelMetaFilePath = _modelDefaultPath + _modelMetaFilePath;
- LOGI("meta file path = %s", _modelMetaFilePath.c_str());
-
- ret = config->getStringAttribute(MV_MODEL_LABEL_FILE_NAME, &_modelLabelFilePath);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get model label file path");
-
- _modelLabelFilePath = _modelDefaultPath + _modelLabelFilePath;
-
- ret = config->getDoubleAttribute(MV_MODEL_CONFIDENCE_THRESHOLD, &_confidence_threshold);
- if (ret != MEDIA_VISION_ERROR_NONE)
- LOGW("threshold value doesn't exist.");
-}
-
-void MachineLearningConfig::parsePluginConfigFile(const std::string &pluginConfigFilePath)
-{
- auto config = make_unique<EngineConfig>(MV_CONFIG_PATH + pluginConfigFilePath);
-
- int ret = config->getStringAttribute(MV_PLUGIN_NAME, &_pluginFileName);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get model plugin file name");
-
- ret = config->getStringAttribute(MV_DEFAULT_MODEL_NAME, &_defaultModelName);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get default model name.");
-
- ret = config->getBooleanAttribute(MV_USE_PLUGIN, &_usePlugin);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get use_plugin value");
-}
-
-void MachineLearningConfig::loadMetaFile(unique_ptr<MetaParser> parser)
-{
- _parser = move(parser);
- _parser->load(_modelMetaFilePath);
-}
-
-}
-}
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "mv_private.h"
+
+#include "native_capi.h"
+#include "MvMlException.h"
+
+using namespace std;
+using namespace mediavision::common;
+using namespace mediavision::machine_learning;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+inline ITask *get_task(void *handle, const std::string &name)
+{
+ auto context = static_cast<Context *>(handle);
+
+ return context->__tasks.at(name);
+}
+
+void *machine_learning_native_create()
+{
+ return static_cast<void *>(new Context());
+}
+
+void machine_learning_native_add(void *handle, const string &task_name, ITask *task)
+{
+ auto context = static_cast<Context *>(handle);
+
+ context->__tasks.insert(make_pair(task_name, task));
+}
+
+void machine_learning_native_destory(void *handle)
+{
+ auto context = static_cast<Context *>(handle);
+
+ for (auto &m : context->__tasks)
+ delete m.second;
+
+ delete context;
+}
+
+void machine_learning_native_configure(void *handle, const string &task_name)
+{
+ auto task = get_task(handle, task_name);
+
+ task->configure();
+}
+
+void machine_learning_native_prepare(void *handle, const string &task_name)
+{
+ auto task = get_task(handle, task_name);
+
+ task->prepare();
+}
+
+void machine_learning_native_inference(void *handle, const string &task_name, InputBaseType &input)
+{
+ auto task = get_task(handle, task_name);
+
+ task->perform(input);
+}
+
+void machine_learning_native_inference_async(void *handle, const string &task_name, InputBaseType &input)
+{
+ auto task = get_task(handle, task_name);
+
+ task->performAsync(input);
+}
+
+OutputBaseType &machine_learning_native_get_result(void *handle, const string &task_name)
+{
+ auto task = get_task(handle, task_name);
+
+ return task->getOutput();
+}
+
+OutputBaseType &machine_learning_native_get_result_cache(void *handle, const string &task_name)
+{
+ auto task = get_task(handle, task_name);
+
+ return task->getOutputCache();
+}
+
+void machine_learning_native_set_model(void *handle, const string &task_name, const char *model_file,
+ const char *meta_file, const char *label_file, const char *model_name)
+{
+ auto task = get_task(handle, task_name);
+
+ task->setModelInfo(model_file, meta_file, label_file, model_name);
+}
+
+void machine_learning_native_set_engine(void *handle, const string &task_name, const char *backend_type,
+ const char *device_type)
+{
+ auto task = get_task(handle, task_name);
+
+ task->setEngineInfo(backend_type, device_type);
+}
+
+void machine_learning_native_get_engine_count(void *handle, const string &task_name, unsigned int *engine_count)
+{
+ auto task = get_task(handle, task_name);
+
+ *engine_count = task->getNumberOfEngines();
+}
+
+void machine_learning_native_get_engine_type(void *handle, const string &task_name, const unsigned int engine_index,
+ char **engine_type)
+{
+ auto task = get_task(handle, task_name);
+
+ *engine_type = (char *) task->getEngineType(engine_index).c_str();
+}
+
+void machine_learning_native_get_device_count(void *handle, const string &task_name, const char *engine_type,
+ unsigned int *device_count)
+{
+ auto task = get_task(handle, task_name);
+
+ *device_count = task->getNumberOfDevices(engine_type);
+}
+
+void machine_learning_native_get_device_type(void *handle, const string &task_name, const char *engine_type,
+ const unsigned int device_index, char **device_type)
+{
+ auto task = get_task(handle, task_name);
+
+ *device_type = (char *) task->getDeviceType(engine_type, device_index).c_str();
+}
+
+}
+}
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __FACE_RECOGNITION_H__
+#define __FACE_RECOGNITION_H__
+
+#include <unordered_map>
+#include <mv_common.h>
+#include <mv_inference_type.h>
+
+#include "training_engine_error.h"
+#include "training_engine_common_impl.h"
+#include "inference_engine_common_impl.h"
+#include "mv_ml_types.h"
+#include "Inference.h"
+#include "LabelManager.h"
+#include "FeatureVectorManager.h"
+#include "DataSetManager.h"
+#include "SimpleShot.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+namespace face_recognition
+{
+enum class WorkingStatus { NONE, INITIALIZED, TRAINED, INFERENCED };
+
+enum class RequestMode { REGISTER, INFERENCE, DELETE };
+}
+
+struct FaceRecognitionInput : public InputBaseType {
+ FaceRecognitionInput(mv_source_h src = nullptr) : InputBaseType(src)
+ {}
+ face_recognition::RequestMode mode {};
+ std::vector<std::vector<float> > inputs;
+ std::vector<std::string> labels;
+};
+
+/**
+ * @brief The face recognition result structure.
+ * @details Contains face recognition result such as label, label index, raw data,
+ * and raw data count.
+ */
+struct FaceRecognitionResult : public OutputBaseType {
+ unsigned int label_idx {}; /**< label index of label file. */
+ std::vector<float> raw_data; /**< raw data to each label. */
+ std::vector<std::string> labels;
+ std::string label; /**< label string. */
+ bool is_valid {}; /**< inference result is valid or not. */
+};
+
+struct FaceRecognitionConfig {
+ training_target_type_e training_target_device_type { TRAINING_TARGET_NONE };
+ training_backend_type_e training_engine_backend_type { TRAINING_BACKEND_NONE };
+ mv_inference_target_device_e inference_target_device_type { MV_INFERENCE_TARGET_DEVICE_NONE };
+ mv_inference_backend_type_e inference_engine_backend_type { MV_INFERENCE_BACKEND_NONE };
+ std::string internal_model_file_path;
+ std::string label_file_path;
+ std::string feature_vector_file_path;
+ double decision_threshold {};
+ std::vector<size_t> input_tensor_shape;
+};
+
+class FaceRecognition
+{
+private:
+ face_recognition::WorkingStatus _status { face_recognition::WorkingStatus::NONE };
+ std::unique_ptr<mediavision::inference::Inference> _internal;
+ std::unique_ptr<mediavision::inference::Inference> _backbone;
+ std::unique_ptr<TrainingModel> _training_model;
+ std::unique_ptr<LabelManager> _label_manager;
+ FaceRecognitionConfig _config;
+ FaceRecognitionResult _result;
+
+ void checkFeatureVectorFile(std::string fv_file_name, std::string new_fv_file_name);
+ void storeDataSet(std::unique_ptr<DataSetManager> &data_set, unsigned int label_cnt);
+ void checkResult();
+ void trainModel();
+
+public:
+ FaceRecognition() = default;
+ ~FaceRecognition() = default;
+
+ int initialize();
+ void setConfig(FaceRecognitionConfig &config);
+ int registerNewFace(std::vector<float> &input_vec, std::string label_name);
+ int recognizeFace(std::vector<float> &input_vec);
+ int deleteLabel(std::string label_name);
+ FaceRecognitionResult &result();
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __FACE_RECOGNITION_ADAPTER_H__
+#define __FACE_RECOGNITION_ADAPTER_H__
+
+#include <dlog.h>
+
+#include "EngineConfig.h"
+#include "ITask.h"
+#include "FaceRecognition.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+class FaceRecognitionAdapter : public mediavision::common::ITask
+{
+private:
+ std::unique_ptr<FaceRecognition> _face_recognition;
+ std::unique_ptr<MediaVision::Common::EngineConfig> _config;
+
+public:
+ FaceRecognitionAdapter();
+ ~FaceRecognitionAdapter();
+
+ std::unique_ptr<MediaVision::Common::EngineConfig> &getConfig()
+ {
+ return _config;
+ }
+
+ void setModelInfo(const std::string &model_file, const std::string &meta_file, const std::string &label_file,
+ const std::string &model_name) override;
+ void setEngineInfo(const std::string &engine_type, const std::string &device_type) override;
+ void configure() override;
+ unsigned int getNumberOfEngines() override;
+ const std::string &getEngineType(unsigned int engine_index) override;
+ unsigned int getNumberOfDevices(const std::string &engine_type) override;
+ const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
+ void prepare() override;
+ void perform(InputBaseType &input) override;
+ void performAsync(InputBaseType &input) override;
+ OutputBaseType &getOutput() override;
+ OutputBaseType &getOutputCache() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __FACE_RECOGNITION_DSM_H__
+#define __FACE_RECOGNITION_DSM_H__
+
+#include <string>
+
+#include "FeatureVectorManager.h"
+#include "DataSetManager.h"
+
+class FaceRecognitionDSM : public DataSetManager
+{
+private:
+ void printHeader(FeaVecHeader &fvh);
+ bool isHeaderValid(const FeaVecHeader &fvh);
+
+public:
+ FaceRecognitionDSM();
+ ~FaceRecognitionDSM() = default;
+
+ bool isFeatureVectorAllowed(unsigned int label_idx) override;
+ void loadDataSet(const std::string &file_name, unsigned int new_label_cnt) override;
+ void addDataSet(std::vector<float> &feature_vec, unsigned int label_idx, unsigned int label_cnt) override;
+};
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __FACE_RECOGNITION_FVM_H__
+#define __FACE_RECOGNITION_FVM_H__
+
+#include <string.h>
+#include <vector>
+
+#include "FeatureVectorManager.h"
+#include "file_util.h"
+
+class FaceRecognitionFVM : public FeatureVectorManager
+{
+public:
+ FaceRecognitionFVM(const std::string feature_vector_file = "feature_vector_file.dat");
+ ~FaceRecognitionFVM() = default;
+
+ void updateHeader(size_t feature_size, size_t label_cnt, unsigned int data_set_cnt) override;
+ void storeData(std::vector<std::vector<float> > &features_vec, std::vector<unsigned int> &label_index) override;
+ void remove() override;
+};
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __FACENET_H__
+#define __FACENET_H__
+
+#include <mv_common.h>
+#include <mv_inference_type.h>
+
+#include "EngineConfig.h"
+#include "inference_engine_common_impl.h"
+#include "Inference.h"
+#include "FacenetParser.h"
+#include "face_recognition_type.h"
+#include "MvMlPreprocess.h"
+#include "mv_ml_types.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+struct FacenetInput : public InputBaseType {
+ FacenetInput(mv_source_h src = nullptr) : InputBaseType(src)
+ {}
+};
+
+struct FacenetOutput : public OutputBaseType {
+ std::vector<std::vector<float> > outputs;
+};
+
+class Facenet
+{
+protected:
+ std::unique_ptr<mediavision::inference::Inference> _inference;
+ std::unique_ptr<MediaVision::Common::EngineConfig> _config;
+ std::unique_ptr<MetaParser> _parser;
+ FacenetOutput _result;
+ inference_engine_tensor_buffer *_outputTensorBuffer {};
+ Preprocess _preprocess;
+ std::string _modelFileName;
+ std::string _modelMetaFileName;
+ std::string _facenetOutputTensorName;
+ int _backendType {};
+ int _targetDeviceType {};
+
+ template<typename T>
+ void preprocess(mv_source_h &mv_src, std::shared_ptr<MetaInfo> metaInfo, std::vector<T> &inputVector);
+ template<typename T> void inference(std::vector<std::vector<T> > &inputVectors);
+
+public:
+ Facenet();
+ virtual ~Facenet() = default;
+ void parseMetaFile();
+ std::shared_ptr<MetaInfo> getInputMetaInfo();
+ void configure();
+ void prepare();
+ template<typename T> void perform(mv_source_h &mv_src, std::shared_ptr<MetaInfo> metaInfo);
+ FacenetOutput &result();
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __FACENET_ADAPTER_H__
+#define __FACENET_ADAPTER_H__
+
+#include <dlog.h>
+
+#include "EngineConfig.h"
+#include "ITask.h"
+#include "mv_ml_types.h"
+#include "Facenet.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+class FacenetAdapter : public mediavision::common::ITask
+{
+private:
+ std::unique_ptr<Facenet> _facenet;
+
+public:
+ FacenetAdapter();
+ ~FacenetAdapter();
+
+ void setModelInfo(const std::string &model_file, const std::string &meta_file, const std::string &label_file,
+ const std::string &model_name) override;
+ void setEngineInfo(const std::string &engine_type, const std::string &device_type) override;
+ void configure() override;
+ unsigned int getNumberOfEngines() override;
+ const std::string &getEngineType(unsigned int engine_index) override;
+ unsigned int getNumberOfDevices(const std::string &engine_type) override;
+ const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
+ void prepare() override;
+ void perform(InputBaseType &input) override;
+ void performAsync(InputBaseType &input) override;
+ OutputBaseType &getOutput() override;
+ OutputBaseType &getOutputCache() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __FACENET_PARSER_H__
+#define __FACENET_PARSER_H__
+
+#include "MetaParser.h"
+#include "PostprocessParser.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+class FacenetParser : public MetaParser
+{
+private:
+ PostprocessParser _postprocessParser;
+
+protected:
+ void parsePostprocess(std::shared_ptr<MetaInfo> meta_info, JsonObject *in_obj) override;
+
+public:
+ FacenetParser(int task_type = 0);
+ ~FacenetParser();
+};
+
+}
+}
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __SIMPLE_SHOT_H__
+#define __SIMPLE_SHOT_H__
+
+#include <mv_inference_type.h>
+#include "TrainingModel.h"
+
+class SimpleShot : public TrainingModel
+{
+private:
+ TrainingEngineBackendInfo _engine_info;
+
+private:
+ void saveModel(const std::string file_path) override;
+ void removeModel(const std::string file_path) override;
+
+public:
+ SimpleShot(const training_backend_type_e backend_type, const training_target_type_e target_type,
+ const std::vector<size_t> input_tensor_shape, const std::string internal_model_file);
+ ~SimpleShot();
+
+ // Configure layers for SimpleShot learning.
+ void configureModel(int num_of_class) override;
+ TrainingEngineBackendInfo &getTrainingEngineInfo() override;
+};
+
+#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __FACE_RECOGNITION_H__
-#define __FACE_RECOGNITION_H__
-
-#include <unordered_map>
-#include <mv_common.h>
-#include <mv_inference_type.h>
-
-#include "training_engine_error.h"
-#include "training_engine_common_impl.h"
-#include "inference_engine_common_impl.h"
-#include "MachineLearningType.h"
-#include "Inference.h"
-#include "label_manager.h"
-#include "feature_vector_manager.h"
-#include "data_set_manager.h"
-#include "simple_shot.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-namespace face_recognition
-{
-enum class WorkingStatus { NONE, INITIALIZED, TRAINED, INFERENCED };
-
-enum class RequestMode { REGISTER, INFERENCE, DELETE };
-}
-
-struct FaceRecognitionInput : public InputBaseType {
- FaceRecognitionInput(mv_source_h src = nullptr) : InputBaseType(src)
- {}
- face_recognition::RequestMode mode {};
- std::vector<std::vector<float> > inputs;
- std::vector<std::string> labels;
-};
-
-/**
- * @brief The face recognition result structure.
- * @details Contains face recognition result such as label, label index, raw data,
- * and raw data count.
- */
-struct FaceRecognitionResult : public OutputBaseType {
- unsigned int label_idx {}; /**< label index of label file. */
- std::vector<float> raw_data; /**< raw data to each label. */
- std::vector<std::string> labels;
- std::string label; /**< label string. */
- bool is_valid {}; /**< inference result is valid or not. */
-};
-
-struct FaceRecognitionConfig {
- training_target_type_e training_target_device_type { TRAINING_TARGET_NONE };
- training_backend_type_e training_engine_backend_type { TRAINING_BACKEND_NONE };
- mv_inference_target_device_e inference_target_device_type { MV_INFERENCE_TARGET_DEVICE_NONE };
- mv_inference_backend_type_e inference_engine_backend_type { MV_INFERENCE_BACKEND_NONE };
- std::string internal_model_file_path;
- std::string label_file_path;
- std::string feature_vector_file_path;
- double decision_threshold {};
- std::vector<size_t> input_tensor_shape;
-};
-
-class FaceRecognition
-{
-private:
- face_recognition::WorkingStatus _status { face_recognition::WorkingStatus::NONE };
- std::unique_ptr<mediavision::inference::Inference> _internal;
- std::unique_ptr<mediavision::inference::Inference> _backbone;
- std::unique_ptr<TrainingModel> _training_model;
- std::unique_ptr<LabelManager> _label_manager;
- FaceRecognitionConfig _config;
- FaceRecognitionResult _result;
-
- void checkFeatureVectorFile(std::string fv_file_name, std::string new_fv_file_name);
- void storeDataSet(std::unique_ptr<DataSetManager> &data_set, unsigned int label_cnt);
- void checkResult();
- void trainModel();
-
-public:
- FaceRecognition() = default;
- ~FaceRecognition() = default;
-
- int initialize();
- void setConfig(FaceRecognitionConfig &config);
- int registerNewFace(std::vector<float> &input_vec, std::string label_name);
- int recognizeFace(std::vector<float> &input_vec);
- int deleteLabel(std::string label_name);
- FaceRecognitionResult &result();
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __FACE_RECOGNITION_ADAPTER_H__
-#define __FACE_RECOGNITION_ADAPTER_H__
-
-#include <dlog.h>
-
-#include "EngineConfig.h"
-#include "itask.h"
-#include "face_recognition.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-class FaceRecognitionAdapter : public mediavision::common::ITask
-{
-private:
- std::unique_ptr<FaceRecognition> _face_recognition;
- std::unique_ptr<MediaVision::Common::EngineConfig> _config;
-
-public:
- FaceRecognitionAdapter();
- ~FaceRecognitionAdapter();
-
- std::unique_ptr<MediaVision::Common::EngineConfig> &getConfig()
- {
- return _config;
- }
-
- void setModelInfo(const std::string &model_file, const std::string &meta_file, const std::string &label_file,
- const std::string &model_name) override;
- void setEngineInfo(const std::string &engine_type, const std::string &device_type) override;
- void configure() override;
- unsigned int getNumberOfEngines() override;
- const std::string &getEngineType(unsigned int engine_index) override;
- unsigned int getNumberOfDevices(const std::string &engine_type) override;
- const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
- void prepare() override;
- void perform(InputBaseType &input) override;
- void performAsync(InputBaseType &input) override;
- OutputBaseType &getOutput() override;
- OutputBaseType &getOutputCache() override;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __FACE_RECOGNITION_DSM_H__
-#define __FACE_RECOGNITION_DSM_H__
-
-#include <string>
-
-#include "feature_vector_manager.h"
-#include "data_set_manager.h"
-
-class FaceRecognitionDSM : public DataSetManager
-{
-private:
- void printHeader(FeaVecHeader &fvh);
- bool isHeaderValid(const FeaVecHeader &fvh);
-
-public:
- FaceRecognitionDSM();
- ~FaceRecognitionDSM() = default;
-
- bool isFeatureVectorAllowed(unsigned int label_idx) override;
- void loadDataSet(const std::string &file_name, unsigned int new_label_cnt) override;
- void addDataSet(std::vector<float> &feature_vec, unsigned int label_idx, unsigned int label_cnt) override;
-};
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __FACE_RECOGNITION_FVM_H__
-#define __FACE_RECOGNITION_FVM_H__
-
-#include <string.h>
-#include <vector>
-
-#include "feature_vector_manager.h"
-#include "file_util.h"
-
-class FaceRecognitionFVM : public FeatureVectorManager
-{
-public:
- FaceRecognitionFVM(const std::string feature_vector_file = "feature_vector_file.dat");
- ~FaceRecognitionFVM() = default;
-
- void updateHeader(size_t feature_size, size_t label_cnt, unsigned int data_set_cnt) override;
- void storeData(std::vector<std::vector<float> > &features_vec, std::vector<unsigned int> &label_index) override;
- void remove() override;
-};
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __FACENET_H__
-#define __FACENET_H__
-
-#include <mv_common.h>
-#include <mv_inference_type.h>
-
-#include "EngineConfig.h"
-#include "inference_engine_common_impl.h"
-#include "Inference.h"
-#include "facenet_parser.h"
-#include "face_recognition_type.h"
-#include "machine_learning_preprocess.h"
-#include "MachineLearningType.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-struct FacenetInput : public InputBaseType {
- FacenetInput(mv_source_h src = nullptr) : InputBaseType(src)
- {}
-};
-
-struct FacenetOutput : public OutputBaseType {
- std::vector<std::vector<float> > outputs;
-};
-
-class Facenet
-{
-protected:
- std::unique_ptr<mediavision::inference::Inference> _inference;
- std::unique_ptr<MediaVision::Common::EngineConfig> _config;
- std::unique_ptr<MetaParser> _parser;
- FacenetOutput _result;
- inference_engine_tensor_buffer *_outputTensorBuffer {};
- Preprocess _preprocess;
- std::string _modelFileName;
- std::string _modelMetaFileName;
- std::string _facenetOutputTensorName;
- int _backendType {};
- int _targetDeviceType {};
-
- template<typename T>
- void preprocess(mv_source_h &mv_src, std::shared_ptr<MetaInfo> metaInfo, std::vector<T> &inputVector);
- template<typename T> void inference(std::vector<std::vector<T> > &inputVectors);
-
-public:
- Facenet();
- virtual ~Facenet() = default;
- void parseMetaFile();
- std::shared_ptr<MetaInfo> getInputMetaInfo();
- void configure();
- void prepare();
- template<typename T> void perform(mv_source_h &mv_src, std::shared_ptr<MetaInfo> metaInfo);
- FacenetOutput &result();
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __FACENET_ADAPTER_H__
-#define __FACENET_ADAPTER_H__
-
-#include <dlog.h>
-
-#include "EngineConfig.h"
-#include "itask.h"
-#include "MachineLearningType.h"
-#include "facenet.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-class FacenetAdapter : public mediavision::common::ITask
-{
-private:
- std::unique_ptr<Facenet> _facenet;
-
-public:
- FacenetAdapter();
- ~FacenetAdapter();
-
- void setModelInfo(const std::string &model_file, const std::string &meta_file, const std::string &label_file,
- const std::string &model_name) override;
- void setEngineInfo(const std::string &engine_type, const std::string &device_type) override;
- void configure() override;
- unsigned int getNumberOfEngines() override;
- const std::string &getEngineType(unsigned int engine_index) override;
- unsigned int getNumberOfDevices(const std::string &engine_type) override;
- const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
- void prepare() override;
- void perform(InputBaseType &input) override;
- void performAsync(InputBaseType &input) override;
- OutputBaseType &getOutput() override;
- OutputBaseType &getOutputCache() override;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __FACENET_PARSER_H__
-#define __FACENET_PARSER_H__
-
-#include "MetaParser.h"
-#include "PostprocessParser.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-class FacenetParser : public MetaParser
-{
-private:
- PostprocessParser _postprocessParser;
-
-protected:
- void parsePostprocess(std::shared_ptr<MetaInfo> meta_info, JsonObject *in_obj) override;
-
-public:
- FacenetParser(int task_type = 0);
- ~FacenetParser();
-};
-
-}
-}
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __SIMPLE_SHOT_H__
-#define __SIMPLE_SHOT_H__
-
-#include <mv_inference_type.h>
-#include "training_model.h"
-
-class SimpleShot : public TrainingModel
-{
-private:
- TrainingEngineBackendInfo _engine_info;
-
-private:
- void saveModel(const std::string file_path) override;
- void removeModel(const std::string file_path) override;
-
-public:
- SimpleShot(const training_backend_type_e backend_type, const training_target_type_e target_type,
- const std::vector<size_t> input_tensor_shape, const std::string internal_model_file);
- ~SimpleShot();
-
- // Configure layers for SimpleShot learning.
- void configureModel(int num_of_class) override;
- TrainingEngineBackendInfo &getTrainingEngineInfo() override;
-};
-
-#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <fstream>
+#include <istream>
+#include <tuple>
+#include <map>
+#include <algorithm>
+
+#include <sys/stat.h>
+
+#include <dlog.h>
+#include <mv_common.h>
+#include <mv_private.h>
+
+#include "MvMlException.h"
+#include "TensorBuffer.h"
+#include "FaceRecognition.h"
+#include "FaceRecognitionFVM.h"
+#include "FaceRecognitionDSM.h"
+#include "file_util.h"
+
+using namespace std;
+using namespace mediavision::inference;
+using namespace TrainingEngineInterface::Common;
+using namespace mediavision::machine_learning::face_recognition;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+void FaceRecognition::checkFeatureVectorFile(string fv_file_name, string new_fv_file_name)
+{
+ // Change new feature vector file to existing one in case that current process is terminated just after removing existing feature vector file but
+ // new feature vector file isn't changed to existing one yet.
+ if (FaceRecogUtil::isFileExist(new_fv_file_name) && !FaceRecogUtil::isFileExist(fv_file_name)) {
+ int ret = ::rename(new_fv_file_name.c_str(), fv_file_name.c_str());
+ if (ret)
+ throw InvalidOperation("Fail to rename new feature vector file to original one.");
+
+ return;
+ }
+
+ // Make sure to remove a temp file in case that current process is terminated just after generating new feature vector file
+ // which is not correct file but existing one isn't removed. In this case, existing file is used again.
+ if (FaceRecogUtil::isFileExist(new_fv_file_name)) {
+ int ret = ::remove(new_fv_file_name.c_str());
+ if (ret)
+ throw InvalidOperation("Fail to remove new feature vector file.");
+ }
+}
+
+void FaceRecognition::storeDataSet(unique_ptr<DataSetManager> &data_set, unsigned int label_cnt)
+{
+ try {
+ unique_ptr<FeatureVectorManager> fvm = make_unique<FaceRecognitionFVM>(_config.feature_vector_file_path);
+ unique_ptr<FeatureVectorManager> fvm_new =
+ make_unique<FaceRecognitionFVM>(_config.feature_vector_file_path + ".new");
+
+ // Make sure feature vector file.
+ checkFeatureVectorFile(fvm->getFileName(), fvm_new->getFileName());
+
+ // Write feature vector header.
+ fvm_new->updateHeader(data_set->getFeaVecSize(), label_cnt, data_set->getData().size());
+
+ // Write feature vector and it's label index.
+ fvm_new->storeData(data_set->getData(), data_set->getLabelIdx());
+
+ int ret = 0;
+
+ // Change new data file to existing one.
+ if (FaceRecogUtil::isFileExist(fvm->getFileName())) {
+ ret = ::remove(fvm->getFileName().c_str());
+ if (ret)
+ throw InvalidOperation("Fail to remove feature vector file.");
+ }
+
+ ret = ::rename(fvm_new->getFileName().c_str(), fvm->getFileName().c_str());
+ if (ret)
+ throw InvalidOperation("Fail to rename new feature vector file to original one.");
+ } catch (const BaseException &e) {
+ LOGE("%s", e.what());
+ throw e;
+ }
+}
+
+void FaceRecognition::setConfig(FaceRecognitionConfig &config)
+{
+ _config = config;
+}
+
+int FaceRecognition::initialize()
+{
+ _training_model = make_unique<SimpleShot>(_config.training_engine_backend_type, _config.training_target_device_type,
+ _config.input_tensor_shape, _config.internal_model_file_path);
+
+ _internal = make_unique<Inference>();
+ _label_manager = make_unique<LabelManager>(_config.label_file_path, _config.decision_threshold);
+
+ int ret = _internal->bind(_config.inference_engine_backend_type, _config.inference_target_device_type);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ return ret;
+
+ _status = WorkingStatus::INITIALIZED;
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int FaceRecognition::registerNewFace(std::vector<float> &input_vec, string label_name)
+{
+ if (_status < WorkingStatus::INITIALIZED) {
+ LOGE("Initialization not ready yet.");
+ return MEDIA_VISION_ERROR_INVALID_OPERATION;
+ }
+
+ // TODO. consider data augmentation.
+ try {
+ // Store a new label name to label file if the given label doesn't exist.
+ if (!_label_manager->isExist(label_name))
+ _label_manager->addLabelToFile(label_name);
+
+ // Import label data from a label file.
+ _label_manager->importLabel();
+
+ // Get label index and count.
+ unsigned int label_idx = _label_manager->getLabelIndex(label_name);
+ unsigned int label_cnt = _label_manager->getMaxLabel();
+
+ unique_ptr<DataSetManager> data_set = make_unique<FaceRecognitionDSM>();
+
+ data_set->clear();
+
+ // Load existing feature vectors if the feature vector file exists.
+ if (FaceRecogUtil::isFileExist(_config.feature_vector_file_path) == true) {
+ LOGI("feature vector file already exists so it loads the file first.");
+ data_set->loadDataSet(_config.feature_vector_file_path, label_cnt);
+ }
+
+ // Add new feature vector only in case that feature vector count of given label_idx is less then 5.
+ // It means that only 5 set of feature vector per a label is valid.
+ // TODO. According to feature vector priority, new feature vector should be added.
+ if (data_set->isFeatureVectorAllowed(label_idx))
+ data_set->addDataSet(input_vec, label_idx, label_cnt);
+
+ // Store dataset to feature vector file.
+ storeDataSet(data_set, label_cnt);
+ data_set->clear();
+
+ trainModel();
+ _result.is_valid = false;
+ } catch (const BaseException &e) {
+ LOGE("%s", e.what());
+ return e.getError();
+ }
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+void FaceRecognition::checkResult()
+{
+ // Check decision threshold.
+ if (_result.raw_data[_result.label_idx] < _label_manager->getDecisionThreshold())
+ throw NoData("Not meet decision threshold.");
+
+ float weighted = _result.raw_data[_result.label_idx] * _label_manager->getDecisionWeight();
+
+ // Check decision weight threshold.
+ for (size_t i = 0; i < _result.raw_data.size(); i++) {
+ if (i != _result.label_idx && _result.raw_data[i] > weighted)
+ throw NoData("Not meet decision weight threshold");
+ }
+}
+
+void FaceRecognition::trainModel()
+{
+ // Load existing feature vectors if the feature vector file exists.
+ if (!FaceRecogUtil::isFileExist(_config.feature_vector_file_path))
+ throw InvalidOperation("Feature vector file does not exist.");
+
+ unsigned int label_cnt = _label_manager->getMaxLabel();
+ unique_ptr<DataSetManager> data_set = make_unique<FaceRecognitionDSM>();
+
+ data_set->loadDataSet(_config.feature_vector_file_path, label_cnt);
+
+ // Configure and train the model
+ // TODO: No need to configure the model again if number of labels don't change
+ _training_model->configureModel(label_cnt);
+ _training_model->applyDataSet(data_set);
+ _training_model->compile();
+ _training_model->train();
+
+ _training_model->clearDataSet();
+
+ _status = WorkingStatus::TRAINED;
+}
+
+int FaceRecognition::recognizeFace(std::vector<float> &input_vec)
+{
+ if (_status == WorkingStatus::NONE) {
+ LOGE("Initialization not ready yet.");
+ return MEDIA_VISION_ERROR_INVALID_OPERATION;
+ }
+
+ // Face Recognition has following steps
+ // ------------------------------------
+ // 1. Import label data to in-memory from a file.
+ // 2. Load internal model.
+ // 3. Do an inference with the internal model to get a label.
+ // 4. Get the label best suitable to the output tensor and return the label.
+
+ try {
+ // Import label data from a label file.
+ _label_manager->importLabel();
+
+ if (_label_manager->isEmpty()) {
+ _result.is_valid = false;
+ _status = WorkingStatus::INFERENCED;
+
+ return MEDIA_VISION_ERROR_NO_DATA;
+ }
+
+ if (!FaceRecogUtil::isFileExist(_config.internal_model_file_path)) {
+ LOGE("Internal model file(%s) doesn't exist.", _config.internal_model_file_path.c_str());
+ return MEDIA_VISION_ERROR_NO_DATA;
+ }
+
+ TrainingEngineBackendInfo engine_info = _training_model->getTrainingEngineInfo();
+ vector<string> &input_layers = engine_info.input_layer_names;
+ vector<inference_engine_tensor_info> &input_tensor_info = engine_info.input_tensor_info;
+ vector<string> &output_layers = engine_info.output_layer_names;
+ vector<inference_engine_tensor_info> &output_tensor_info = engine_info.output_tensor_info;
+
+ if (_status != WorkingStatus::INFERENCED) {
+ // Tensor order is NCHW.
+ size_t width = input_tensor_info[0].shape[0];
+ size_t height = input_tensor_info[0].shape[1];
+ size_t ch = input_tensor_info[0].shape[2];
+
+ _internal->configureInputInfo(width, height, 1, ch, 1.0f, 0.0f, MV_INFERENCE_DATA_FLOAT32, input_layers);
+
+ // Output tensor size should be a number of labels so update it.
+ output_tensor_info[0].shape[0] = _label_manager->getMaxLabel();
+
+ _internal->configureOutputInfo(output_layers, output_tensor_info);
+ _internal->configureModelFiles("", _config.internal_model_file_path, "");
+
+ // Load the trained internal model.
+ if (_internal->load() != INFERENCE_ENGINE_ERROR_NONE) {
+ LOGE("Fail to Load.");
+ return MEDIA_VISION_ERROR_INVALID_OPERATION;
+ }
+ }
+
+ std::vector<std::vector<float> > input_tensors = { input_vec };
+
+ // Do inference to the internal model.
+ if (_internal->run<float>(input_tensors) != INFERENCE_ENGINE_ERROR_NONE) {
+ LOGE("fail to inference internal model.");
+ return MEDIA_VISION_ERROR_INVALID_OPERATION;
+ }
+
+ // output layer size should be 1.
+ TensorBuffer tensorBuffer = _internal->getOutputTensorBuffer();
+ inference_engine_tensor_buffer *internal_output_buffer = tensorBuffer.getTensorBuffer(output_layers[0]);
+ if (!internal_output_buffer) {
+ LOGE("fail to get internal output tensor buffer.");
+ return MEDIA_VISION_ERROR_INVALID_PARAMETER;
+ }
+
+ auto raw_buffer = static_cast<float *>(internal_output_buffer->buffer);
+
+ // Update the result because user can want to get the result regardless of the decision threshold
+ // to check how many people are registered to the label file.
+ //
+ // TODO. as for this, we may need to introduce a new API to provide the number of people later.
+ _result.raw_data.clear();
+ copy(raw_buffer, raw_buffer + internal_output_buffer->size / sizeof(float), back_inserter(_result.raw_data));
+
+ string result_str;
+
+ for (const auto &r : _result.raw_data)
+ result_str += to_string(r) + " ";
+
+ LOGD("raw data = %s", result_str.c_str());
+
+ _result.labels.clear();
+ _result.labels = _label_manager->getLabels();
+
+ unsigned int answer_index =
+ max_element(_result.raw_data.begin(), _result.raw_data.end()) - _result.raw_data.begin();
+ _result.label_idx = answer_index;
+ _result.is_valid = true;
+
+ _status = WorkingStatus::INFERENCED;
+
+ checkResult();
+ } catch (const BaseException &e) {
+ LOGE("%s", e.what());
+ return e.getError();
+ }
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int FaceRecognition::deleteLabel(string label_name)
+{
+ if (_status < WorkingStatus::INITIALIZED) {
+ LOGE("Initialization not ready yet.");
+ return MEDIA_VISION_ERROR_INVALID_OPERATION;
+ }
+
+ // Deleting a given label is to remove existing registered person from label and feature vector files.
+
+ try {
+ if (!_label_manager->isExist(label_name)) {
+ LOGE("%s doesn't exist in label file.", label_name.c_str());
+ return MEDIA_VISION_ERROR_INVALID_OPERATION;
+ }
+
+ // Import label data from a label file.
+ _label_manager->importLabel();
+
+ unsigned int target_label_idx = _label_manager->getLabelIndex(label_name);
+ auto label_cnt_ori = _label_manager->getMaxLabel();
+
+ LOGD("Current label count is %zu", label_cnt_ori);
+ LOGD("The index of label name(%s) is %d", label_name.c_str(), target_label_idx);
+
+ // Get label count after removing a given label from the label file.
+ _label_manager->removeLabel(label_name);
+
+ auto label_cnt = _label_manager->getMaxLabel();
+
+ LOGD("Current label count is %zu after removing the label(%s)", label_cnt, label_name.c_str());
+
+ unique_ptr<DataSetManager> data_set = make_unique<FaceRecognitionDSM>();
+ unique_ptr<DataSetManager> data_set_new = make_unique<FaceRecognitionDSM>();
+
+ // feature vectors corresponding to given label aren't removed yet from feature vector file.
+ // So label_cnt_ori is needed.
+ LOGD("Load the original feature vector data from the feature vector file.");
+ data_set->loadDataSet(_config.feature_vector_file_path, label_cnt_ori);
+
+ vector<vector<float> > feature_vectors_old = data_set->getData();
+ vector<unsigned int> label_idx_vectors_old = data_set->getLabelIdx();
+
+ // Write existing feature vectors and its one-hot encoding table with updated label.
+ for (unsigned int idx = 0; idx < feature_vectors_old.size(); ++idx) {
+ // Except the data sets with a given target_label_idx.
+ if (label_idx_vectors_old[idx] == target_label_idx)
+ continue;
+
+ // One-hot encoding table should be updated.
+ // Assume that below label file exists for example,
+ // In label file
+ // -------------
+ // offset 0 : label 1
+ // offset 1 : label 2
+ // offset 2 : label 3
+ //
+ // One hot encoding table should be updated like below after removing label 1,
+ // In label file
+ // -------------
+ // offset 0 : label 2
+ // offset 1 : label 3
+ //
+ // So if the index of removed label less than remaining index then decrease each index.
+ if (label_idx_vectors_old[idx] > target_label_idx)
+ label_idx_vectors_old[idx]--;
+
+ data_set_new->addDataSet(feature_vectors_old[idx], label_idx_vectors_old[idx], label_cnt);
+ }
+
+ // Store only in case that feature vectors exist.
+ if (data_set_new->getData().size() > 0) {
+ storeDataSet(data_set_new, label_cnt);
+ trainModel();
+ } else {
+ _training_model->removeModel();
+ _label_manager->removeFile();
+
+ LOGD("No training data so removed all relevant files.");
+ }
+
+ _result.is_valid = false;
+ } catch (const BaseException &e) {
+ LOGE("%s", e.what());
+ return e.getError();
+ }
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+FaceRecognitionResult &FaceRecognition::result()
+{
+ if (!_result.is_valid)
+ throw NoData("Inference result not ready yet.");
+
+ if (!_label_manager)
+ throw NoData("Label file doesn't exist.");
+
+ try {
+ _label_manager->getLabelString(_result.label, _result.label_idx);
+ } catch (const BaseException &e) {
+ LOGE("%s", e.what());
+ throw e;
+ }
+
+ return _result;
+}
+
+} // machine_learning
+} // mediavision
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "MvMlException.h"
+#include "FaceRecognitionAdapter.h"
+#include "face_recognition_type.h"
+#include <app_common.h>
+
+using namespace std;
+using namespace MediaVision::Common;
+using namespace mediavision::machine_learning::exception;
+using namespace mediavision::machine_learning::face_recognition;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+FaceRecognitionAdapter::FaceRecognitionAdapter()
+{
+ _face_recognition = make_unique<FaceRecognition>();
+}
+
+FaceRecognitionAdapter::~FaceRecognitionAdapter()
+{}
+
+void FaceRecognitionAdapter::setModelInfo(const string &model_file, const string &meta_file, const string &label_file,
+ const string &model_name)
+{
+ throw InvalidOperation("Not yet implemented");
+}
+
+void FaceRecognitionAdapter::setEngineInfo(const string &engine_type, const string &device_type)
+{
+ throw InvalidOperation("Not yet implemented");
+}
+
+void FaceRecognitionAdapter::configure()
+{
+ _config = make_unique<EngineConfig>(string(MV_CONFIG_PATH) + string(FACE_RECOGNITION_META_FILE_NAME));
+
+ string defaultPath;
+
+ int ret = _config->getStringAttribute(string(MV_FACE_RECOGNITION_DEFAULT_PATH), &defaultPath);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get default path.");
+
+ auto trusted_path = app_get_shared_trusted_path();
+
+ if (trusted_path) {
+ defaultPath.insert(0, string(trusted_path));
+ free(trusted_path);
+ } else {
+ defaultPath.insert(0, TEST_RES_PATH "/res/face_recognition/");
+ }
+
+ LOGD("Default path : %s", defaultPath.c_str());
+
+ double decisionThreshold = 0.0f;
+ ret = _config->getDoubleAttribute(string(MV_FACE_RECOGNITION_DECISION_THRESHOLD), &decisionThreshold);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get default decision threshold value.");
+
+ std::vector<int> vecIntValues;
+ ret = _config->getIntegerAttribute(string(MV_FACE_RECOGNITION_INPUT_TENSOR_SHAPE), &vecIntValues);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get input tensor shape.");
+
+ FaceRecognitionConfig config = { TRAINING_TARGET_CPU, // device type for training.
+ TRAINING_BACKEND_NNTRAINER, // backend type for training.
+ MV_INFERENCE_TARGET_DEVICE_CPU, // device type for internal model interface.
+ MV_INFERENCE_BACKEND_NNTRAINER, // backend type for internal model inference.
+ string(defaultPath) + "model_and_weights.ini",
+ string(defaultPath) + "labels.dat",
+ string(defaultPath) + "feature_vector_file.dat",
+ decisionThreshold };
+
+ for (auto value : vecIntValues)
+ config.input_tensor_shape.push_back(static_cast<size_t>(value));
+
+ _face_recognition->setConfig(config);
+}
+
+unsigned int FaceRecognitionAdapter::getNumberOfEngines()
+{
+ throw InvalidOperation("Not yet implemented");
+}
+
+const string &FaceRecognitionAdapter::getEngineType(unsigned int engine_index)
+{
+ throw InvalidOperation("Not yet implemented");
+}
+
+unsigned int FaceRecognitionAdapter::getNumberOfDevices(const string &engine_type)
+{
+ throw InvalidOperation("Not yet implemented");
+}
+
+const string &FaceRecognitionAdapter::getDeviceType(const string &engine_type, unsigned int device_index)
+{
+ throw InvalidOperation("Not yet implemented");
+}
+
+void FaceRecognitionAdapter::prepare()
+{
+ int ret = _face_recognition->initialize();
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to initialize face recognition.");
+}
+
+void FaceRecognitionAdapter::perform(InputBaseType &input)
+{
+ auto &source = static_cast<FaceRecognitionInput &>(input);
+
+ if (source.mode == RequestMode::REGISTER) {
+ if (source.inputs.size() != source.labels.size())
+ throw InvalidParameter("The number of inputs and labels are not matched.");
+
+ for (size_t idx = 0; idx < source.inputs.size(); ++idx) {
+ int ret = _face_recognition->registerNewFace(source.inputs[idx], source.labels[idx]);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to register new face.");
+ }
+
+ return;
+ }
+
+ if (source.mode == RequestMode::INFERENCE) {
+ // _source.inputs.size should be 1.
+ int ret = _face_recognition->recognizeFace(source.inputs[0]);
+ if (ret == MEDIA_VISION_ERROR_NO_DATA)
+ throw NoData("Label not found.");
+
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to request a recognition.");
+
+ return;
+ }
+
+ if (source.mode == RequestMode::DELETE) {
+ for (auto &l : source.labels) {
+ int ret = _face_recognition->deleteLabel(l);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to unregister a given label.");
+ }
+
+ return;
+ }
+}
+
+void FaceRecognitionAdapter::performAsync(InputBaseType &input)
+{
+ throw InvalidOperation("Not support yet.");
+}
+
+OutputBaseType &FaceRecognitionAdapter::getOutput()
+{
+ return _face_recognition->result();
+}
+
+OutputBaseType &FaceRecognitionAdapter::getOutputCache()
+{
+ throw InvalidOperation("Not support yet.");
+}
+
+}
+}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <dlog.h>
+#include <mv_private.h>
+#include <algorithm>
+
+#include "MvMlException.h"
+#include "FaceRecognitionDSM.h"
+
+#define MAX_FEATURE_VECTOR_CNT 5
+#define MAX_FEATURE_SIZE 1024
+#define MAX_NUMBER_OF_LABELS 20
+#define MAX_NUMBER_OF_DATA_SETS (MAX_FEATURE_VECTOR_CNT * MAX_NUMBER_OF_LABELS)
+
+using namespace std;
+using namespace mediavision::machine_learning::exception;
+
+void FaceRecognitionDSM::printHeader(FeaVecHeader &fvh)
+{
+ LOGD("signature = %u", fvh.signature);
+ LOGD("feature vector size = %zu", fvh.feature_size);
+ LOGD("label count = %zu", fvh.label_cnt);
+ LOGD("data set count = %u", fvh.data_set_cnt);
+}
+
+bool FaceRecognitionDSM::isHeaderValid(const FeaVecHeader &fvh)
+{
+ return !(FeatureVectorManager::feature_vector_signature != fvh.signature || MAX_FEATURE_SIZE < fvh.feature_size ||
+ MAX_NUMBER_OF_LABELS < fvh.label_cnt || MAX_NUMBER_OF_DATA_SETS < fvh.data_set_cnt);
+}
+
+FaceRecognitionDSM::FaceRecognitionDSM() : DataSetManager()
+{}
+
+bool FaceRecognitionDSM::isFeatureVectorAllowed(unsigned int label_idx)
+{
+ return (_fv_cnt_per_label[label_idx] < MAX_FEATURE_VECTOR_CNT);
+}
+
+void FaceRecognitionDSM::addDataSet(std::vector<float> &feature_vec, unsigned int label_idx, unsigned int label_cnt)
+{
+ _data.push_back(feature_vec);
+ _label_index.push_back(label_idx);
+ _fv_cnt_per_label[label_idx]++;
+
+ vector<float> oneHotEncoding;
+
+ for (size_t num = 0; num < label_cnt; ++num)
+ oneHotEncoding.push_back(label_idx == num ? 1.0f : 0.0f);
+
+ _labels.push_back(oneHotEncoding);
+ _feature_vector_size = feature_vec.size();
+ _label_count = label_cnt;
+}
+
+void FaceRecognitionDSM::loadDataSet(const string &file_name, unsigned int new_label_cnt)
+{
+ std::ifstream inFile(file_name);
+
+ if (!inFile.is_open())
+ throw InvalidOperation("fail to open a file.");
+
+ FeaVecHeader fvh;
+
+ inFile.read((char *) &fvh, sizeof(FeaVecHeader));
+ if (inFile.gcount() != sizeof(FeaVecHeader))
+ throw InvalidOperation("Invalid feature vector file.");
+
+ printHeader(fvh);
+
+ if (!isHeaderValid(fvh))
+ throw InvalidOperation("Wrong feature vector header.");
+
+ /*
+ * stride line format is as follows
+ * ********************************
+ * ____________________________
+ * |feature vector|label index|
+ * ----------------------------
+ */
+ size_t line_size_in_bytes = fvh.feature_size * sizeof(float) + sizeof(unsigned int);
+
+ _feature_vector_size = fvh.feature_size;
+ _label_count = fvh.label_cnt;
+
+ vector<float> line_data(fvh.feature_size + 1);
+
+ for (size_t idx = 0; idx < fvh.data_set_cnt; ++idx) {
+ inFile.read(reinterpret_cast<char *>(line_data.data()), static_cast<streamsize>(line_size_in_bytes));
+
+ vector<float> data;
+
+ copy_n(line_data.begin(), _feature_vector_size, back_inserter(data));
+ _data.push_back(data);
+
+ unsigned int label_idx;
+
+ memcpy(&label_idx, reinterpret_cast<void *>(line_data.data() + _feature_vector_size), sizeof(unsigned int));
+
+ vector<float> label;
+
+ // max label count may be changed so update one hot encoding table.
+ for (size_t num = 0; num < new_label_cnt; ++num)
+ label.push_back(label_idx == num ? 1.0f : 0.0f);
+
+ _labels.push_back(label);
+ _label_index.push_back(label_idx);
+
+ _fv_cnt_per_label[label_idx]++;
+ }
+}
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <fstream>
+#include <unistd.h>
+
+#include "MvMlException.h"
+#include "FaceRecognitionFVM.h"
+
+using namespace std;
+using namespace mediavision::machine_learning::exception;
+
+FaceRecognitionFVM::FaceRecognitionFVM(const string feature_vector_file) : FeatureVectorManager(feature_vector_file)
+{}
+
+void FaceRecognitionFVM::updateHeader(size_t feature_size, size_t label_cnt, unsigned int data_set_cnt)
+{
+ fstream headerFile {};
+
+ // When trying to write header data for the first time, the feature vector file doesn't exist,
+ // so create a new file with ios::out flag. Otherwise, open the feature vector file with ios::in | ios::out flags
+ // to update existing header data in the feature vector file.
+ if (access(_feature_vector_file.c_str(), F_OK))
+ headerFile.open(_feature_vector_file.c_str(), ios::out | ios::binary);
+ else
+ headerFile.open(_feature_vector_file.c_str(), ios::in | ios::out | ios::binary);
+
+ if (!headerFile.is_open())
+ throw InvalidOperation("fail to open a file");
+
+ FeaVecHeader header { FeatureVectorManager::feature_vector_signature, feature_size, label_cnt, data_set_cnt };
+
+ headerFile.write((const char *) &header, sizeof(FeaVecHeader));
+}
+
+void FaceRecognitionFVM::storeData(vector<vector<float> > &features_vec, vector<unsigned int> &label_index)
+{
+ fstream writeFile { _feature_vector_file, ios::binary | ios::app };
+
+ if (!writeFile.is_open())
+ throw InvalidOperation("fail to open a file.");
+
+ for (size_t idx = 0; idx < features_vec.size(); ++idx) {
+ writeFile.write(reinterpret_cast<char *>(features_vec[idx].data()),
+ static_cast<streamsize>(features_vec[idx].size() * sizeof(float)));
+ writeFile.write(reinterpret_cast<char *>(&label_index[idx]), static_cast<streamsize>(sizeof(unsigned int)));
+ }
+}
+
+void FaceRecognitionFVM::remove()
+{
+ // Remove existing file forcely.
+ int ret = ::remove(_feature_vector_file.c_str());
+ if (ret)
+ throw InvalidOperation("Fail to remove feature vector file.");
+}
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <map>
+#include <memory>
+#include <algorithm>
+
+#include "MvMlException.h"
+#include "Facenet.h"
+#include "face_recognition_type.h"
+
+using namespace std;
+using namespace mediavision::inference;
+using namespace MediaVision::Common;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+Facenet::Facenet() : _backendType(), _targetDeviceType()
+{
+ _inference = make_unique<Inference>();
+ _parser = make_unique<FacenetParser>();
+}
+
+static bool isJsonFile(const string &fileName)
+{
+ return (!fileName.substr(fileName.find_last_of(".") + 1).compare("json"));
+}
+
+void Facenet::parseMetaFile()
+{
+ _config = make_unique<EngineConfig>(string(MV_CONFIG_PATH) + string(FACE_RECOGNITION_META_FILE_NAME));
+
+ int ret = _config->getIntegerAttribute(string(MV_FACENET_BACKEND_TYPE), &_backendType);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get backend engine type.");
+
+ ret = _config->getIntegerAttribute(string(MV_FACENET_TARGET_DEVICE_TYPE), &_targetDeviceType);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get target device type.");
+
+ string modelDefaultPath;
+
+ ret = _config->getStringAttribute(MV_FACENET_MODEL_DEFAULT_PATH, &modelDefaultPath);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get model default path.");
+
+ ret = _config->getStringAttribute(MV_FACENET_MODEL_FILE_NAME, &_modelFileName);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get model file name.");
+
+ _modelFileName = modelDefaultPath + _modelFileName;
+
+ ret = _config->getStringAttribute(MV_FACENET_MODEL_META_FILE_NAME, &_modelMetaFileName);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get model meta file name.");
+
+ _modelMetaFileName = modelDefaultPath + _modelMetaFileName;
+
+ ret = _config->getStringAttribute(MV_FACENET_OUTPUT_TENSOR_NAME, &_facenetOutputTensorName);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to get facenet output tensor name.");
+
+ if (_modelMetaFileName.empty())
+ throw InvalidOperation("Model meta file doesn't exist.");
+
+ if (!isJsonFile(_modelMetaFileName))
+ throw InvalidOperation("Model meta file should be json.");
+
+ _parser->load(_modelMetaFileName);
+}
+
+void Facenet::configure()
+{
+ int ret = _inference->bind(_backendType, _targetDeviceType);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to bind a backend engine.");
+}
+
+void Facenet::prepare()
+{
+ int ret = _inference->configureInputMetaInfo(_parser->getInputMetaMap());
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to configure input tensor info from meta file.");
+
+ ret = _inference->configureOutputMetaInfo(_parser->getOutputMetaMap());
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to configure output tensor info from meta file.");
+
+ _inference->configureModelFiles("", _modelFileName, "");
+
+ // Request to load model files to a backend engine.
+ ret = _inference->load();
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to load model files.");
+}
+
+shared_ptr<MetaInfo> Facenet::getInputMetaInfo()
+{
+ TensorBuffer &tensor_buffer = _inference->getInputTensorBuffer();
+ IETensorBuffer &tensor_info_map = tensor_buffer.getIETensorBuffer();
+
+ // TODO. consider using multiple tensors later.
+ if (tensor_info_map.size() != 1)
+ throw InvalidOperation("Input tensor count not invalid.");
+
+ auto tensor_buffer_iter = tensor_info_map.begin();
+
+ // Get the meta information corresponding to a given input tensor name.
+ return _parser->getInputMetaMap()[tensor_buffer_iter->first];
+}
+
+template<typename T>
+void Facenet::preprocess(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo, vector<T> &inputVector)
+{
+ LOGI("ENTER");
+
+ PreprocessConfig config = { false,
+ metaInfo->colorSpace,
+ metaInfo->dataType,
+ metaInfo->getChannel(),
+ metaInfo->getWidth(),
+ metaInfo->getHeight() };
+
+ auto normalization = static_pointer_cast<DecodingNormal>(metaInfo->decodingTypeMap.at(DecodingType::NORMAL));
+ if (normalization) {
+ config.normalize = normalization->use;
+ config.mean = normalization->mean;
+ config.std = normalization->std;
+ }
+
+ auto quantization =
+ static_pointer_cast<DecodingQuantization>(metaInfo->decodingTypeMap.at(DecodingType::QUANTIZATION));
+ if (quantization) {
+ config.quantize = quantization->use;
+ config.scale = quantization->scale;
+ config.zeropoint = quantization->zeropoint;
+ }
+
+ _preprocess.setConfig(config);
+ _preprocess.run<T>(mv_src, inputVector);
+
+ LOGI("LEAVE");
+}
+
+template<typename T> void Facenet::inference(vector<vector<T> > &inputVectors)
+{
+ LOGI("ENTER");
+
+ int ret = _inference->run<T>(inputVectors);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to run inference");
+
+ LOGI("LEAVE");
+}
+
+template<typename T> void Facenet::perform(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo)
+{
+ vector<T> inputVector;
+
+ preprocess<T>(mv_src, metaInfo, inputVector);
+
+ vector<vector<T> > inputVectors = { inputVector };
+
+ inference<T>(inputVectors);
+}
+
+FacenetOutput &Facenet::result()
+{
+ TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
+
+ // Make sure to clear _result.outputs vectors because if not clear then other output_vector will be pushed to _result.outputs
+ // and it results in sending wrong output vector to face recognition framework.
+ _result.outputs.clear();
+
+ _outputTensorBuffer = tensor_buffer_obj.getTensorBuffer(_facenetOutputTensorName);
+ if (!_outputTensorBuffer)
+ throw InvalidOperation("No output tensor.");
+
+ float *buffer = reinterpret_cast<float *>(_outputTensorBuffer->buffer);
+
+ _result.outputs.push_back(vector<float>(buffer, buffer + _outputTensorBuffer->size / sizeof(float)));
+
+ return _result;
+}
+
+template void Facenet::preprocess<float>(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo,
+ vector<float> &inputVector);
+template void Facenet::inference<float>(vector<vector<float> > &inputVectors);
+template void Facenet::perform<float>(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo);
+
+template void Facenet::preprocess<unsigned char>(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo,
+ vector<unsigned char> &inputVector);
+template void Facenet::inference<unsigned char>(vector<vector<unsigned char> > &inputVectors);
+template void Facenet::perform<unsigned char>(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo);
+
+}
+}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "MvMlException.h"
+#include "FacenetAdapter.h"
+
+using namespace std;
+using namespace MediaVision::Common;
+using namespace mediavision::machine_learning;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+FacenetAdapter::FacenetAdapter()
+{
+ _facenet = make_unique<Facenet>();
+}
+
+FacenetAdapter::~FacenetAdapter()
+{}
+
+void FacenetAdapter::setModelInfo(const string &model_file, const string &meta_file, const string &label_file,
+ const string &model_name)
+{}
+
+void FacenetAdapter::setEngineInfo(const string &engine_type, const string &device_type)
+{}
+
+void FacenetAdapter::configure()
+{
+ _facenet->parseMetaFile();
+ _facenet->configure();
+}
+
+unsigned int FacenetAdapter::getNumberOfEngines()
+{
+ throw InvalidOperation("Not yet implemented");
+}
+
+const string &FacenetAdapter::getEngineType(unsigned int engine_index)
+{
+ throw InvalidOperation("Not yet implemented");
+}
+
+unsigned int FacenetAdapter::getNumberOfDevices(const string &engine_type)
+{
+ throw InvalidOperation("Not yet implemented");
+}
+
+const string &FacenetAdapter::getDeviceType(const string &engine_type, unsigned int device_index)
+{
+ throw InvalidOperation("Not yet implemented");
+}
+
+void FacenetAdapter::prepare()
+{
+ _facenet->prepare();
+}
+
+void FacenetAdapter::perform(InputBaseType &input)
+{
+ auto &source = static_cast<FacenetInput &>(input);
+ shared_ptr<MetaInfo> metaInfo = _facenet->getInputMetaInfo();
+ if (metaInfo->dataType == MV_INFERENCE_DATA_UINT8)
+ _facenet->perform<unsigned char>(source.inference_src, metaInfo);
+ else if (metaInfo->dataType == MV_INFERENCE_DATA_FLOAT32)
+ _facenet->perform<float>(source.inference_src, metaInfo);
+ else
+ throw InvalidOperation("Invalid model data type.");
+}
+
+void FacenetAdapter::performAsync(InputBaseType &input)
+{
+ throw InvalidOperation("Not support yet.");
+}
+
+OutputBaseType &FacenetAdapter::getOutput()
+{
+ return _facenet->result();
+}
+
+OutputBaseType &FacenetAdapter::getOutputCache()
+{
+ throw InvalidOperation("Not support yet.");
+}
+
+}
+}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "MvMlException.h"
+#include "FacenetParser.h"
+
+using namespace std;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+FacenetParser::FacenetParser(int task_type)
+{
+ LOGI("ENTER");
+ LOGI("LEAVE");
+}
+
+FacenetParser::~FacenetParser()
+{}
+
+void FacenetParser::parsePostprocess(shared_ptr<MetaInfo> meta_info, JsonObject *in_obj)
+{
+ LOGI("ENTER");
+
+ LOGI("tensor name : %s", meta_info->name.c_str());
+
+ if (json_object_has_member(in_obj, "box"))
+ _postprocessParser.parseBox(meta_info, in_obj);
+
+ LOGI("LEAVE");
+}
+
+}
+}
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <string.h>
-#include <fstream>
-#include <istream>
-#include <tuple>
-#include <map>
-#include <algorithm>
-
-#include <sys/stat.h>
-
-#include <dlog.h>
-#include <mv_common.h>
-#include <mv_private.h>
-
-#include "machine_learning_exception.h"
-#include "TensorBuffer.h"
-#include "face_recognition.h"
-#include "face_recognition_fvm.h"
-#include "face_recognition_dsm.h"
-#include "file_util.h"
-
-using namespace std;
-using namespace mediavision::inference;
-using namespace TrainingEngineInterface::Common;
-using namespace mediavision::machine_learning::face_recognition;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-void FaceRecognition::checkFeatureVectorFile(string fv_file_name, string new_fv_file_name)
-{
- // Change new feature vector file to existing one in case that current process is terminated just after removing existing feature vector file but
- // new feature vector file isn't changed to existing one yet.
- if (FaceRecogUtil::isFileExist(new_fv_file_name) && !FaceRecogUtil::isFileExist(fv_file_name)) {
- int ret = ::rename(new_fv_file_name.c_str(), fv_file_name.c_str());
- if (ret)
- throw InvalidOperation("Fail to rename new feature vector file to original one.");
-
- return;
- }
-
- // Make sure to remove a temp file in case that current process is terminated just after generating new feature vector file
- // which is not correct file but existing one isn't removed. In this case, existing file is used again.
- if (FaceRecogUtil::isFileExist(new_fv_file_name)) {
- int ret = ::remove(new_fv_file_name.c_str());
- if (ret)
- throw InvalidOperation("Fail to remove new feature vector file.");
- }
-}
-
-void FaceRecognition::storeDataSet(unique_ptr<DataSetManager> &data_set, unsigned int label_cnt)
-{
- try {
- unique_ptr<FeatureVectorManager> fvm = make_unique<FaceRecognitionFVM>(_config.feature_vector_file_path);
- unique_ptr<FeatureVectorManager> fvm_new =
- make_unique<FaceRecognitionFVM>(_config.feature_vector_file_path + ".new");
-
- // Make sure feature vector file.
- checkFeatureVectorFile(fvm->getFileName(), fvm_new->getFileName());
-
- // Write feature vector header.
- fvm_new->updateHeader(data_set->getFeaVecSize(), label_cnt, data_set->getData().size());
-
- // Write feature vector and it's label index.
- fvm_new->storeData(data_set->getData(), data_set->getLabelIdx());
-
- int ret = 0;
-
- // Change new data file to existing one.
- if (FaceRecogUtil::isFileExist(fvm->getFileName())) {
- ret = ::remove(fvm->getFileName().c_str());
- if (ret)
- throw InvalidOperation("Fail to remove feature vector file.");
- }
-
- ret = ::rename(fvm_new->getFileName().c_str(), fvm->getFileName().c_str());
- if (ret)
- throw InvalidOperation("Fail to rename new feature vector file to original one.");
- } catch (const BaseException &e) {
- LOGE("%s", e.what());
- throw e;
- }
-}
-
-void FaceRecognition::setConfig(FaceRecognitionConfig &config)
-{
- _config = config;
-}
-
-int FaceRecognition::initialize()
-{
- _training_model = make_unique<SimpleShot>(_config.training_engine_backend_type, _config.training_target_device_type,
- _config.input_tensor_shape, _config.internal_model_file_path);
-
- _internal = make_unique<Inference>();
- _label_manager = make_unique<LabelManager>(_config.label_file_path, _config.decision_threshold);
-
- int ret = _internal->bind(_config.inference_engine_backend_type, _config.inference_target_device_type);
- if (ret != MEDIA_VISION_ERROR_NONE)
- return ret;
-
- _status = WorkingStatus::INITIALIZED;
-
- return MEDIA_VISION_ERROR_NONE;
-}
-
-int FaceRecognition::registerNewFace(std::vector<float> &input_vec, string label_name)
-{
- if (_status < WorkingStatus::INITIALIZED) {
- LOGE("Initialization not ready yet.");
- return MEDIA_VISION_ERROR_INVALID_OPERATION;
- }
-
- // TODO. consider data augmentation.
- try {
- // Store a new label name to label file if the given label doesn't exist.
- if (!_label_manager->isExist(label_name))
- _label_manager->addLabelToFile(label_name);
-
- // Import label data from a label file.
- _label_manager->importLabel();
-
- // Get label index and count.
- unsigned int label_idx = _label_manager->getLabelIndex(label_name);
- unsigned int label_cnt = _label_manager->getMaxLabel();
-
- unique_ptr<DataSetManager> data_set = make_unique<FaceRecognitionDSM>();
-
- data_set->clear();
-
- // Load existing feature vectors if the feature vector file exists.
- if (FaceRecogUtil::isFileExist(_config.feature_vector_file_path) == true) {
- LOGI("feature vector file already exists so it loads the file first.");
- data_set->loadDataSet(_config.feature_vector_file_path, label_cnt);
- }
-
- // Add new feature vector only in case that feature vector count of given label_idx is less then 5.
- // It means that only 5 set of feature vector per a label is valid.
- // TODO. According to feature vector priority, new feature vector should be added.
- if (data_set->isFeatureVectorAllowed(label_idx))
- data_set->addDataSet(input_vec, label_idx, label_cnt);
-
- // Store dataset to feature vector file.
- storeDataSet(data_set, label_cnt);
- data_set->clear();
-
- trainModel();
- _result.is_valid = false;
- } catch (const BaseException &e) {
- LOGE("%s", e.what());
- return e.getError();
- }
-
- return MEDIA_VISION_ERROR_NONE;
-}
-
-void FaceRecognition::checkResult()
-{
- // Check decision threshold.
- if (_result.raw_data[_result.label_idx] < _label_manager->getDecisionThreshold())
- throw NoData("Not meet decision threshold.");
-
- float weighted = _result.raw_data[_result.label_idx] * _label_manager->getDecisionWeight();
-
- // Check decision weight threshold.
- for (size_t i = 0; i < _result.raw_data.size(); i++) {
- if (i != _result.label_idx && _result.raw_data[i] > weighted)
- throw NoData("Not meet decision weight threshold");
- }
-}
-
-void FaceRecognition::trainModel()
-{
- // Load existing feature vectors if the feature vector file exists.
- if (!FaceRecogUtil::isFileExist(_config.feature_vector_file_path))
- throw InvalidOperation("Feature vector file does not exist.");
-
- unsigned int label_cnt = _label_manager->getMaxLabel();
- unique_ptr<DataSetManager> data_set = make_unique<FaceRecognitionDSM>();
-
- data_set->loadDataSet(_config.feature_vector_file_path, label_cnt);
-
- // Configure and train the model
- // TODO: No need to configure the model again if number of labels don't change
- _training_model->configureModel(label_cnt);
- _training_model->applyDataSet(data_set);
- _training_model->compile();
- _training_model->train();
-
- _training_model->clearDataSet();
-
- _status = WorkingStatus::TRAINED;
-}
-
-int FaceRecognition::recognizeFace(std::vector<float> &input_vec)
-{
- if (_status == WorkingStatus::NONE) {
- LOGE("Initialization not ready yet.");
- return MEDIA_VISION_ERROR_INVALID_OPERATION;
- }
-
- // Face Recognition has following steps
- // ------------------------------------
- // 1. Import label data to in-memory from a file.
- // 2. Load internal model.
- // 3. Do an inference with the internal model to get a label.
- // 4. Get the label best suitable to the output tensor and return the label.
-
- try {
- // Import label data from a label file.
- _label_manager->importLabel();
-
- if (_label_manager->isEmpty()) {
- _result.is_valid = false;
- _status = WorkingStatus::INFERENCED;
-
- return MEDIA_VISION_ERROR_NO_DATA;
- }
-
- if (!FaceRecogUtil::isFileExist(_config.internal_model_file_path)) {
- LOGE("Internal model file(%s) doesn't exist.", _config.internal_model_file_path.c_str());
- return MEDIA_VISION_ERROR_NO_DATA;
- }
-
- TrainingEngineBackendInfo engine_info = _training_model->getTrainingEngineInfo();
- vector<string> &input_layers = engine_info.input_layer_names;
- vector<inference_engine_tensor_info> &input_tensor_info = engine_info.input_tensor_info;
- vector<string> &output_layers = engine_info.output_layer_names;
- vector<inference_engine_tensor_info> &output_tensor_info = engine_info.output_tensor_info;
-
- if (_status != WorkingStatus::INFERENCED) {
- // Tensor order is NCHW.
- size_t width = input_tensor_info[0].shape[0];
- size_t height = input_tensor_info[0].shape[1];
- size_t ch = input_tensor_info[0].shape[2];
-
- _internal->configureInputInfo(width, height, 1, ch, 1.0f, 0.0f, MV_INFERENCE_DATA_FLOAT32, input_layers);
-
- // Output tensor size should be a number of labels so update it.
- output_tensor_info[0].shape[0] = _label_manager->getMaxLabel();
-
- _internal->configureOutputInfo(output_layers, output_tensor_info);
- _internal->configureModelFiles("", _config.internal_model_file_path, "");
-
- // Load the trained internal model.
- if (_internal->load() != INFERENCE_ENGINE_ERROR_NONE) {
- LOGE("Fail to Load.");
- return MEDIA_VISION_ERROR_INVALID_OPERATION;
- }
- }
-
- std::vector<std::vector<float> > input_tensors = { input_vec };
-
- // Do inference to the internal model.
- if (_internal->run<float>(input_tensors) != INFERENCE_ENGINE_ERROR_NONE) {
- LOGE("fail to inference internal model.");
- return MEDIA_VISION_ERROR_INVALID_OPERATION;
- }
-
- // output layer size should be 1.
- TensorBuffer tensorBuffer = _internal->getOutputTensorBuffer();
- inference_engine_tensor_buffer *internal_output_buffer = tensorBuffer.getTensorBuffer(output_layers[0]);
- if (!internal_output_buffer) {
- LOGE("fail to get internal output tensor buffer.");
- return MEDIA_VISION_ERROR_INVALID_PARAMETER;
- }
-
- auto raw_buffer = static_cast<float *>(internal_output_buffer->buffer);
-
- // Update the result because user can want to get the result regardless of the decision threshold
- // to check how many people are registered to the label file.
- //
- // TODO. as for this, we may need to introduce a new API to provide the number of people later.
- _result.raw_data.clear();
- copy(raw_buffer, raw_buffer + internal_output_buffer->size / sizeof(float), back_inserter(_result.raw_data));
-
- string result_str;
-
- for (const auto &r : _result.raw_data)
- result_str += to_string(r) + " ";
-
- LOGD("raw data = %s", result_str.c_str());
-
- _result.labels.clear();
- _result.labels = _label_manager->getLabels();
-
- unsigned int answer_index =
- max_element(_result.raw_data.begin(), _result.raw_data.end()) - _result.raw_data.begin();
- _result.label_idx = answer_index;
- _result.is_valid = true;
-
- _status = WorkingStatus::INFERENCED;
-
- checkResult();
- } catch (const BaseException &e) {
- LOGE("%s", e.what());
- return e.getError();
- }
-
- return MEDIA_VISION_ERROR_NONE;
-}
-
-int FaceRecognition::deleteLabel(string label_name)
-{
- if (_status < WorkingStatus::INITIALIZED) {
- LOGE("Initialization not ready yet.");
- return MEDIA_VISION_ERROR_INVALID_OPERATION;
- }
-
- // Deleting a given label is to remove existing registered person from label and feature vector files.
-
- try {
- if (!_label_manager->isExist(label_name)) {
- LOGE("%s doesn't exist in label file.", label_name.c_str());
- return MEDIA_VISION_ERROR_INVALID_OPERATION;
- }
-
- // Import label data from a label file.
- _label_manager->importLabel();
-
- unsigned int target_label_idx = _label_manager->getLabelIndex(label_name);
- auto label_cnt_ori = _label_manager->getMaxLabel();
-
- LOGD("Current label count is %zu", label_cnt_ori);
- LOGD("The index of label name(%s) is %d", label_name.c_str(), target_label_idx);
-
- // Get label count after removing a given label from the label file.
- _label_manager->removeLabel(label_name);
-
- auto label_cnt = _label_manager->getMaxLabel();
-
- LOGD("Current label count is %zu after removing the label(%s)", label_cnt, label_name.c_str());
-
- unique_ptr<DataSetManager> data_set = make_unique<FaceRecognitionDSM>();
- unique_ptr<DataSetManager> data_set_new = make_unique<FaceRecognitionDSM>();
-
- // feature vectors corresponding to given label aren't removed yet from feature vector file.
- // So label_cnt_ori is needed.
- LOGD("Load the original feature vector data from the feature vector file.");
- data_set->loadDataSet(_config.feature_vector_file_path, label_cnt_ori);
-
- vector<vector<float> > feature_vectors_old = data_set->getData();
- vector<unsigned int> label_idx_vectors_old = data_set->getLabelIdx();
-
- // Write existing feature vectors and its one-hot encoding table with updated label.
- for (unsigned int idx = 0; idx < feature_vectors_old.size(); ++idx) {
- // Except the data sets with a given target_label_idx.
- if (label_idx_vectors_old[idx] == target_label_idx)
- continue;
-
- // One-hot encoding table should be updated.
- // Assume that below label file exists for example,
- // In label file
- // -------------
- // offset 0 : label 1
- // offset 1 : label 2
- // offset 2 : label 3
- //
- // One hot encoding table should be updated like below after removing label 1,
- // In label file
- // -------------
- // offset 0 : label 2
- // offset 1 : label 3
- //
- // So if the index of removed label less than remaining index then decrease each index.
- if (label_idx_vectors_old[idx] > target_label_idx)
- label_idx_vectors_old[idx]--;
-
- data_set_new->addDataSet(feature_vectors_old[idx], label_idx_vectors_old[idx], label_cnt);
- }
-
- // Store only in case that feature vectors exist.
- if (data_set_new->getData().size() > 0) {
- storeDataSet(data_set_new, label_cnt);
- trainModel();
- } else {
- _training_model->removeModel();
- _label_manager->removeFile();
-
- LOGD("No training data so removed all relevant files.");
- }
-
- _result.is_valid = false;
- } catch (const BaseException &e) {
- LOGE("%s", e.what());
- return e.getError();
- }
-
- return MEDIA_VISION_ERROR_NONE;
-}
-
-FaceRecognitionResult &FaceRecognition::result()
-{
- if (!_result.is_valid)
- throw NoData("Inference result not ready yet.");
-
- if (!_label_manager)
- throw NoData("Label file doesn't exist.");
-
- try {
- _label_manager->getLabelString(_result.label, _result.label_idx);
- } catch (const BaseException &e) {
- LOGE("%s", e.what());
- throw e;
- }
-
- return _result;
-}
-
-} // machine_learning
-} // mediavision
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "machine_learning_exception.h"
-#include "face_recognition_adapter.h"
-#include "face_recognition_type.h"
-#include <app_common.h>
-
-using namespace std;
-using namespace MediaVision::Common;
-using namespace mediavision::machine_learning::exception;
-using namespace mediavision::machine_learning::face_recognition;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-FaceRecognitionAdapter::FaceRecognitionAdapter()
-{
- _face_recognition = make_unique<FaceRecognition>();
-}
-
-FaceRecognitionAdapter::~FaceRecognitionAdapter()
-{}
-
-void FaceRecognitionAdapter::setModelInfo(const string &model_file, const string &meta_file, const string &label_file,
- const string &model_name)
-{
- throw InvalidOperation("Not yet implemented");
-}
-
-void FaceRecognitionAdapter::setEngineInfo(const string &engine_type, const string &device_type)
-{
- throw InvalidOperation("Not yet implemented");
-}
-
-void FaceRecognitionAdapter::configure()
-{
- _config = make_unique<EngineConfig>(string(MV_CONFIG_PATH) + string(FACE_RECOGNITION_META_FILE_NAME));
-
- string defaultPath;
-
- int ret = _config->getStringAttribute(string(MV_FACE_RECOGNITION_DEFAULT_PATH), &defaultPath);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get default path.");
-
- auto trusted_path = app_get_shared_trusted_path();
-
- if (trusted_path) {
- defaultPath.insert(0, string(trusted_path));
- free(trusted_path);
- } else {
- defaultPath.insert(0, TEST_RES_PATH "/res/face_recognition/");
- }
-
- LOGD("Default path : %s", defaultPath.c_str());
-
- double decisionThreshold = 0.0f;
- ret = _config->getDoubleAttribute(string(MV_FACE_RECOGNITION_DECISION_THRESHOLD), &decisionThreshold);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get default decision threshold value.");
-
- std::vector<int> vecIntValues;
- ret = _config->getIntegerAttribute(string(MV_FACE_RECOGNITION_INPUT_TENSOR_SHAPE), &vecIntValues);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get input tensor shape.");
-
- FaceRecognitionConfig config = { TRAINING_TARGET_CPU, // device type for training.
- TRAINING_BACKEND_NNTRAINER, // backend type for training.
- MV_INFERENCE_TARGET_DEVICE_CPU, // device type for internal model interface.
- MV_INFERENCE_BACKEND_NNTRAINER, // backend type for internal model inference.
- string(defaultPath) + "model_and_weights.ini",
- string(defaultPath) + "labels.dat",
- string(defaultPath) + "feature_vector_file.dat",
- decisionThreshold };
-
- for (auto value : vecIntValues)
- config.input_tensor_shape.push_back(static_cast<size_t>(value));
-
- _face_recognition->setConfig(config);
-}
-
-unsigned int FaceRecognitionAdapter::getNumberOfEngines()
-{
- throw InvalidOperation("Not yet implemented");
-}
-
-const string &FaceRecognitionAdapter::getEngineType(unsigned int engine_index)
-{
- throw InvalidOperation("Not yet implemented");
-}
-
-unsigned int FaceRecognitionAdapter::getNumberOfDevices(const string &engine_type)
-{
- throw InvalidOperation("Not yet implemented");
-}
-
-const string &FaceRecognitionAdapter::getDeviceType(const string &engine_type, unsigned int device_index)
-{
- throw InvalidOperation("Not yet implemented");
-}
-
-void FaceRecognitionAdapter::prepare()
-{
- int ret = _face_recognition->initialize();
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to initialize face recognition.");
-}
-
-void FaceRecognitionAdapter::perform(InputBaseType &input)
-{
- auto &source = static_cast<FaceRecognitionInput &>(input);
-
- if (source.mode == RequestMode::REGISTER) {
- if (source.inputs.size() != source.labels.size())
- throw InvalidParameter("The number of inputs and labels are not matched.");
-
- for (size_t idx = 0; idx < source.inputs.size(); ++idx) {
- int ret = _face_recognition->registerNewFace(source.inputs[idx], source.labels[idx]);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to register new face.");
- }
-
- return;
- }
-
- if (source.mode == RequestMode::INFERENCE) {
- // _source.inputs.size should be 1.
- int ret = _face_recognition->recognizeFace(source.inputs[0]);
- if (ret == MEDIA_VISION_ERROR_NO_DATA)
- throw NoData("Label not found.");
-
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to request a recognition.");
-
- return;
- }
-
- if (source.mode == RequestMode::DELETE) {
- for (auto &l : source.labels) {
- int ret = _face_recognition->deleteLabel(l);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to unregister a given label.");
- }
-
- return;
- }
-}
-
-void FaceRecognitionAdapter::performAsync(InputBaseType &input)
-{
- throw InvalidOperation("Not support yet.");
-}
-
-OutputBaseType &FaceRecognitionAdapter::getOutput()
-{
- return _face_recognition->result();
-}
-
-OutputBaseType &FaceRecognitionAdapter::getOutputCache()
-{
- throw InvalidOperation("Not support yet.");
-}
-
-}
-}
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <dlog.h>
-#include <mv_private.h>
-#include <algorithm>
-
-#include "machine_learning_exception.h"
-#include "face_recognition_dsm.h"
-
-#define MAX_FEATURE_VECTOR_CNT 5
-#define MAX_FEATURE_SIZE 1024
-#define MAX_NUMBER_OF_LABELS 20
-#define MAX_NUMBER_OF_DATA_SETS (MAX_FEATURE_VECTOR_CNT * MAX_NUMBER_OF_LABELS)
-
-using namespace std;
-using namespace mediavision::machine_learning::exception;
-
-void FaceRecognitionDSM::printHeader(FeaVecHeader &fvh)
-{
- LOGD("signature = %u", fvh.signature);
- LOGD("feature vector size = %zu", fvh.feature_size);
- LOGD("label count = %zu", fvh.label_cnt);
- LOGD("data set count = %u", fvh.data_set_cnt);
-}
-
-bool FaceRecognitionDSM::isHeaderValid(const FeaVecHeader &fvh)
-{
- return !(FeatureVectorManager::feature_vector_signature != fvh.signature || MAX_FEATURE_SIZE < fvh.feature_size ||
- MAX_NUMBER_OF_LABELS < fvh.label_cnt || MAX_NUMBER_OF_DATA_SETS < fvh.data_set_cnt);
-}
-
-FaceRecognitionDSM::FaceRecognitionDSM() : DataSetManager()
-{}
-
-bool FaceRecognitionDSM::isFeatureVectorAllowed(unsigned int label_idx)
-{
- return (_fv_cnt_per_label[label_idx] < MAX_FEATURE_VECTOR_CNT);
-}
-
-void FaceRecognitionDSM::addDataSet(std::vector<float> &feature_vec, unsigned int label_idx, unsigned int label_cnt)
-{
- _data.push_back(feature_vec);
- _label_index.push_back(label_idx);
- _fv_cnt_per_label[label_idx]++;
-
- vector<float> oneHotEncoding;
-
- for (size_t num = 0; num < label_cnt; ++num)
- oneHotEncoding.push_back(label_idx == num ? 1.0f : 0.0f);
-
- _labels.push_back(oneHotEncoding);
- _feature_vector_size = feature_vec.size();
- _label_count = label_cnt;
-}
-
-void FaceRecognitionDSM::loadDataSet(const string &file_name, unsigned int new_label_cnt)
-{
- std::ifstream inFile(file_name);
-
- if (!inFile.is_open())
- throw InvalidOperation("fail to open a file.");
-
- FeaVecHeader fvh;
-
- inFile.read((char *) &fvh, sizeof(FeaVecHeader));
- if (inFile.gcount() != sizeof(FeaVecHeader))
- throw InvalidOperation("Invalid feature vector file.");
-
- printHeader(fvh);
-
- if (!isHeaderValid(fvh))
- throw InvalidOperation("Wrong feature vector header.");
-
- /*
- * stride line format is as follows
- * ********************************
- * ____________________________
- * |feature vector|label index|
- * ----------------------------
- */
- size_t line_size_in_bytes = fvh.feature_size * sizeof(float) + sizeof(unsigned int);
-
- _feature_vector_size = fvh.feature_size;
- _label_count = fvh.label_cnt;
-
- vector<float> line_data(fvh.feature_size + 1);
-
- for (size_t idx = 0; idx < fvh.data_set_cnt; ++idx) {
- inFile.read(reinterpret_cast<char *>(line_data.data()), static_cast<streamsize>(line_size_in_bytes));
-
- vector<float> data;
-
- copy_n(line_data.begin(), _feature_vector_size, back_inserter(data));
- _data.push_back(data);
-
- unsigned int label_idx;
-
- memcpy(&label_idx, reinterpret_cast<void *>(line_data.data() + _feature_vector_size), sizeof(unsigned int));
-
- vector<float> label;
-
- // max label count may be changed so update one hot encoding table.
- for (size_t num = 0; num < new_label_cnt; ++num)
- label.push_back(label_idx == num ? 1.0f : 0.0f);
-
- _labels.push_back(label);
- _label_index.push_back(label_idx);
-
- _fv_cnt_per_label[label_idx]++;
- }
-}
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <fstream>
-#include <unistd.h>
-
-#include "machine_learning_exception.h"
-#include "face_recognition_fvm.h"
-
-using namespace std;
-using namespace mediavision::machine_learning::exception;
-
-FaceRecognitionFVM::FaceRecognitionFVM(const string feature_vector_file) : FeatureVectorManager(feature_vector_file)
-{}
-
-void FaceRecognitionFVM::updateHeader(size_t feature_size, size_t label_cnt, unsigned int data_set_cnt)
-{
- fstream headerFile {};
-
- // When trying to write header data for the first time, the feature vector file doesn't exist,
- // so create a new file with ios::out flag. Otherwise, open the feature vector file with ios::in | ios::out flags
- // to update existing header data in the feature vector file.
- if (access(_feature_vector_file.c_str(), F_OK))
- headerFile.open(_feature_vector_file.c_str(), ios::out | ios::binary);
- else
- headerFile.open(_feature_vector_file.c_str(), ios::in | ios::out | ios::binary);
-
- if (!headerFile.is_open())
- throw InvalidOperation("fail to open a file");
-
- FeaVecHeader header { FeatureVectorManager::feature_vector_signature, feature_size, label_cnt, data_set_cnt };
-
- headerFile.write((const char *) &header, sizeof(FeaVecHeader));
-}
-
-void FaceRecognitionFVM::storeData(vector<vector<float> > &features_vec, vector<unsigned int> &label_index)
-{
- fstream writeFile { _feature_vector_file, ios::binary | ios::app };
-
- if (!writeFile.is_open())
- throw InvalidOperation("fail to open a file.");
-
- for (size_t idx = 0; idx < features_vec.size(); ++idx) {
- writeFile.write(reinterpret_cast<char *>(features_vec[idx].data()),
- static_cast<streamsize>(features_vec[idx].size() * sizeof(float)));
- writeFile.write(reinterpret_cast<char *>(&label_index[idx]), static_cast<streamsize>(sizeof(unsigned int)));
- }
-}
-
-void FaceRecognitionFVM::remove()
-{
- // Remove existing file forcely.
- int ret = ::remove(_feature_vector_file.c_str());
- if (ret)
- throw InvalidOperation("Fail to remove feature vector file.");
-}
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <string.h>
-#include <map>
-#include <memory>
-#include <algorithm>
-
-#include "machine_learning_exception.h"
-#include "facenet.h"
-#include "face_recognition_type.h"
-
-using namespace std;
-using namespace mediavision::inference;
-using namespace MediaVision::Common;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-Facenet::Facenet() : _backendType(), _targetDeviceType()
-{
- _inference = make_unique<Inference>();
- _parser = make_unique<FacenetParser>();
-}
-
-static bool isJsonFile(const string &fileName)
-{
- return (!fileName.substr(fileName.find_last_of(".") + 1).compare("json"));
-}
-
-void Facenet::parseMetaFile()
-{
- _config = make_unique<EngineConfig>(string(MV_CONFIG_PATH) + string(FACE_RECOGNITION_META_FILE_NAME));
-
- int ret = _config->getIntegerAttribute(string(MV_FACENET_BACKEND_TYPE), &_backendType);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get backend engine type.");
-
- ret = _config->getIntegerAttribute(string(MV_FACENET_TARGET_DEVICE_TYPE), &_targetDeviceType);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get target device type.");
-
- string modelDefaultPath;
-
- ret = _config->getStringAttribute(MV_FACENET_MODEL_DEFAULT_PATH, &modelDefaultPath);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get model default path.");
-
- ret = _config->getStringAttribute(MV_FACENET_MODEL_FILE_NAME, &_modelFileName);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get model file name.");
-
- _modelFileName = modelDefaultPath + _modelFileName;
-
- ret = _config->getStringAttribute(MV_FACENET_MODEL_META_FILE_NAME, &_modelMetaFileName);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get model meta file name.");
-
- _modelMetaFileName = modelDefaultPath + _modelMetaFileName;
-
- ret = _config->getStringAttribute(MV_FACENET_OUTPUT_TENSOR_NAME, &_facenetOutputTensorName);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to get facenet output tensor name.");
-
- if (_modelMetaFileName.empty())
- throw InvalidOperation("Model meta file doesn't exist.");
-
- if (!isJsonFile(_modelMetaFileName))
- throw InvalidOperation("Model meta file should be json.");
-
- _parser->load(_modelMetaFileName);
-}
-
-void Facenet::configure()
-{
- int ret = _inference->bind(_backendType, _targetDeviceType);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to bind a backend engine.");
-}
-
-void Facenet::prepare()
-{
- int ret = _inference->configureInputMetaInfo(_parser->getInputMetaMap());
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to configure input tensor info from meta file.");
-
- ret = _inference->configureOutputMetaInfo(_parser->getOutputMetaMap());
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to configure output tensor info from meta file.");
-
- _inference->configureModelFiles("", _modelFileName, "");
-
- // Request to load model files to a backend engine.
- ret = _inference->load();
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to load model files.");
-}
-
-shared_ptr<MetaInfo> Facenet::getInputMetaInfo()
-{
- TensorBuffer &tensor_buffer = _inference->getInputTensorBuffer();
- IETensorBuffer &tensor_info_map = tensor_buffer.getIETensorBuffer();
-
- // TODO. consider using multiple tensors later.
- if (tensor_info_map.size() != 1)
- throw InvalidOperation("Input tensor count not invalid.");
-
- auto tensor_buffer_iter = tensor_info_map.begin();
-
- // Get the meta information corresponding to a given input tensor name.
- return _parser->getInputMetaMap()[tensor_buffer_iter->first];
-}
-
-template<typename T>
-void Facenet::preprocess(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo, vector<T> &inputVector)
-{
- LOGI("ENTER");
-
- PreprocessConfig config = { false,
- metaInfo->colorSpace,
- metaInfo->dataType,
- metaInfo->getChannel(),
- metaInfo->getWidth(),
- metaInfo->getHeight() };
-
- auto normalization = static_pointer_cast<DecodingNormal>(metaInfo->decodingTypeMap.at(DecodingType::NORMAL));
- if (normalization) {
- config.normalize = normalization->use;
- config.mean = normalization->mean;
- config.std = normalization->std;
- }
-
- auto quantization =
- static_pointer_cast<DecodingQuantization>(metaInfo->decodingTypeMap.at(DecodingType::QUANTIZATION));
- if (quantization) {
- config.quantize = quantization->use;
- config.scale = quantization->scale;
- config.zeropoint = quantization->zeropoint;
- }
-
- _preprocess.setConfig(config);
- _preprocess.run<T>(mv_src, inputVector);
-
- LOGI("LEAVE");
-}
-
-template<typename T> void Facenet::inference(vector<vector<T> > &inputVectors)
-{
- LOGI("ENTER");
-
- int ret = _inference->run<T>(inputVectors);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to run inference");
-
- LOGI("LEAVE");
-}
-
-template<typename T> void Facenet::perform(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo)
-{
- vector<T> inputVector;
-
- preprocess<T>(mv_src, metaInfo, inputVector);
-
- vector<vector<T> > inputVectors = { inputVector };
-
- inference<T>(inputVectors);
-}
-
-FacenetOutput &Facenet::result()
-{
- TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
-
- // Make sure to clear _result.outputs vectors because if not clear then other output_vector will be pushed to _result.outputs
- // and it results in sending wrong output vector to face recognition framework.
- _result.outputs.clear();
-
- _outputTensorBuffer = tensor_buffer_obj.getTensorBuffer(_facenetOutputTensorName);
- if (!_outputTensorBuffer)
- throw InvalidOperation("No output tensor.");
-
- float *buffer = reinterpret_cast<float *>(_outputTensorBuffer->buffer);
-
- _result.outputs.push_back(vector<float>(buffer, buffer + _outputTensorBuffer->size / sizeof(float)));
-
- return _result;
-}
-
-template void Facenet::preprocess<float>(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo,
- vector<float> &inputVector);
-template void Facenet::inference<float>(vector<vector<float> > &inputVectors);
-template void Facenet::perform<float>(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo);
-
-template void Facenet::preprocess<unsigned char>(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo,
- vector<unsigned char> &inputVector);
-template void Facenet::inference<unsigned char>(vector<vector<unsigned char> > &inputVectors);
-template void Facenet::perform<unsigned char>(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo);
-
-}
-}
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "machine_learning_exception.h"
-#include "facenet_adapter.h"
-
-using namespace std;
-using namespace MediaVision::Common;
-using namespace mediavision::machine_learning;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-FacenetAdapter::FacenetAdapter()
-{
- _facenet = make_unique<Facenet>();
-}
-
-FacenetAdapter::~FacenetAdapter()
-{}
-
-void FacenetAdapter::setModelInfo(const string &model_file, const string &meta_file, const string &label_file,
- const string &model_name)
-{}
-
-void FacenetAdapter::setEngineInfo(const string &engine_type, const string &device_type)
-{}
-
-void FacenetAdapter::configure()
-{
- _facenet->parseMetaFile();
- _facenet->configure();
-}
-
-unsigned int FacenetAdapter::getNumberOfEngines()
-{
- throw InvalidOperation("Not yet implemented");
-}
-
-const string &FacenetAdapter::getEngineType(unsigned int engine_index)
-{
- throw InvalidOperation("Not yet implemented");
-}
-
-unsigned int FacenetAdapter::getNumberOfDevices(const string &engine_type)
-{
- throw InvalidOperation("Not yet implemented");
-}
-
-const string &FacenetAdapter::getDeviceType(const string &engine_type, unsigned int device_index)
-{
- throw InvalidOperation("Not yet implemented");
-}
-
-void FacenetAdapter::prepare()
-{
- _facenet->prepare();
-}
-
-void FacenetAdapter::perform(InputBaseType &input)
-{
- auto &source = static_cast<FacenetInput &>(input);
- shared_ptr<MetaInfo> metaInfo = _facenet->getInputMetaInfo();
- if (metaInfo->dataType == MV_INFERENCE_DATA_UINT8)
- _facenet->perform<unsigned char>(source.inference_src, metaInfo);
- else if (metaInfo->dataType == MV_INFERENCE_DATA_FLOAT32)
- _facenet->perform<float>(source.inference_src, metaInfo);
- else
- throw InvalidOperation("Invalid model data type.");
-}
-
-void FacenetAdapter::performAsync(InputBaseType &input)
-{
- throw InvalidOperation("Not support yet.");
-}
-
-OutputBaseType &FacenetAdapter::getOutput()
-{
- return _facenet->result();
-}
-
-OutputBaseType &FacenetAdapter::getOutputCache()
-{
- throw InvalidOperation("Not support yet.");
-}
-
-}
-}
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "machine_learning_exception.h"
-#include "facenet_parser.h"
-
-using namespace std;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-FacenetParser::FacenetParser(int task_type)
-{
- LOGI("ENTER");
- LOGI("LEAVE");
-}
-
-FacenetParser::~FacenetParser()
-{}
-
-void FacenetParser::parsePostprocess(shared_ptr<MetaInfo> meta_info, JsonObject *in_obj)
-{
- LOGI("ENTER");
-
- LOGI("tensor name : %s", meta_info->name.c_str());
-
- if (json_object_has_member(in_obj, "box"))
- _postprocessParser.parseBox(meta_info, in_obj);
-
- LOGI("LEAVE");
-}
-
-}
-}
\ No newline at end of file
#include "mv_private.h"
#include "mv_feature_key.h"
-#include "context.h"
-#include "machine_learning_exception.h"
-#include "face_recognition_adapter.h"
-#include "facenet_adapter.h"
-#include "MachineLearningNative.h"
+#include "Context.h"
+#include "MvMlException.h"
+#include "FaceRecognitionAdapter.h"
+#include "FacenetAdapter.h"
+#include "native_capi.h"
#include "mv_face_recognition.h"
#include "mv_face_recognition_internal.h"
#include <sys/stat.h>
-#include "machine_learning_exception.h"
-#include "simple_shot.h"
-#include "data_set_manager.h"
-#include "feature_vector_manager.h"
+#include "MvMlException.h"
+#include "SimpleShot.h"
+#include "DataSetManager.h"
+#include "FeatureVectorManager.h"
#include "file_util.h"
using namespace std;
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __IIMAGE_CLASSIFICATION_H__
+#define __IIMAGE_CLASSIFICATION_H__
+
+#include <mv_common.h>
+
+#include "image_classification_type.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+class IImageClassification
+{
+public:
+ virtual ~IImageClassification() {};
+
+ virtual void preDestroy() = 0;
+ virtual void setEngineInfo(std::string engine_type, std::string device_type) = 0;
+ virtual unsigned int getNumberOfEngines() = 0;
+ virtual const std::string &getEngineType(unsigned int engine_index) = 0;
+ virtual unsigned int getNumberOfDevices(const std::string &engine_type) = 0;
+ virtual const std::string &getDeviceType(const std::string &engine_type, const unsigned int device_index) = 0;
+ virtual void configure() = 0;
+ virtual void prepare() = 0;
+ virtual void perform(mv_source_h &mv_src) = 0;
+ virtual void performAsync(ImageClassificationInput &input) = 0;
+ virtual ImageClassificationResult &getOutput() = 0;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __IMAGE_CLASSIFICATION_H__
+#define __IMAGE_CLASSIFICATION_H__
+
+#include <mv_common.h>
+#include <mv_inference_type.h>
+#include "mv_private.h"
+
+#include "EngineConfig.h"
+#include "inference_engine_common_impl.h"
+#include "Inference.h"
+#include "image_classification_type.h"
+#include "MetaParser.h"
+#include "MvMlConfig.h"
+#include "ImageClassificationParser.h"
+#include "MvMlPreprocess.h"
+#include "AsyncManager.h"
+#include "IImageClassification.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T> class ImageClassification : public IImageClassification
+{
+private:
+ std::unique_ptr<AsyncManager<T, ImageClassificationResult> > _async_manager;
+ ImageClassificationResult _current_result;
+
+ void loadLabel();
+ void getEngineList();
+ void getDeviceList(const std::string &engine_type);
+ void preprocess(mv_source_h &mv_src, std::shared_ptr<MetaInfo> metaInfo, std::vector<T> &inputVector);
+ std::shared_ptr<MetaInfo> getInputMetaInfo();
+
+protected:
+ std::unique_ptr<mediavision::inference::Inference> _inference;
+ std::shared_ptr<Config> _config;
+ Preprocess _preprocess;
+ std::vector<std::string> _labels;
+ std::vector<std::string> _valid_backends;
+ std::vector<std::string> _valid_devices;
+
+ void getOutputNames(std::vector<std::string> &names);
+ void getOutpuTensor(std::string &target_name, std::vector<float> &tensor);
+ void inference(std::vector<std::vector<T> > &inputVectors);
+ virtual ImageClassificationResult &result() = 0;
+
+public:
+ explicit ImageClassification(std::shared_ptr<Config> config);
+ virtual ~ImageClassification() = default;
+
+ void preDestroy();
+ void setEngineInfo(std::string engine_type_name, std::string device_type_name);
+ unsigned int getNumberOfEngines();
+ const std::string &getEngineType(unsigned int engine_index);
+ unsigned int getNumberOfDevices(const std::string &engine_type);
+ const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index);
+ void configure();
+ void prepare();
+ void perform(mv_source_h &mv_src);
+ void performAsync(ImageClassificationInput &input);
+ ImageClassificationResult &getOutput();
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __IMAGE_CLASSIFICATION_ADAPTER_H__
+#define __IMAGE_CLASSIFICATION_ADAPTER_H__
+
+#include <dlog.h>
+
+#include "EngineConfig.h"
+#include "ITask.h"
+#include "MvMlConfig.h"
+#include "ImageClassificationDefault.h"
+#include "IImageClassification.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+class ImageClassificationAdapter : public mediavision::common::ITask
+{
+private:
+ std::unique_ptr<IImageClassification> _image_classification;
+ std::shared_ptr<Config> _config;
+ const std::string _config_file_name = "image_classification.json";
+
+ void create();
+
+public:
+ ImageClassificationAdapter();
+ ~ImageClassificationAdapter();
+
+ void setModelInfo(const std::string &model_file, const std::string &meta_file, const std::string &label_file,
+ const std::string &model_name) override;
+ void setEngineInfo(const std::string &engine_type, const std::string &device_type) override;
+ void configure() override;
+ unsigned int getNumberOfEngines() override;
+ const std::string &getEngineType(unsigned int engine_index) override;
+ unsigned int getNumberOfDevices(const std::string &engine_type) override;
+ const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
+ void prepare() override;
+ void perform(InputBaseType &input) override;
+ void performAsync(InputBaseType &input) override;
+ OutputBaseType &getOutput() override;
+ OutputBaseType &getOutputCache() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __IMAGE_CLASSIFICATION_FLOAT_H__
+#define __IMAGE_CLASSIFICATION_FLOAT_H__
+
+#include <memory>
+#include <mv_common.h>
+#include "mv_private.h"
+
+#include "ImageClassification.h"
+#include <mv_inference_type.h>
+#include "EngineConfig.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T> class ImageClassificationDefault : public ImageClassification<T>
+{
+ using ImageClassification<T>::_config;
+ using ImageClassification<T>::_labels;
+
+private:
+ ImageClassificationResult _result;
+
+public:
+ ImageClassificationDefault(std::shared_ptr<Config> config);
+ ~ImageClassificationDefault();
+
+ ImageClassificationResult &result() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __IIMAGE_CLASSIFICATION_H__
-#define __IIMAGE_CLASSIFICATION_H__
-
-#include <mv_common.h>
-
-#include "image_classification_type.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-class IImageClassification
-{
-public:
- virtual ~IImageClassification() {};
-
- virtual void preDestroy() = 0;
- virtual void setEngineInfo(std::string engine_type, std::string device_type) = 0;
- virtual unsigned int getNumberOfEngines() = 0;
- virtual const std::string &getEngineType(unsigned int engine_index) = 0;
- virtual unsigned int getNumberOfDevices(const std::string &engine_type) = 0;
- virtual const std::string &getDeviceType(const std::string &engine_type, const unsigned int device_index) = 0;
- virtual void configure() = 0;
- virtual void prepare() = 0;
- virtual void perform(mv_source_h &mv_src) = 0;
- virtual void performAsync(ImageClassificationInput &input) = 0;
- virtual ImageClassificationResult &getOutput() = 0;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __IMAGE_CLASSIFICATION_H__
-#define __IMAGE_CLASSIFICATION_H__
-
-#include <mv_common.h>
-#include <mv_inference_type.h>
-#include "mv_private.h"
-
-#include "EngineConfig.h"
-#include "inference_engine_common_impl.h"
-#include "Inference.h"
-#include "image_classification_type.h"
-#include "MetaParser.h"
-#include "machine_learning_config.h"
-#include "ImageClassificationParser.h"
-#include "machine_learning_preprocess.h"
-#include "async_manager.h"
-#include "iimage_classification.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T> class ImageClassification : public IImageClassification
-{
-private:
- std::unique_ptr<AsyncManager<T, ImageClassificationResult> > _async_manager;
- ImageClassificationResult _current_result;
-
- void loadLabel();
- void getEngineList();
- void getDeviceList(const std::string &engine_type);
- void preprocess(mv_source_h &mv_src, std::shared_ptr<MetaInfo> metaInfo, std::vector<T> &inputVector);
- std::shared_ptr<MetaInfo> getInputMetaInfo();
-
-protected:
- std::unique_ptr<mediavision::inference::Inference> _inference;
- std::shared_ptr<MachineLearningConfig> _config;
- Preprocess _preprocess;
- std::vector<std::string> _labels;
- std::vector<std::string> _valid_backends;
- std::vector<std::string> _valid_devices;
-
- void getOutputNames(std::vector<std::string> &names);
- void getOutpuTensor(std::string &target_name, std::vector<float> &tensor);
- void inference(std::vector<std::vector<T> > &inputVectors);
- virtual ImageClassificationResult &result() = 0;
-
-public:
- explicit ImageClassification(std::shared_ptr<MachineLearningConfig> config);
- virtual ~ImageClassification() = default;
-
- void preDestroy();
- void setEngineInfo(std::string engine_type_name, std::string device_type_name);
- unsigned int getNumberOfEngines();
- const std::string &getEngineType(unsigned int engine_index);
- unsigned int getNumberOfDevices(const std::string &engine_type);
- const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index);
- void configure();
- void prepare();
- void perform(mv_source_h &mv_src);
- void performAsync(ImageClassificationInput &input);
- ImageClassificationResult &getOutput();
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __IMAGE_CLASSIFICATION_ADAPTER_H__
-#define __IMAGE_CLASSIFICATION_ADAPTER_H__
-
-#include <dlog.h>
-
-#include "EngineConfig.h"
-#include "itask.h"
-#include "machine_learning_config.h"
-#include "image_classification_default.h"
-#include "iimage_classification.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-class ImageClassificationAdapter : public mediavision::common::ITask
-{
-private:
- std::unique_ptr<IImageClassification> _image_classification;
- std::shared_ptr<MachineLearningConfig> _config;
- const std::string _config_file_name = "image_classification.json";
-
- void create();
-
-public:
- ImageClassificationAdapter();
- ~ImageClassificationAdapter();
-
- void setModelInfo(const std::string &model_file, const std::string &meta_file, const std::string &label_file,
- const std::string &model_name) override;
- void setEngineInfo(const std::string &engine_type, const std::string &device_type) override;
- void configure() override;
- unsigned int getNumberOfEngines() override;
- const std::string &getEngineType(unsigned int engine_index) override;
- unsigned int getNumberOfDevices(const std::string &engine_type) override;
- const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
- void prepare() override;
- void perform(InputBaseType &input) override;
- void performAsync(InputBaseType &input) override;
- OutputBaseType &getOutput() override;
- OutputBaseType &getOutputCache() override;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __IMAGE_CLASSIFICATION_FLOAT_H__
-#define __IMAGE_CLASSIFICATION_FLOAT_H__
-
-#include <memory>
-#include <mv_common.h>
-#include "mv_private.h"
-
-#include "image_classification.h"
-#include <mv_inference_type.h>
-#include "EngineConfig.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T> class ImageClassificationDefault : public ImageClassification<T>
-{
- using ImageClassification<T>::_config;
- using ImageClassification<T>::_labels;
-
-private:
- ImageClassificationResult _result;
-
-public:
- ImageClassificationDefault(std::shared_ptr<MachineLearningConfig> config);
- ~ImageClassificationDefault();
-
- ImageClassificationResult &result() override;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
#include <mv_common.h>
#include <mv_inference_type.h>
-#include "MachineLearningType.h"
+#include "mv_ml_types.h"
namespace mediavision
{
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <fstream>
+#include <map>
+#include <memory>
+#include <algorithm>
+
+#include "MvMlException.h"
+#include "common.h"
+#include "mv_image_classification_config.h"
+#include "ImageClassification.h"
+
+using namespace std;
+using namespace mediavision::inference;
+using namespace MediaVision::Common;
+using namespace mediavision::common;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T> ImageClassification<T>::ImageClassification(std::shared_ptr<Config> config) : _config(config)
+{
+ _inference = make_unique<Inference>();
+}
+
+template<typename T> void ImageClassification<T>::preDestroy()
+{
+ if (!_async_manager)
+ return;
+
+ _async_manager->stop();
+}
+
+template<typename T> void ImageClassification<T>::configure()
+{
+ loadLabel();
+
+ int ret = _inference->bind(_config->getBackendType(), _config->getTargetDeviceType());
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to bind a backend engine.");
+}
+
+template<typename T> void ImageClassification<T>::loadLabel()
+{
+ if (_config->getLabelFilePath().empty())
+ return;
+
+ ifstream readFile;
+
+ _labels.clear();
+ readFile.open(_config->getLabelFilePath().c_str());
+
+ if (readFile.fail())
+ throw InvalidOperation("Fail to open " + _config->getLabelFilePath() + " file.");
+
+ string line;
+
+ while (getline(readFile, line))
+ _labels.push_back(line);
+
+ readFile.close();
+}
+
+template<typename T> void ImageClassification<T>::getEngineList()
+{
+ for (auto idx = MV_INFERENCE_BACKEND_NONE + 1; idx < MV_INFERENCE_BACKEND_MAX; ++idx) {
+ auto backend = _inference->getSupportedInferenceBackend(idx);
+ // TODO. we need to describe what inference engines are supported by each Task API,
+ // and based on it, below inference engine types should be checked
+ // if a given type is supported by this Task API later. As of now, tflite only.
+ if (backend.second == true && backend.first.compare("tflite") == 0)
+ _valid_backends.push_back(backend.first);
+ }
+}
+
+template<typename T> void ImageClassification<T>::getDeviceList(const string &engine_type)
+{
+ // TODO. add device types available for a given engine type later.
+ // In default, cpu and gpu only.
+ _valid_devices.push_back("cpu");
+ _valid_devices.push_back("gpu");
+}
+
+template<typename T>
+void ImageClassification<T>::setEngineInfo(std::string engine_type_name, std::string device_type_name)
+{
+ if (engine_type_name.empty() || device_type_name.empty())
+ throw InvalidParameter("Invalid engine info.");
+
+ transform(engine_type_name.begin(), engine_type_name.end(), engine_type_name.begin(), ::toupper);
+ transform(device_type_name.begin(), device_type_name.end(), device_type_name.begin(), ::toupper);
+
+ int engine_type = GetBackendType(engine_type_name);
+ int device_type = GetDeviceType(device_type_name);
+
+ if (engine_type == MEDIA_VISION_ERROR_INVALID_PARAMETER || device_type == MEDIA_VISION_ERROR_INVALID_PARAMETER)
+ throw InvalidParameter("backend or target device type not found.");
+
+ _config->setBackendType(engine_type);
+ _config->setTargetDeviceType(device_type);
+
+ LOGI("Engine type : %s => %d, Device type : %s => %d", engine_type_name.c_str(), engine_type,
+ device_type_name.c_str(), device_type);
+}
+
+template<typename T> unsigned int ImageClassification<T>::getNumberOfEngines()
+{
+ if (!_valid_backends.empty()) {
+ return _valid_backends.size();
+ }
+
+ getEngineList();
+ return _valid_backends.size();
+}
+
+template<typename T> const string &ImageClassification<T>::getEngineType(unsigned int engine_index)
+{
+ if (!_valid_backends.empty()) {
+ if (_valid_backends.size() <= engine_index)
+ throw InvalidParameter("Invalid engine index.");
+
+ return _valid_backends[engine_index];
+ }
+
+ getEngineList();
+
+ if (_valid_backends.size() <= engine_index)
+ throw InvalidParameter("Invalid engine index.");
+
+ return _valid_backends[engine_index];
+}
+
+template<typename T> unsigned int ImageClassification<T>::getNumberOfDevices(const string &engine_type)
+{
+ if (!_valid_devices.empty()) {
+ return _valid_devices.size();
+ }
+
+ getDeviceList(engine_type);
+ return _valid_devices.size();
+}
+
+template<typename T>
+const string &ImageClassification<T>::getDeviceType(const string &engine_type, unsigned int device_index)
+{
+ if (!_valid_devices.empty()) {
+ if (_valid_devices.size() <= device_index)
+ throw InvalidParameter("Invalid device index.");
+
+ return _valid_devices[device_index];
+ }
+
+ getDeviceList(engine_type);
+
+ if (_valid_devices.size() <= device_index)
+ throw InvalidParameter("Invalid device index.");
+
+ return _valid_devices[device_index];
+}
+
+template<typename T> void ImageClassification<T>::prepare()
+{
+ int ret = _inference->configureInputMetaInfo(_config->getInputMetaMap());
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to configure input tensor info from meta file.");
+
+ ret = _inference->configureOutputMetaInfo(_config->getOutputMetaMap());
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to configure output tensor info from meta file.");
+
+ _inference->configureModelFiles("", _config->getModelFilePath(), "");
+
+ // Request to load model files to a backend engine.
+ ret = _inference->load();
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to load model files.");
+}
+
+template<typename T> shared_ptr<MetaInfo> ImageClassification<T>::getInputMetaInfo()
+{
+ TensorBuffer &tensor_buffer = _inference->getInputTensorBuffer();
+ IETensorBuffer &tensor_info_map = tensor_buffer.getIETensorBuffer();
+
+ // TODO. consider using multiple tensors later.
+ if (tensor_info_map.size() != 1)
+ throw InvalidOperation("Input tensor count not invalid.");
+
+ auto tensor_buffer_iter = tensor_info_map.begin();
+
+ // Get the meta information corresponding to a given input tensor name.
+ return _config->getInputMetaMap()[tensor_buffer_iter->first];
+}
+
+template<typename T>
+void ImageClassification<T>::preprocess(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo, vector<T> &inputVector)
+{
+ LOGI("ENTER");
+
+ PreprocessConfig config = { false,
+ metaInfo->colorSpace,
+ metaInfo->dataType,
+ metaInfo->getChannel(),
+ metaInfo->getWidth(),
+ metaInfo->getHeight() };
+
+ auto normalization = static_pointer_cast<DecodingNormal>(metaInfo->decodingTypeMap.at(DecodingType::NORMAL));
+ if (normalization) {
+ config.normalize = normalization->use;
+ config.mean = normalization->mean;
+ config.std = normalization->std;
+ }
+
+ auto quantization =
+ static_pointer_cast<DecodingQuantization>(metaInfo->decodingTypeMap.at(DecodingType::QUANTIZATION));
+ if (quantization) {
+ config.quantize = quantization->use;
+ config.scale = quantization->scale;
+ config.zeropoint = quantization->zeropoint;
+ }
+
+ _preprocess.setConfig(config);
+ _preprocess.run<T>(mv_src, inputVector);
+
+ LOGI("LEAVE");
+}
+
+template<typename T> void ImageClassification<T>::inference(vector<vector<T> > &inputVectors)
+{
+ LOGI("ENTER");
+
+ int ret = _inference->run<T>(inputVectors);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to run inference");
+
+ LOGI("LEAVE");
+}
+
+template<typename T> void ImageClassification<T>::perform(mv_source_h &mv_src)
+{
+ shared_ptr<MetaInfo> metaInfo = getInputMetaInfo();
+ vector<T> inputVector;
+
+ preprocess(mv_src, metaInfo, inputVector);
+
+ vector<vector<T> > inputVectors = { inputVector };
+ inference(inputVectors);
+}
+
+template<typename T> ImageClassificationResult &ImageClassification<T>::getOutput()
+{
+ if (_async_manager) {
+ if (!_async_manager->isWorking())
+ throw InvalidOperation("Image classification has been already destroyed so invalid operation.");
+
+ _current_result = _async_manager->pop();
+ } else {
+ // TODO. Check if inference request is completed or not here.
+ // If not then throw an exception.
+ _current_result = result();
+ }
+
+ return _current_result;
+}
+
+template<typename T> void ImageClassification<T>::performAsync(ImageClassificationInput &input)
+{
+ if (!_async_manager) {
+ _async_manager = make_unique<AsyncManager<T, ImageClassificationResult> >([this]() {
+ AsyncInputQueue<T> inputQueue = _async_manager->popFromInput();
+
+ inference(inputQueue.inputs);
+
+ ImageClassificationResult &resultQueue = result();
+
+ resultQueue.frame_number = inputQueue.frame_number;
+ _async_manager->pushToOutput(resultQueue);
+ });
+ }
+
+ shared_ptr<MetaInfo> metaInfo = getInputMetaInfo();
+ vector<T> inputVector;
+
+ preprocess(input.inference_src, metaInfo, inputVector);
+
+ vector<vector<T> > inputVectors = { inputVector };
+ _async_manager->push(inputVectors);
+}
+
+template<typename T> void ImageClassification<T>::getOutputNames(vector<string> &names)
+{
+ TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
+ IETensorBuffer &ie_tensor_buffer = tensor_buffer_obj.getIETensorBuffer();
+
+ for (IETensorBuffer::iterator it = ie_tensor_buffer.begin(); it != ie_tensor_buffer.end(); it++)
+ names.push_back(it->first);
+}
+
+template<typename T> void ImageClassification<T>::getOutpuTensor(string &target_name, vector<float> &tensor)
+{
+ LOGI("ENTER");
+
+ TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
+
+ inference_engine_tensor_buffer *tensor_buffer = tensor_buffer_obj.getTensorBuffer(target_name);
+ if (!tensor_buffer)
+ throw InvalidOperation("Fail to get tensor buffer.");
+
+ auto raw_buffer = static_cast<float *>(tensor_buffer->buffer);
+
+ copy(&raw_buffer[0], &raw_buffer[tensor_buffer->size / sizeof(float)], back_inserter(tensor));
+
+ LOGI("LEAVE");
+}
+
+template class ImageClassification<unsigned char>;
+template class ImageClassification<float>;
+
+}
+}
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string>
+
+#include "MvMlException.h"
+#include "mv_image_classification_config.h"
+#include "ImageClassificationAdapter.h"
+
+using namespace std;
+using namespace MediaVision::Common;
+using namespace mediavision::machine_learning;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+ImageClassificationAdapter::ImageClassificationAdapter()
+{
+ _config = make_shared<Config>();
+ _config->parseConfigFile(_config_file_name);
+
+ create();
+}
+
+ImageClassificationAdapter::~ImageClassificationAdapter()
+{
+ _image_classification->preDestroy();
+}
+
+void ImageClassificationAdapter::create()
+{
+ _config->loadMetaFile(make_unique<ImageClassificationParser>());
+ mv_inference_data_type_e dataType = _config->getInputMetaMap().begin()->second->dataType;
+
+ switch (dataType) {
+ case MV_INFERENCE_DATA_UINT8:
+ _image_classification = make_unique<ImageClassificationDefault<unsigned char> >(_config);
+ break;
+ case MV_INFERENCE_DATA_FLOAT32:
+ _image_classification = make_unique<ImageClassificationDefault<float> >(_config);
+ break;
+ default:
+ throw InvalidOperation("Invalid image classification data type.");
+ }
+}
+
+void ImageClassificationAdapter::setModelInfo(const string &model_file, const string &meta_file,
+ const string &label_file, const string &model_name)
+{
+ _config->setUserModel(model_file, meta_file, label_file);
+ create();
+
+ if (model_file.empty() && meta_file.empty()) {
+ LOGW("Given model info is invalid so default model info will be used instead.");
+ return;
+ }
+}
+
+void ImageClassificationAdapter::setEngineInfo(const string &engine_type, const string &device_type)
+{
+ _image_classification->setEngineInfo(engine_type, device_type);
+}
+
+void ImageClassificationAdapter::configure()
+{
+ _image_classification->configure();
+}
+
+unsigned int ImageClassificationAdapter::getNumberOfEngines()
+{
+ return _image_classification->getNumberOfEngines();
+}
+
+const string &ImageClassificationAdapter::getEngineType(unsigned int engine_index)
+{
+ return _image_classification->getEngineType(engine_index);
+}
+
+unsigned int ImageClassificationAdapter::getNumberOfDevices(const string &engine_type)
+{
+ return _image_classification->getNumberOfDevices(engine_type);
+}
+
+const string &ImageClassificationAdapter::getDeviceType(const string &engine_type, unsigned int device_index)
+{
+ return _image_classification->getDeviceType(engine_type, device_index);
+}
+
+void ImageClassificationAdapter::prepare()
+{
+ _image_classification->prepare();
+}
+
+void ImageClassificationAdapter::perform(InputBaseType &input)
+{
+ _image_classification->perform(input.inference_src);
+}
+
+void ImageClassificationAdapter::performAsync(InputBaseType &input)
+{
+ _image_classification->performAsync(static_cast<ImageClassificationInput &>(input));
+}
+
+OutputBaseType &ImageClassificationAdapter::getOutput()
+{
+ return _image_classification->getOutput();
+}
+
+OutputBaseType &ImageClassificationAdapter::getOutputCache()
+{
+ throw InvalidOperation("Not support yet.");
+}
+
+}
+}
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <map>
+#include <algorithm>
+
+#include "MvMlException.h"
+#include "ImageClassificationDefault.h"
+#include "Postprocess.h"
+
+using namespace std;
+using namespace mediavision::inference;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T>
+ImageClassificationDefault<T>::ImageClassificationDefault(shared_ptr<Config> config)
+ : ImageClassification<T>(config), _result()
+{}
+
+template<typename T> ImageClassificationDefault<T>::~ImageClassificationDefault()
+{}
+
+template<typename T> ImageClassificationResult &ImageClassificationDefault<T>::result()
+{
+ vector<string> names;
+
+ ImageClassification<T>::getOutputNames(names);
+
+ vector<float> output_vec;
+
+ // In case of image classification model, only one output tensor is used.
+ ImageClassification<T>::getOutpuTensor(names[0], output_vec);
+
+ auto metaInfo = _config->getOutputMetaMap().at(names[0]);
+ auto decodingScore = static_pointer_cast<DecodingScore>(metaInfo->decodingTypeMap.at(DecodingType::SCORE));
+
+ if (decodingScore->type == ScoreType::SIGMOID) {
+ for (size_t idx = 0; idx < output_vec.size(); ++idx)
+ output_vec[idx] = PostProcess::sigmoid(output_vec[idx]);
+ }
+
+ _result.label = _labels[max_element(output_vec.begin(), output_vec.end()) - output_vec.begin()];
+ LOGI("Label = %s", _result.label.c_str());
+
+ return _result;
+}
+
+template class ImageClassificationDefault<unsigned char>;
+template class ImageClassificationDefault<float>;
+
+}
+}
* limitations under the License.
*/
-#include "machine_learning_exception.h"
+#include "MvMlException.h"
#include "ImageClassificationParser.h"
using namespace std;
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <string.h>
-#include <fstream>
-#include <map>
-#include <memory>
-#include <algorithm>
-
-#include "machine_learning_exception.h"
-#include "mv_machine_learning_common.h"
-#include "mv_image_classification_config.h"
-#include "image_classification.h"
-
-using namespace std;
-using namespace mediavision::inference;
-using namespace MediaVision::Common;
-using namespace mediavision::common;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T>
-ImageClassification<T>::ImageClassification(std::shared_ptr<MachineLearningConfig> config) : _config(config)
-{
- _inference = make_unique<Inference>();
-}
-
-template<typename T> void ImageClassification<T>::preDestroy()
-{
- if (!_async_manager)
- return;
-
- _async_manager->stop();
-}
-
-template<typename T> void ImageClassification<T>::configure()
-{
- loadLabel();
-
- int ret = _inference->bind(_config->getBackendType(), _config->getTargetDeviceType());
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to bind a backend engine.");
-}
-
-template<typename T> void ImageClassification<T>::loadLabel()
-{
- if (_config->getLabelFilePath().empty())
- return;
-
- ifstream readFile;
-
- _labels.clear();
- readFile.open(_config->getLabelFilePath().c_str());
-
- if (readFile.fail())
- throw InvalidOperation("Fail to open " + _config->getLabelFilePath() + " file.");
-
- string line;
-
- while (getline(readFile, line))
- _labels.push_back(line);
-
- readFile.close();
-}
-
-template<typename T> void ImageClassification<T>::getEngineList()
-{
- for (auto idx = MV_INFERENCE_BACKEND_NONE + 1; idx < MV_INFERENCE_BACKEND_MAX; ++idx) {
- auto backend = _inference->getSupportedInferenceBackend(idx);
- // TODO. we need to describe what inference engines are supported by each Task API,
- // and based on it, below inference engine types should be checked
- // if a given type is supported by this Task API later. As of now, tflite only.
- if (backend.second == true && backend.first.compare("tflite") == 0)
- _valid_backends.push_back(backend.first);
- }
-}
-
-template<typename T> void ImageClassification<T>::getDeviceList(const string &engine_type)
-{
- // TODO. add device types available for a given engine type later.
- // In default, cpu and gpu only.
- _valid_devices.push_back("cpu");
- _valid_devices.push_back("gpu");
-}
-
-template<typename T>
-void ImageClassification<T>::setEngineInfo(std::string engine_type_name, std::string device_type_name)
-{
- if (engine_type_name.empty() || device_type_name.empty())
- throw InvalidParameter("Invalid engine info.");
-
- transform(engine_type_name.begin(), engine_type_name.end(), engine_type_name.begin(), ::toupper);
- transform(device_type_name.begin(), device_type_name.end(), device_type_name.begin(), ::toupper);
-
- int engine_type = GetBackendType(engine_type_name);
- int device_type = GetDeviceType(device_type_name);
-
- if (engine_type == MEDIA_VISION_ERROR_INVALID_PARAMETER || device_type == MEDIA_VISION_ERROR_INVALID_PARAMETER)
- throw InvalidParameter("backend or target device type not found.");
-
- _config->setBackendType(engine_type);
- _config->setTargetDeviceType(device_type);
-
- LOGI("Engine type : %s => %d, Device type : %s => %d", engine_type_name.c_str(), engine_type,
- device_type_name.c_str(), device_type);
-}
-
-template<typename T> unsigned int ImageClassification<T>::getNumberOfEngines()
-{
- if (!_valid_backends.empty()) {
- return _valid_backends.size();
- }
-
- getEngineList();
- return _valid_backends.size();
-}
-
-template<typename T> const string &ImageClassification<T>::getEngineType(unsigned int engine_index)
-{
- if (!_valid_backends.empty()) {
- if (_valid_backends.size() <= engine_index)
- throw InvalidParameter("Invalid engine index.");
-
- return _valid_backends[engine_index];
- }
-
- getEngineList();
-
- if (_valid_backends.size() <= engine_index)
- throw InvalidParameter("Invalid engine index.");
-
- return _valid_backends[engine_index];
-}
-
-template<typename T> unsigned int ImageClassification<T>::getNumberOfDevices(const string &engine_type)
-{
- if (!_valid_devices.empty()) {
- return _valid_devices.size();
- }
-
- getDeviceList(engine_type);
- return _valid_devices.size();
-}
-
-template<typename T>
-const string &ImageClassification<T>::getDeviceType(const string &engine_type, unsigned int device_index)
-{
- if (!_valid_devices.empty()) {
- if (_valid_devices.size() <= device_index)
- throw InvalidParameter("Invalid device index.");
-
- return _valid_devices[device_index];
- }
-
- getDeviceList(engine_type);
-
- if (_valid_devices.size() <= device_index)
- throw InvalidParameter("Invalid device index.");
-
- return _valid_devices[device_index];
-}
-
-template<typename T> void ImageClassification<T>::prepare()
-{
- int ret = _inference->configureInputMetaInfo(_config->getInputMetaMap());
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to configure input tensor info from meta file.");
-
- ret = _inference->configureOutputMetaInfo(_config->getOutputMetaMap());
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to configure output tensor info from meta file.");
-
- _inference->configureModelFiles("", _config->getModelFilePath(), "");
-
- // Request to load model files to a backend engine.
- ret = _inference->load();
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to load model files.");
-}
-
-template<typename T> shared_ptr<MetaInfo> ImageClassification<T>::getInputMetaInfo()
-{
- TensorBuffer &tensor_buffer = _inference->getInputTensorBuffer();
- IETensorBuffer &tensor_info_map = tensor_buffer.getIETensorBuffer();
-
- // TODO. consider using multiple tensors later.
- if (tensor_info_map.size() != 1)
- throw InvalidOperation("Input tensor count not invalid.");
-
- auto tensor_buffer_iter = tensor_info_map.begin();
-
- // Get the meta information corresponding to a given input tensor name.
- return _config->getInputMetaMap()[tensor_buffer_iter->first];
-}
-
-template<typename T>
-void ImageClassification<T>::preprocess(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo, vector<T> &inputVector)
-{
- LOGI("ENTER");
-
- PreprocessConfig config = { false,
- metaInfo->colorSpace,
- metaInfo->dataType,
- metaInfo->getChannel(),
- metaInfo->getWidth(),
- metaInfo->getHeight() };
-
- auto normalization = static_pointer_cast<DecodingNormal>(metaInfo->decodingTypeMap.at(DecodingType::NORMAL));
- if (normalization) {
- config.normalize = normalization->use;
- config.mean = normalization->mean;
- config.std = normalization->std;
- }
-
- auto quantization =
- static_pointer_cast<DecodingQuantization>(metaInfo->decodingTypeMap.at(DecodingType::QUANTIZATION));
- if (quantization) {
- config.quantize = quantization->use;
- config.scale = quantization->scale;
- config.zeropoint = quantization->zeropoint;
- }
-
- _preprocess.setConfig(config);
- _preprocess.run<T>(mv_src, inputVector);
-
- LOGI("LEAVE");
-}
-
-template<typename T> void ImageClassification<T>::inference(vector<vector<T> > &inputVectors)
-{
- LOGI("ENTER");
-
- int ret = _inference->run<T>(inputVectors);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to run inference");
-
- LOGI("LEAVE");
-}
-
-template<typename T> void ImageClassification<T>::perform(mv_source_h &mv_src)
-{
- shared_ptr<MetaInfo> metaInfo = getInputMetaInfo();
- vector<T> inputVector;
-
- preprocess(mv_src, metaInfo, inputVector);
-
- vector<vector<T> > inputVectors = { inputVector };
- inference(inputVectors);
-}
-
-template<typename T> ImageClassificationResult &ImageClassification<T>::getOutput()
-{
- if (_async_manager) {
- if (!_async_manager->isWorking())
- throw InvalidOperation("Image classification has been already destroyed so invalid operation.");
-
- _current_result = _async_manager->pop();
- } else {
- // TODO. Check if inference request is completed or not here.
- // If not then throw an exception.
- _current_result = result();
- }
-
- return _current_result;
-}
-
-template<typename T> void ImageClassification<T>::performAsync(ImageClassificationInput &input)
-{
- if (!_async_manager) {
- _async_manager = make_unique<AsyncManager<T, ImageClassificationResult> >([this]() {
- AsyncInputQueue<T> inputQueue = _async_manager->popFromInput();
-
- inference(inputQueue.inputs);
-
- ImageClassificationResult &resultQueue = result();
-
- resultQueue.frame_number = inputQueue.frame_number;
- _async_manager->pushToOutput(resultQueue);
- });
- }
-
- shared_ptr<MetaInfo> metaInfo = getInputMetaInfo();
- vector<T> inputVector;
-
- preprocess(input.inference_src, metaInfo, inputVector);
-
- vector<vector<T> > inputVectors = { inputVector };
- _async_manager->push(inputVectors);
-}
-
-template<typename T> void ImageClassification<T>::getOutputNames(vector<string> &names)
-{
- TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
- IETensorBuffer &ie_tensor_buffer = tensor_buffer_obj.getIETensorBuffer();
-
- for (IETensorBuffer::iterator it = ie_tensor_buffer.begin(); it != ie_tensor_buffer.end(); it++)
- names.push_back(it->first);
-}
-
-template<typename T> void ImageClassification<T>::getOutpuTensor(string &target_name, vector<float> &tensor)
-{
- LOGI("ENTER");
-
- TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
-
- inference_engine_tensor_buffer *tensor_buffer = tensor_buffer_obj.getTensorBuffer(target_name);
- if (!tensor_buffer)
- throw InvalidOperation("Fail to get tensor buffer.");
-
- auto raw_buffer = static_cast<float *>(tensor_buffer->buffer);
-
- copy(&raw_buffer[0], &raw_buffer[tensor_buffer->size / sizeof(float)], back_inserter(tensor));
-
- LOGI("LEAVE");
-}
-
-template class ImageClassification<unsigned char>;
-template class ImageClassification<float>;
-
-}
-}
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <string>
-
-#include "machine_learning_exception.h"
-#include "mv_image_classification_config.h"
-#include "image_classification_adapter.h"
-
-using namespace std;
-using namespace MediaVision::Common;
-using namespace mediavision::machine_learning;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-ImageClassificationAdapter::ImageClassificationAdapter()
-{
- _config = make_shared<MachineLearningConfig>();
- _config->parseConfigFile(_config_file_name);
-
- create();
-}
-
-ImageClassificationAdapter::~ImageClassificationAdapter()
-{
- _image_classification->preDestroy();
-}
-
-void ImageClassificationAdapter::create()
-{
- _config->loadMetaFile(make_unique<ImageClassificationParser>());
- mv_inference_data_type_e dataType = _config->getInputMetaMap().begin()->second->dataType;
-
- switch (dataType) {
- case MV_INFERENCE_DATA_UINT8:
- _image_classification = make_unique<ImageClassificationDefault<unsigned char> >(_config);
- break;
- case MV_INFERENCE_DATA_FLOAT32:
- _image_classification = make_unique<ImageClassificationDefault<float> >(_config);
- break;
- default:
- throw InvalidOperation("Invalid image classification data type.");
- }
-}
-
-void ImageClassificationAdapter::setModelInfo(const string &model_file, const string &meta_file,
- const string &label_file, const string &model_name)
-{
- _config->setUserModel(model_file, meta_file, label_file);
- create();
-
- if (model_file.empty() && meta_file.empty()) {
- LOGW("Given model info is invalid so default model info will be used instead.");
- return;
- }
-}
-
-void ImageClassificationAdapter::setEngineInfo(const string &engine_type, const string &device_type)
-{
- _image_classification->setEngineInfo(engine_type, device_type);
-}
-
-void ImageClassificationAdapter::configure()
-{
- _image_classification->configure();
-}
-
-unsigned int ImageClassificationAdapter::getNumberOfEngines()
-{
- return _image_classification->getNumberOfEngines();
-}
-
-const string &ImageClassificationAdapter::getEngineType(unsigned int engine_index)
-{
- return _image_classification->getEngineType(engine_index);
-}
-
-unsigned int ImageClassificationAdapter::getNumberOfDevices(const string &engine_type)
-{
- return _image_classification->getNumberOfDevices(engine_type);
-}
-
-const string &ImageClassificationAdapter::getDeviceType(const string &engine_type, unsigned int device_index)
-{
- return _image_classification->getDeviceType(engine_type, device_index);
-}
-
-void ImageClassificationAdapter::prepare()
-{
- _image_classification->prepare();
-}
-
-void ImageClassificationAdapter::perform(InputBaseType &input)
-{
- _image_classification->perform(input.inference_src);
-}
-
-void ImageClassificationAdapter::performAsync(InputBaseType &input)
-{
- _image_classification->performAsync(static_cast<ImageClassificationInput &>(input));
-}
-
-OutputBaseType &ImageClassificationAdapter::getOutput()
-{
- return _image_classification->getOutput();
-}
-
-OutputBaseType &ImageClassificationAdapter::getOutputCache()
-{
- throw InvalidOperation("Not support yet.");
-}
-
-}
-}
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <string.h>
-#include <map>
-#include <algorithm>
-
-#include "machine_learning_exception.h"
-#include "image_classification_default.h"
-#include "Postprocess.h"
-
-using namespace std;
-using namespace mediavision::inference;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T>
-ImageClassificationDefault<T>::ImageClassificationDefault(shared_ptr<MachineLearningConfig> config)
- : ImageClassification<T>(config), _result()
-{}
-
-template<typename T> ImageClassificationDefault<T>::~ImageClassificationDefault()
-{}
-
-template<typename T> ImageClassificationResult &ImageClassificationDefault<T>::result()
-{
- vector<string> names;
-
- ImageClassification<T>::getOutputNames(names);
-
- vector<float> output_vec;
-
- // In case of image classification model, only one output tensor is used.
- ImageClassification<T>::getOutpuTensor(names[0], output_vec);
-
- auto metaInfo = _config->getOutputMetaMap().at(names[0]);
- auto decodingScore = static_pointer_cast<DecodingScore>(metaInfo->decodingTypeMap.at(DecodingType::SCORE));
-
- if (decodingScore->type == ScoreType::SIGMOID) {
- for (size_t idx = 0; idx < output_vec.size(); ++idx)
- output_vec[idx] = PostProcess::sigmoid(output_vec[idx]);
- }
-
- _result.label = _labels[max_element(output_vec.begin(), output_vec.end()) - output_vec.begin()];
- LOGI("Label = %s", _result.label.c_str());
-
- return _result;
-}
-
-template class ImageClassificationDefault<unsigned char>;
-template class ImageClassificationDefault<float>;
-
-}
-}
#include "mv_private.h"
#include "mv_feature_key.h"
-#include "itask.h"
+#include "ITask.h"
#include "mv_image_classification_internal.h"
-#include "image_classification_adapter.h"
-#include "machine_learning_exception.h"
-#include "MachineLearningNative.h"
+#include "ImageClassificationAdapter.h"
+#include "MvMlException.h"
+#include "native_capi.h"
#include "image_classification_type.h"
-#include "context.h"
+#include "Context.h"
#include <new>
#include <unistd.h>
install(
DIRECTORY ${PROJECT_SOURCE_DIR}/include/ DESTINATION include/media
FILES_MATCHING
- PATTERN "iimage_segmentation.h"
+ PATTERN "IImageSegmentation.h"
PATTERN "image_segmentation_type.h"
)
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __IIMAGE_SEGMENTATION_H__
+#define __IIMAGE_SEGMENTATION_H__
+
+#include <mv_common.h>
+
+#include "image_segmentation_type.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+class IImageSegmentation
+{
+public:
+ virtual ~IImageSegmentation() {};
+
+ virtual void preDestroy() = 0;
+ virtual void setUserModel(std::string model_file, std::string meta_file, std::string label_file) = 0;
+ virtual void setEngineInfo(std::string engine_type, std::string device_type) = 0;
+ virtual unsigned int getNumberOfEngines() = 0;
+ virtual const std::string &getEngineType(unsigned int engine_index) = 0;
+ virtual unsigned int getNumberOfDevices(const std::string &engine_type) = 0;
+ virtual const std::string &getDeviceType(const std::string &engine_type, const unsigned int device_index) = 0;
+ virtual void configure() = 0;
+ virtual void prepare() = 0;
+ virtual void perform(mv_source_h &mv_src) = 0;
+ virtual void performAsync(ImageSegmentationInput &input) = 0;
+ virtual ImageSegmentationResult &getOutput() = 0;
+ virtual ImageSegmentationResult &getOutputCache() = 0;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __IMAGE_SEGMENTATION_H__
+#define __IMAGE_SEGMENTATION_H__
+
+#include <queue>
+#include <thread>
+#include <mutex>
+#include <atomic>
+
+#include <mv_common.h>
+#include <mv_inference_type.h>
+#include "mv_private.h"
+
+#include "EngineConfig.h"
+#include "inference_engine_common_impl.h"
+#include "Inference.h"
+#include "image_segmentation_type.h"
+#include "MetaParser.h"
+#include "MvMlConfig.h"
+#include "ImageSegmentationParser.h"
+#include "MvMlPreprocess.h"
+#include "IImageSegmentation.h"
+#include "AsyncManager.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T> class ImageSegmentation : public IImageSegmentation
+{
+private:
+ std::unique_ptr<AsyncManager<T, ImageSegmentationResult> > _async_manager;
+ ImageSegmentationResult _current_result {};
+
+ void loadLabel();
+ void getEngineList();
+ void getDeviceList(const std::string &engine_type);
+ void preprocess(mv_source_h &mv_src, std::shared_ptr<MetaInfo> metaInfo, std::vector<T> &inputVector);
+ std::shared_ptr<MetaInfo> getInputMetaInfo();
+
+protected:
+ std::unique_ptr<mediavision::inference::Inference> _inference;
+ std::shared_ptr<Config> _config;
+ std::vector<std::string> _labels;
+ std::vector<std::string> _valid_backends;
+ std::vector<std::string> _valid_devices;
+ Preprocess _preprocess;
+
+ void getOutputNames(std::vector<std::string> &names);
+ void getOutputTensor(std::string target_name, std::vector<float> &tensor);
+ void inference(std::vector<std::vector<T> > &inputVectors);
+ virtual ImageSegmentationResult &result() = 0;
+
+public:
+ explicit ImageSegmentation(std::shared_ptr<Config> config);
+ virtual ~ImageSegmentation() = default;
+
+ void preDestroy() override;
+ void setUserModel(std::string model_file, std::string meta_file, std::string label_file) override;
+ void setEngineInfo(std::string engine_type, std::string device_type) override;
+ unsigned int getNumberOfEngines() override;
+ const std::string &getEngineType(unsigned int engine_index) override;
+ unsigned int getNumberOfDevices(const std::string &engine_type) override;
+ const std::string &getDeviceType(const std::string &engine_type, const unsigned int device_index) override;
+ void configure() override;
+ void prepare() override;
+ void perform(mv_source_h &mv_src) override;
+ void performAsync(ImageSegmentationInput &input) override;
+ ImageSegmentationResult &getOutput() override;
+ ImageSegmentationResult &getOutputCache() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __SELFIE_SEGMENTATION_ADAPTER_H__
+#define __SELFIE_SEGMENTATION_ADAPTER_H__
+
+#include <dlog.h>
+
+#include "EngineConfig.h"
+#include "ITask.h"
+#include "ImageSegmentation.h"
+#include "IImageSegmentation.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+class ImageSegmentationAdapter : public mediavision::common::ITask
+{
+private:
+ std::unique_ptr<IImageSegmentation> _selfie_segmentation;
+ std::shared_ptr<Config> _config;
+ const std::string _config_file_name = "selfie_segmentation.json";
+ const std::string _plugin_config_file_name = "selfie_segmentation_plugin.json";
+
+ void create(std::string model_name = "");
+ template<typename U> void create(ImageSegmentationTaskType task_type);
+ ImageSegmentationTaskType convertToTaskType(std::string model_name);
+
+public:
+ ImageSegmentationAdapter();
+ ~ImageSegmentationAdapter();
+
+ void setModelInfo(const std::string &model_file, const std::string &meta_file, const std::string &label_file,
+ const std::string &model_name) override;
+ void setEngineInfo(const std::string &engine_type, const std::string &device_type) override;
+ void configure() override;
+ unsigned int getNumberOfEngines() override;
+ const std::string &getEngineType(unsigned int engine_index) override;
+ unsigned int getNumberOfDevices(const std::string &engine_type) override;
+ const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
+ void prepare() override;
+ void perform(InputBaseType &input) override;
+ void performAsync(InputBaseType &input) override;
+ OutputBaseType &getOutput() override;
+ OutputBaseType &getOutputCache() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __IMAGE_SEGMENTATION_EXTERNAL_H__
+#define __IMAGE_SEGMENTATION_EXTERNAL_H__
+
+#include <mv_common.h>
+#include <mv_inference_type.h>
+
+#include <opencv2/core.hpp>
+#include <opencv2/imgproc.hpp>
+
+#include "image_segmentation_type.h"
+#include "IImageSegmentation.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+class ImageSegmentationExternal : public IImageSegmentation
+{
+private:
+ void *_plugin_handle {};
+ IImageSegmentation *_image_segmentation_plugin {};
+ ImageSegmentationTaskType _task_type { ImageSegmentationTaskType::IMAGE_SEGMENTATION_TASK_NONE };
+ ImageSegmentationResult _current_result;
+
+public:
+ ImageSegmentationExternal(ImageSegmentationTaskType task_type, const std::string &plugin_name);
+ virtual ~ImageSegmentationExternal();
+
+ void preDestroy() override;
+ void setUserModel(std::string model_file, std::string meta_file, std::string label_file) override;
+ void setEngineInfo(std::string engine_type, std::string device_type) override;
+ unsigned int getNumberOfEngines() override;
+ const std::string &getEngineType(unsigned int engine_index) override;
+ unsigned int getNumberOfDevices(const std::string &engine_type) override;
+ const std::string &getDeviceType(const std::string &engine_type, const unsigned int device_index) override;
+ void configure() override;
+ void prepare() override;
+ void perform(mv_source_h &mv_src) override;
+ void performAsync(ImageSegmentationInput &input) override;
+ ImageSegmentationResult &getOutput() override;
+ ImageSegmentationResult &getOutputCache() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __IMAGE_SEGMENTATION_PARSER_H__
+#define __IMAGE_SEGMENTATION_PARSER_H__
+
+#include "MetaParser.h"
+#include "PostprocessParser.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+class ImageSegmentationParser : public MetaParser
+{
+private:
+ PostprocessParser _postprocessParser;
+
+protected:
+ void parsePostprocess(std::shared_ptr<MetaInfo> meta_info, JsonObject *in_obj) override;
+
+public:
+ ImageSegmentationParser(int task_type = 0);
+ ~ImageSegmentationParser();
+};
+
+}
+}
+
+#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __IIMAGE_SEGMENTATION_H__
-#define __IIMAGE_SEGMENTATION_H__
-
-#include <mv_common.h>
-
-#include "image_segmentation_type.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-class IImageSegmentation
-{
-public:
- virtual ~IImageSegmentation() {};
-
- virtual void preDestroy() = 0;
- virtual void setUserModel(std::string model_file, std::string meta_file, std::string label_file) = 0;
- virtual void setEngineInfo(std::string engine_type, std::string device_type) = 0;
- virtual unsigned int getNumberOfEngines() = 0;
- virtual const std::string &getEngineType(unsigned int engine_index) = 0;
- virtual unsigned int getNumberOfDevices(const std::string &engine_type) = 0;
- virtual const std::string &getDeviceType(const std::string &engine_type, const unsigned int device_index) = 0;
- virtual void configure() = 0;
- virtual void prepare() = 0;
- virtual void perform(mv_source_h &mv_src) = 0;
- virtual void performAsync(ImageSegmentationInput &input) = 0;
- virtual ImageSegmentationResult &getOutput() = 0;
- virtual ImageSegmentationResult &getOutputCache() = 0;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __IMAGE_SEGMENTATION_H__
-#define __IMAGE_SEGMENTATION_H__
-
-#include <queue>
-#include <thread>
-#include <mutex>
-#include <atomic>
-
-#include <mv_common.h>
-#include <mv_inference_type.h>
-#include "mv_private.h"
-
-#include "EngineConfig.h"
-#include "inference_engine_common_impl.h"
-#include "Inference.h"
-#include "image_segmentation_type.h"
-#include "MetaParser.h"
-#include "machine_learning_config.h"
-#include "image_segmentation_parser.h"
-#include "machine_learning_preprocess.h"
-#include "iimage_segmentation.h"
-#include "async_manager.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T> class ImageSegmentation : public IImageSegmentation
-{
-private:
- std::unique_ptr<AsyncManager<T, ImageSegmentationResult> > _async_manager;
- ImageSegmentationResult _current_result {};
-
- void loadLabel();
- void getEngineList();
- void getDeviceList(const std::string &engine_type);
- void preprocess(mv_source_h &mv_src, std::shared_ptr<MetaInfo> metaInfo, std::vector<T> &inputVector);
- std::shared_ptr<MetaInfo> getInputMetaInfo();
-
-protected:
- std::unique_ptr<mediavision::inference::Inference> _inference;
- std::shared_ptr<MachineLearningConfig> _config;
- std::vector<std::string> _labels;
- std::vector<std::string> _valid_backends;
- std::vector<std::string> _valid_devices;
- Preprocess _preprocess;
-
- void getOutputNames(std::vector<std::string> &names);
- void getOutputTensor(std::string target_name, std::vector<float> &tensor);
- void inference(std::vector<std::vector<T> > &inputVectors);
- virtual ImageSegmentationResult &result() = 0;
-
-public:
- explicit ImageSegmentation(std::shared_ptr<MachineLearningConfig> config);
- virtual ~ImageSegmentation() = default;
-
- void preDestroy() override;
- void setUserModel(std::string model_file, std::string meta_file, std::string label_file) override;
- void setEngineInfo(std::string engine_type, std::string device_type) override;
- unsigned int getNumberOfEngines() override;
- const std::string &getEngineType(unsigned int engine_index) override;
- unsigned int getNumberOfDevices(const std::string &engine_type) override;
- const std::string &getDeviceType(const std::string &engine_type, const unsigned int device_index) override;
- void configure() override;
- void prepare() override;
- void perform(mv_source_h &mv_src) override;
- void performAsync(ImageSegmentationInput &input) override;
- ImageSegmentationResult &getOutput() override;
- ImageSegmentationResult &getOutputCache() override;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __IMAGE_SEGMENTATION_EXTERNAL_H__
-#define __IMAGE_SEGMENTATION_EXTERNAL_H__
-
-#include <mv_common.h>
-#include <mv_inference_type.h>
-
-#include <opencv2/core.hpp>
-#include <opencv2/imgproc.hpp>
-
-#include "image_segmentation_type.h"
-#include "iimage_segmentation.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-class ImageSegmentationExternal : public IImageSegmentation
-{
-private:
- void *_plugin_handle {};
- IImageSegmentation *_image_segmentation_plugin {};
- ImageSegmentationTaskType _task_type { ImageSegmentationTaskType::IMAGE_SEGMENTATION_TASK_NONE };
- ImageSegmentationResult _current_result;
-
-public:
- ImageSegmentationExternal(ImageSegmentationTaskType task_type, const std::string &plugin_name);
- virtual ~ImageSegmentationExternal();
-
- void preDestroy() override;
- void setUserModel(std::string model_file, std::string meta_file, std::string label_file) override;
- void setEngineInfo(std::string engine_type, std::string device_type) override;
- unsigned int getNumberOfEngines() override;
- const std::string &getEngineType(unsigned int engine_index) override;
- unsigned int getNumberOfDevices(const std::string &engine_type) override;
- const std::string &getDeviceType(const std::string &engine_type, const unsigned int device_index) override;
- void configure() override;
- void prepare() override;
- void perform(mv_source_h &mv_src) override;
- void performAsync(ImageSegmentationInput &input) override;
- ImageSegmentationResult &getOutput() override;
- ImageSegmentationResult &getOutputCache() override;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __IMAGE_SEGMENTATION_PARSER_H__
-#define __IMAGE_SEGMENTATION_PARSER_H__
-
-#include "MetaParser.h"
-#include "PostprocessParser.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-class ImageSegmentationParser : public MetaParser
-{
-private:
- PostprocessParser _postprocessParser;
-
-protected:
- void parsePostprocess(std::shared_ptr<MetaInfo> meta_info, JsonObject *in_obj) override;
-
-public:
- ImageSegmentationParser(int task_type = 0);
- ~ImageSegmentationParser();
-};
-
-}
-}
-
-#endif
\ No newline at end of file
#include <mv_common.h>
#include <mv_inference_type.h>
-#include "MachineLearningType.h"
+#include "mv_ml_types.h"
namespace mediavision
{
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __SELFIE_SEGMENTATION_ADAPTER_H__
-#define __SELFIE_SEGMENTATION_ADAPTER_H__
-
-#include <dlog.h>
-
-#include "EngineConfig.h"
-#include "itask.h"
-#include "image_segmentation.h"
-#include "iimage_segmentation.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-class ImageSegmentationAdapter : public mediavision::common::ITask
-{
-private:
- std::unique_ptr<IImageSegmentation> _selfie_segmentation;
- std::shared_ptr<MachineLearningConfig> _config;
- const std::string _config_file_name = "selfie_segmentation.json";
- const std::string _plugin_config_file_name = "selfie_segmentation_plugin.json";
-
- void create(std::string model_name = "");
- template<typename U> void create(ImageSegmentationTaskType task_type);
- ImageSegmentationTaskType convertToTaskType(std::string model_name);
-
-public:
- ImageSegmentationAdapter();
- ~ImageSegmentationAdapter();
-
- void setModelInfo(const std::string &model_file, const std::string &meta_file, const std::string &label_file,
- const std::string &model_name) override;
- void setEngineInfo(const std::string &engine_type, const std::string &device_type) override;
- void configure() override;
- unsigned int getNumberOfEngines() override;
- const std::string &getEngineType(unsigned int engine_index) override;
- unsigned int getNumberOfDevices(const std::string &engine_type) override;
- const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
- void prepare() override;
- void perform(InputBaseType &input) override;
- void performAsync(InputBaseType &input) override;
- OutputBaseType &getOutput() override;
- OutputBaseType &getOutputCache() override;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <fstream>
+#include <map>
+#include <memory>
+
+#include "MvMlException.h"
+#include "common.h"
+#include "mv_image_segmentation_config.h"
+#include "ImageSegmentation.h"
+
+using namespace std;
+using namespace mediavision::inference;
+using namespace MediaVision::Common;
+using namespace mediavision::common;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T> ImageSegmentation<T>::ImageSegmentation(shared_ptr<Config> config) : _config(config)
+{
+ _inference = make_unique<Inference>();
+}
+
+template<typename T> void ImageSegmentation<T>::preDestroy()
+{
+ if (!_async_manager)
+ return;
+
+ _async_manager->stop();
+}
+
+template<typename T> void ImageSegmentation<T>::getEngineList()
+{
+ for (auto idx = MV_INFERENCE_BACKEND_NONE + 1; idx < MV_INFERENCE_BACKEND_MAX; ++idx) {
+ auto backend = _inference->getSupportedInferenceBackend(idx);
+ // TODO. we need to describe what inference engines are supported by each Task API,
+ // and based on it, below inference engine types should be checked
+ // if a given type is supported by this Task API later. As of now, tflite only.
+ if (backend.second == true && backend.first.compare("tflite") == 0)
+ _valid_backends.push_back(backend.first);
+ }
+}
+
+template<typename T> void ImageSegmentation<T>::getDeviceList(const string &engine_type)
+{
+ // TODO. add device types available for a given engine type later.
+ // In default, cpu and gpu only.
+ _valid_devices.push_back("cpu");
+ _valid_devices.push_back("gpu");
+}
+
+template<typename T>
+void ImageSegmentation<T>::setUserModel(std::string model_file, std::string meta_file, std::string label_file)
+{}
+
+template<typename T>
+void ImageSegmentation<T>::setEngineInfo(std::string engine_type_name, std::string device_type_name)
+{
+ if (engine_type_name.empty() || device_type_name.empty())
+ throw InvalidParameter("Invalid engine info.");
+
+ transform(engine_type_name.begin(), engine_type_name.end(), engine_type_name.begin(), ::toupper);
+ transform(device_type_name.begin(), device_type_name.end(), device_type_name.begin(), ::toupper);
+
+ int engine_type = GetBackendType(engine_type_name);
+ int device_type = GetDeviceType(device_type_name);
+
+ if (engine_type == MEDIA_VISION_ERROR_INVALID_PARAMETER || device_type == MEDIA_VISION_ERROR_INVALID_PARAMETER)
+ throw InvalidParameter("backend or target device type not found.");
+
+ _config->setBackendType(engine_type);
+ _config->setTargetDeviceType(device_type);
+
+ LOGI("Engine type : %s => %d, Device type : %s => %d", engine_type_name.c_str(), engine_type,
+ device_type_name.c_str(), device_type);
+}
+
+template<typename T> unsigned int ImageSegmentation<T>::getNumberOfEngines()
+{
+ if (!_valid_backends.empty()) {
+ return _valid_backends.size();
+ }
+
+ getEngineList();
+ return _valid_backends.size();
+}
+
+template<typename T> const string &ImageSegmentation<T>::getEngineType(unsigned int engine_index)
+{
+ if (!_valid_backends.empty()) {
+ if (_valid_backends.size() <= engine_index)
+ throw InvalidParameter("Invalid engine index.");
+
+ return _valid_backends[engine_index];
+ }
+
+ getEngineList();
+
+ if (_valid_backends.size() <= engine_index)
+ throw InvalidParameter("Invalid engine index.");
+
+ return _valid_backends[engine_index];
+}
+
+template<typename T> unsigned int ImageSegmentation<T>::getNumberOfDevices(const string &engine_type)
+{
+ if (!_valid_devices.empty()) {
+ return _valid_devices.size();
+ }
+
+ getDeviceList(engine_type);
+ return _valid_devices.size();
+}
+
+template<typename T>
+const string &ImageSegmentation<T>::getDeviceType(const string &engine_type, unsigned int device_index)
+{
+ if (!_valid_devices.empty()) {
+ if (_valid_devices.size() <= device_index)
+ throw InvalidParameter("Invalid device index.");
+
+ return _valid_devices[device_index];
+ }
+
+ getDeviceList(engine_type);
+
+ if (_valid_devices.size() <= device_index)
+ throw InvalidParameter("Invalid device index.");
+
+ return _valid_devices[device_index];
+}
+
+template<typename T> void ImageSegmentation<T>::loadLabel()
+{
+ if (_config->getLabelFilePath().empty())
+ return;
+
+ ifstream readFile;
+
+ _labels.clear();
+ readFile.open(_config->getLabelFilePath().c_str());
+
+ if (readFile.fail())
+ throw InvalidOperation("Fail to open " + _config->getLabelFilePath() + " file.");
+
+ string line;
+
+ while (getline(readFile, line))
+ _labels.push_back(line);
+
+ readFile.close();
+}
+
+template<typename T> void ImageSegmentation<T>::configure()
+{
+ loadLabel();
+
+ int ret = _inference->bind(_config->getBackendType(), _config->getTargetDeviceType());
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to bind a backend engine.");
+}
+
+template<typename T> void ImageSegmentation<T>::prepare()
+{
+ int ret = _inference->configureInputMetaInfo(_config->getInputMetaMap());
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to configure input tensor info from meta file.");
+
+ ret = _inference->configureOutputMetaInfo(_config->getOutputMetaMap());
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to configure output tensor info from meta file.");
+
+ _inference->configureModelFiles("", _config->getModelFilePath(), "");
+
+ // Request to load model files to a backend engine.
+ ret = _inference->load();
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to load model files.");
+}
+
+template<typename T> shared_ptr<MetaInfo> ImageSegmentation<T>::getInputMetaInfo()
+{
+ TensorBuffer &tensor_buffer = _inference->getInputTensorBuffer();
+ IETensorBuffer &tensor_info_map = tensor_buffer.getIETensorBuffer();
+
+ // TODO. consider using multiple tensors later.
+ if (tensor_info_map.size() != 1)
+ throw InvalidOperation("Input tensor count not invalid.");
+
+ auto tensor_buffer_iter = tensor_info_map.begin();
+
+ // Get the meta information corresponding to a given input tensor name.
+ return _config->getInputMetaMap()[tensor_buffer_iter->first];
+}
+
+template<typename T>
+void ImageSegmentation<T>::preprocess(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo, vector<T> &inputVector)
+{
+ LOGI("ENTER");
+
+ PreprocessConfig config = { false,
+ metaInfo->colorSpace,
+ metaInfo->dataType,
+ metaInfo->getChannel(),
+ metaInfo->getWidth(),
+ metaInfo->getHeight() };
+
+ auto normalization = static_pointer_cast<DecodingNormal>(metaInfo->decodingTypeMap.at(DecodingType::NORMAL));
+ if (normalization) {
+ config.normalize = normalization->use;
+ config.mean = normalization->mean;
+ config.std = normalization->std;
+ }
+
+ auto quantization =
+ static_pointer_cast<DecodingQuantization>(metaInfo->decodingTypeMap.at(DecodingType::QUANTIZATION));
+ if (quantization) {
+ config.quantize = quantization->use;
+ config.scale = quantization->scale;
+ config.zeropoint = quantization->zeropoint;
+ }
+
+ _preprocess.setConfig(config);
+ _preprocess.run<T>(mv_src, inputVector);
+
+ LOGI("LEAVE");
+}
+
+template<typename T> void ImageSegmentation<T>::inference(vector<vector<T> > &inputVectors)
+{
+ LOGI("ENTER");
+
+ int ret = _inference->run<T>(inputVectors);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to run inference");
+
+ LOGI("LEAVE");
+}
+
+template<typename T> void ImageSegmentation<T>::perform(mv_source_h &mv_src)
+{
+ shared_ptr<MetaInfo> metaInfo = getInputMetaInfo();
+ vector<T> inputVector;
+
+ preprocess(mv_src, metaInfo, inputVector);
+
+ vector<vector<T> > inputVectors = { inputVector };
+ inference(inputVectors);
+}
+
+template<typename T> void ImageSegmentation<T>::performAsync(ImageSegmentationInput &input)
+{
+ if (!_async_manager) {
+ _async_manager = make_unique<AsyncManager<T, ImageSegmentationResult> >([this]() {
+ AsyncInputQueue<T> inputQueue = _async_manager->popFromInput();
+
+ inference(inputQueue.inputs);
+
+ ImageSegmentationResult &resultQueue = result();
+
+ resultQueue.frame_number = inputQueue.frame_number;
+ _async_manager->pushToOutput(resultQueue);
+ });
+ }
+
+ shared_ptr<MetaInfo> metaInfo = getInputMetaInfo();
+ vector<T> inputVector;
+
+ preprocess(input.inference_src, metaInfo, inputVector);
+
+ vector<vector<T> > inputVectors = { inputVector };
+ _async_manager->push(inputVectors);
+}
+
+template<typename T> ImageSegmentationResult &ImageSegmentation<T>::getOutput()
+{
+ if (_async_manager) {
+ if (!_async_manager->isWorking())
+ throw InvalidOperation("Object detection has been already destroyed so invalid operation.");
+
+ _current_result = _async_manager->pop();
+ } else {
+ // TODO. Check if inference request is completed or not here.
+ // If not then throw an exception.
+ _current_result = result();
+ }
+
+ return _current_result;
+}
+
+template<typename T> ImageSegmentationResult &ImageSegmentation<T>::getOutputCache()
+{
+ return _current_result;
+}
+
+template<typename T> void ImageSegmentation<T>::getOutputNames(vector<string> &names)
+{
+ TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
+ IETensorBuffer &ie_tensor_buffer = tensor_buffer_obj.getIETensorBuffer();
+
+ for (IETensorBuffer::iterator it = ie_tensor_buffer.begin(); it != ie_tensor_buffer.end(); it++)
+ names.push_back(it->first);
+}
+
+template<typename T> void ImageSegmentation<T>::getOutputTensor(string target_name, vector<float> &tensor)
+{
+ TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
+
+ inference_engine_tensor_buffer *tensor_buffer = tensor_buffer_obj.getTensorBuffer(target_name);
+ if (!tensor_buffer)
+ throw InvalidOperation("Fail to get tensor buffer.");
+
+ auto raw_buffer = static_cast<float *>(tensor_buffer->buffer);
+
+ copy(&raw_buffer[0], &raw_buffer[tensor_buffer->size / sizeof(float)], back_inserter(tensor));
+}
+
+template class ImageSegmentation<unsigned char>;
+template class ImageSegmentation<float>;
+
+}
+}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "MvMlException.h"
+#include "ImageSegmentationAdapter.h"
+#include "ImageSegmentationExternal.h"
+#include "mv_image_segmentation_config.h"
+
+using namespace std;
+using namespace MediaVision::Common;
+using namespace mediavision::machine_learning;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+ImageSegmentationAdapter::ImageSegmentationAdapter()
+{
+ _config = make_shared<Config>();
+
+ // If the model type needs external plugin then bypass to load the meta file and just create the external plugin.
+ // In this case, external plugin will use its own meta file approach regardless of Mediavision's one.
+ _config->parsePluginConfigFile(_plugin_config_file_name);
+ if (!_config->isPluginUsed())
+ _config->parseConfigFile(_config_file_name);
+
+ create(_config->getDefaultModelName());
+}
+
+ImageSegmentationAdapter::~ImageSegmentationAdapter()
+{
+ _selfie_segmentation->preDestroy();
+}
+
+template<typename U> void ImageSegmentationAdapter::create(ImageSegmentationTaskType task_type)
+{
+ // TODO. add switch-case statement here for Mediavision own task types.
+}
+
+void ImageSegmentationAdapter::create(std::string model_name)
+{
+ if (model_name.empty())
+ model_name = _config->getDefaultModelName();
+
+ auto task_type = convertToTaskType(model_name);
+
+ if (_config->isPluginUsed()) {
+ const auto &plugin_name = _config->getPluginFileName();
+
+ _selfie_segmentation = make_unique<ImageSegmentationExternal>(task_type, plugin_name.c_str());
+ return;
+ }
+
+ _config->loadMetaFile(make_unique<ImageSegmentationParser>(static_cast<int>(task_type)));
+ mv_inference_data_type_e dataType = _config->getInputMetaMap().begin()->second->dataType;
+
+ switch (dataType) {
+ case MV_INFERENCE_DATA_UINT8:
+ create<unsigned char>(task_type);
+ break;
+ case MV_INFERENCE_DATA_FLOAT32:
+ create<float>(task_type);
+ break;
+ default:
+ throw InvalidOperation("Invalid image segmentation data type.");
+ }
+}
+
+ImageSegmentationTaskType ImageSegmentationAdapter::convertToTaskType(string model_name)
+{
+ if (model_name.empty())
+ throw InvalidParameter("model name is empty.");
+
+ transform(model_name.begin(), model_name.end(), model_name.begin(), ::toupper);
+
+ if (model_name == "SELFIE_SEGMENTATION")
+ return ImageSegmentationTaskType::SELFIE_SEGMENTATION;
+
+ throw InvalidParameter("Invalid selfie segmentation model name.");
+}
+
+void ImageSegmentationAdapter::setModelInfo(const string &model_file, const string &meta_file, const string &label_file,
+ const string &model_name)
+{
+ try {
+ _config->setUserModel(model_file, meta_file, label_file);
+ create(model_name);
+ } catch (const BaseException &e) {
+ LOGW("A given model name is invalid so default task type will be used.");
+ }
+
+ if (model_file.empty() && meta_file.empty()) {
+ LOGW("Given model info is invalid so default model info will be used instead.");
+ return;
+ }
+
+ _selfie_segmentation->setUserModel(model_file, meta_file, label_file);
+}
+
+void ImageSegmentationAdapter::setEngineInfo(const string &engine_type, const string &device_type)
+{
+ _selfie_segmentation->setEngineInfo(engine_type, device_type);
+}
+
+void ImageSegmentationAdapter::configure()
+{
+ _selfie_segmentation->configure();
+}
+
+unsigned int ImageSegmentationAdapter::getNumberOfEngines()
+{
+ return _selfie_segmentation->getNumberOfEngines();
+}
+
+const string &ImageSegmentationAdapter::getEngineType(unsigned int engine_index)
+{
+ return _selfie_segmentation->getEngineType(engine_index);
+}
+
+unsigned int ImageSegmentationAdapter::getNumberOfDevices(const string &engine_type)
+{
+ return _selfie_segmentation->getNumberOfDevices(engine_type);
+}
+
+const string &ImageSegmentationAdapter::getDeviceType(const string &engine_type, unsigned int device_index)
+{
+ return _selfie_segmentation->getDeviceType(engine_type, device_index);
+}
+
+void ImageSegmentationAdapter::prepare()
+{
+ _selfie_segmentation->prepare();
+}
+
+void ImageSegmentationAdapter::perform(InputBaseType &input)
+{
+ _selfie_segmentation->perform(input.inference_src);
+}
+
+OutputBaseType &ImageSegmentationAdapter::getOutput()
+{
+ return _selfie_segmentation->getOutput();
+}
+
+OutputBaseType &ImageSegmentationAdapter::getOutputCache()
+{
+ return _selfie_segmentation->getOutputCache();
+}
+
+void ImageSegmentationAdapter::performAsync(InputBaseType &input)
+{
+ _selfie_segmentation->performAsync(static_cast<ImageSegmentationInput &>(input));
+}
+
+}
+}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <dlfcn.h>
+#include <sys/stat.h>
+
+#include "mv_private.h"
+
+#include "MvMlException.h"
+#include "ImageSegmentationExternal.h"
+
+using namespace std;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+typedef IImageSegmentation *create_t(ImageSegmentationTaskType task_type);
+typedef void destroy_t(IImageSegmentation *);
+
+ImageSegmentationExternal::ImageSegmentationExternal(ImageSegmentationTaskType task_type, const string &plugin_name)
+ : _plugin_handle(), _image_segmentation_plugin()
+{
+ // Load external plugin library.
+ LOGI("lib: %s", plugin_name.c_str());
+ _plugin_handle = dlopen(plugin_name.c_str(), RTLD_NOW);
+
+ if (!_plugin_handle)
+ throw InvalidOperation("Fail to open plugin library.");
+
+ create_t *createPlugin = (create_t *) dlsym(_plugin_handle, "createPlugin");
+ if (createPlugin == NULL) {
+ dlclose(_plugin_handle);
+ createPlugin = NULL;
+ throw InvalidOperation("Fail to get symbol from plugin library.");
+ }
+
+ _image_segmentation_plugin = createPlugin(task_type);
+ if (_image_segmentation_plugin == NULL) {
+ dlclose(_plugin_handle);
+ _plugin_handle = NULL;
+ throw InvalidOperation("Fail to create plugin module.");
+ }
+}
+
+ImageSegmentationExternal::~ImageSegmentationExternal()
+{
+ if (_plugin_handle) {
+ destroy_t *destroyPlugin = (destroy_t *) dlsym(_plugin_handle, "destroyPlugin");
+
+ destroyPlugin(_image_segmentation_plugin);
+ dlclose(_plugin_handle);
+ _image_segmentation_plugin = nullptr;
+ _plugin_handle = nullptr;
+ }
+}
+
+void ImageSegmentationExternal::preDestroy()
+{
+ _image_segmentation_plugin->preDestroy();
+}
+
+void ImageSegmentationExternal::setUserModel(string model_file, string meta_file, string label_file)
+{
+ _image_segmentation_plugin->setUserModel(model_file, meta_file, label_file);
+}
+
+void ImageSegmentationExternal::setEngineInfo(string engine_type, string device_type)
+{
+ _image_segmentation_plugin->setEngineInfo(engine_type, device_type);
+}
+
+unsigned int ImageSegmentationExternal::getNumberOfEngines()
+{
+ return _image_segmentation_plugin->getNumberOfEngines();
+}
+
+const string &ImageSegmentationExternal::getEngineType(unsigned int engine_index)
+{
+ return _image_segmentation_plugin->getEngineType(engine_index);
+}
+
+unsigned int ImageSegmentationExternal::getNumberOfDevices(const string &engine_type)
+{
+ return _image_segmentation_plugin->getNumberOfDevices(engine_type.c_str());
+}
+
+const string &ImageSegmentationExternal::getDeviceType(const string &engine_type, unsigned int device_index)
+{
+ return _image_segmentation_plugin->getDeviceType(engine_type.c_str(), device_index);
+}
+
+void ImageSegmentationExternal::configure()
+{
+ _image_segmentation_plugin->configure();
+}
+
+void ImageSegmentationExternal::prepare()
+{
+ _image_segmentation_plugin->prepare();
+}
+
+void ImageSegmentationExternal::perform(mv_source_h &mv_src)
+{
+ _image_segmentation_plugin->perform(mv_src);
+}
+
+void ImageSegmentationExternal::performAsync(ImageSegmentationInput &input)
+{
+ _image_segmentation_plugin->performAsync(input);
+}
+
+ImageSegmentationResult &ImageSegmentationExternal::getOutput()
+{
+ _current_result = _image_segmentation_plugin->getOutput();
+
+ return _current_result;
+}
+
+ImageSegmentationResult &ImageSegmentationExternal::getOutputCache()
+{
+ return _current_result;
+}
+
+}
+}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <memory>
+
+#include "MvMlException.h"
+#include "ImageSegmentationParser.h"
+#include "image_segmentation_type.h"
+
+using namespace std;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+ImageSegmentationParser::ImageSegmentationParser(int task_type)
+{
+ LOGI("ENTER");
+ LOGI("LEAVE");
+}
+
+ImageSegmentationParser::~ImageSegmentationParser()
+{
+ LOGI("ENTER");
+ LOGI("LEAVE");
+}
+
+void ImageSegmentationParser::parsePostprocess(shared_ptr<MetaInfo> meta_info, JsonObject *in_obj)
+{
+ LOGI("ENTER");
+ LOGI("LEAVE");
+}
+
+}
+}
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <string.h>
-#include <fstream>
-#include <map>
-#include <memory>
-
-#include "machine_learning_exception.h"
-#include "mv_machine_learning_common.h"
-#include "mv_image_segmentation_config.h"
-#include "image_segmentation.h"
-
-using namespace std;
-using namespace mediavision::inference;
-using namespace MediaVision::Common;
-using namespace mediavision::common;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T> ImageSegmentation<T>::ImageSegmentation(shared_ptr<MachineLearningConfig> config) : _config(config)
-{
- _inference = make_unique<Inference>();
-}
-
-template<typename T> void ImageSegmentation<T>::preDestroy()
-{
- if (!_async_manager)
- return;
-
- _async_manager->stop();
-}
-
-template<typename T> void ImageSegmentation<T>::getEngineList()
-{
- for (auto idx = MV_INFERENCE_BACKEND_NONE + 1; idx < MV_INFERENCE_BACKEND_MAX; ++idx) {
- auto backend = _inference->getSupportedInferenceBackend(idx);
- // TODO. we need to describe what inference engines are supported by each Task API,
- // and based on it, below inference engine types should be checked
- // if a given type is supported by this Task API later. As of now, tflite only.
- if (backend.second == true && backend.first.compare("tflite") == 0)
- _valid_backends.push_back(backend.first);
- }
-}
-
-template<typename T> void ImageSegmentation<T>::getDeviceList(const string &engine_type)
-{
- // TODO. add device types available for a given engine type later.
- // In default, cpu and gpu only.
- _valid_devices.push_back("cpu");
- _valid_devices.push_back("gpu");
-}
-
-template<typename T>
-void ImageSegmentation<T>::setUserModel(std::string model_file, std::string meta_file, std::string label_file)
-{}
-
-template<typename T>
-void ImageSegmentation<T>::setEngineInfo(std::string engine_type_name, std::string device_type_name)
-{
- if (engine_type_name.empty() || device_type_name.empty())
- throw InvalidParameter("Invalid engine info.");
-
- transform(engine_type_name.begin(), engine_type_name.end(), engine_type_name.begin(), ::toupper);
- transform(device_type_name.begin(), device_type_name.end(), device_type_name.begin(), ::toupper);
-
- int engine_type = GetBackendType(engine_type_name);
- int device_type = GetDeviceType(device_type_name);
-
- if (engine_type == MEDIA_VISION_ERROR_INVALID_PARAMETER || device_type == MEDIA_VISION_ERROR_INVALID_PARAMETER)
- throw InvalidParameter("backend or target device type not found.");
-
- _config->setBackendType(engine_type);
- _config->setTargetDeviceType(device_type);
-
- LOGI("Engine type : %s => %d, Device type : %s => %d", engine_type_name.c_str(), engine_type,
- device_type_name.c_str(), device_type);
-}
-
-template<typename T> unsigned int ImageSegmentation<T>::getNumberOfEngines()
-{
- if (!_valid_backends.empty()) {
- return _valid_backends.size();
- }
-
- getEngineList();
- return _valid_backends.size();
-}
-
-template<typename T> const string &ImageSegmentation<T>::getEngineType(unsigned int engine_index)
-{
- if (!_valid_backends.empty()) {
- if (_valid_backends.size() <= engine_index)
- throw InvalidParameter("Invalid engine index.");
-
- return _valid_backends[engine_index];
- }
-
- getEngineList();
-
- if (_valid_backends.size() <= engine_index)
- throw InvalidParameter("Invalid engine index.");
-
- return _valid_backends[engine_index];
-}
-
-template<typename T> unsigned int ImageSegmentation<T>::getNumberOfDevices(const string &engine_type)
-{
- if (!_valid_devices.empty()) {
- return _valid_devices.size();
- }
-
- getDeviceList(engine_type);
- return _valid_devices.size();
-}
-
-template<typename T>
-const string &ImageSegmentation<T>::getDeviceType(const string &engine_type, unsigned int device_index)
-{
- if (!_valid_devices.empty()) {
- if (_valid_devices.size() <= device_index)
- throw InvalidParameter("Invalid device index.");
-
- return _valid_devices[device_index];
- }
-
- getDeviceList(engine_type);
-
- if (_valid_devices.size() <= device_index)
- throw InvalidParameter("Invalid device index.");
-
- return _valid_devices[device_index];
-}
-
-template<typename T> void ImageSegmentation<T>::loadLabel()
-{
- if (_config->getLabelFilePath().empty())
- return;
-
- ifstream readFile;
-
- _labels.clear();
- readFile.open(_config->getLabelFilePath().c_str());
-
- if (readFile.fail())
- throw InvalidOperation("Fail to open " + _config->getLabelFilePath() + " file.");
-
- string line;
-
- while (getline(readFile, line))
- _labels.push_back(line);
-
- readFile.close();
-}
-
-template<typename T> void ImageSegmentation<T>::configure()
-{
- loadLabel();
-
- int ret = _inference->bind(_config->getBackendType(), _config->getTargetDeviceType());
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to bind a backend engine.");
-}
-
-template<typename T> void ImageSegmentation<T>::prepare()
-{
- int ret = _inference->configureInputMetaInfo(_config->getInputMetaMap());
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to configure input tensor info from meta file.");
-
- ret = _inference->configureOutputMetaInfo(_config->getOutputMetaMap());
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to configure output tensor info from meta file.");
-
- _inference->configureModelFiles("", _config->getModelFilePath(), "");
-
- // Request to load model files to a backend engine.
- ret = _inference->load();
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to load model files.");
-}
-
-template<typename T> shared_ptr<MetaInfo> ImageSegmentation<T>::getInputMetaInfo()
-{
- TensorBuffer &tensor_buffer = _inference->getInputTensorBuffer();
- IETensorBuffer &tensor_info_map = tensor_buffer.getIETensorBuffer();
-
- // TODO. consider using multiple tensors later.
- if (tensor_info_map.size() != 1)
- throw InvalidOperation("Input tensor count not invalid.");
-
- auto tensor_buffer_iter = tensor_info_map.begin();
-
- // Get the meta information corresponding to a given input tensor name.
- return _config->getInputMetaMap()[tensor_buffer_iter->first];
-}
-
-template<typename T>
-void ImageSegmentation<T>::preprocess(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo, vector<T> &inputVector)
-{
- LOGI("ENTER");
-
- PreprocessConfig config = { false,
- metaInfo->colorSpace,
- metaInfo->dataType,
- metaInfo->getChannel(),
- metaInfo->getWidth(),
- metaInfo->getHeight() };
-
- auto normalization = static_pointer_cast<DecodingNormal>(metaInfo->decodingTypeMap.at(DecodingType::NORMAL));
- if (normalization) {
- config.normalize = normalization->use;
- config.mean = normalization->mean;
- config.std = normalization->std;
- }
-
- auto quantization =
- static_pointer_cast<DecodingQuantization>(metaInfo->decodingTypeMap.at(DecodingType::QUANTIZATION));
- if (quantization) {
- config.quantize = quantization->use;
- config.scale = quantization->scale;
- config.zeropoint = quantization->zeropoint;
- }
-
- _preprocess.setConfig(config);
- _preprocess.run<T>(mv_src, inputVector);
-
- LOGI("LEAVE");
-}
-
-template<typename T> void ImageSegmentation<T>::inference(vector<vector<T> > &inputVectors)
-{
- LOGI("ENTER");
-
- int ret = _inference->run<T>(inputVectors);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to run inference");
-
- LOGI("LEAVE");
-}
-
-template<typename T> void ImageSegmentation<T>::perform(mv_source_h &mv_src)
-{
- shared_ptr<MetaInfo> metaInfo = getInputMetaInfo();
- vector<T> inputVector;
-
- preprocess(mv_src, metaInfo, inputVector);
-
- vector<vector<T> > inputVectors = { inputVector };
- inference(inputVectors);
-}
-
-template<typename T> void ImageSegmentation<T>::performAsync(ImageSegmentationInput &input)
-{
- if (!_async_manager) {
- _async_manager = make_unique<AsyncManager<T, ImageSegmentationResult> >([this]() {
- AsyncInputQueue<T> inputQueue = _async_manager->popFromInput();
-
- inference(inputQueue.inputs);
-
- ImageSegmentationResult &resultQueue = result();
-
- resultQueue.frame_number = inputQueue.frame_number;
- _async_manager->pushToOutput(resultQueue);
- });
- }
-
- shared_ptr<MetaInfo> metaInfo = getInputMetaInfo();
- vector<T> inputVector;
-
- preprocess(input.inference_src, metaInfo, inputVector);
-
- vector<vector<T> > inputVectors = { inputVector };
- _async_manager->push(inputVectors);
-}
-
-template<typename T> ImageSegmentationResult &ImageSegmentation<T>::getOutput()
-{
- if (_async_manager) {
- if (!_async_manager->isWorking())
- throw InvalidOperation("Object detection has been already destroyed so invalid operation.");
-
- _current_result = _async_manager->pop();
- } else {
- // TODO. Check if inference request is completed or not here.
- // If not then throw an exception.
- _current_result = result();
- }
-
- return _current_result;
-}
-
-template<typename T> ImageSegmentationResult &ImageSegmentation<T>::getOutputCache()
-{
- return _current_result;
-}
-
-template<typename T> void ImageSegmentation<T>::getOutputNames(vector<string> &names)
-{
- TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
- IETensorBuffer &ie_tensor_buffer = tensor_buffer_obj.getIETensorBuffer();
-
- for (IETensorBuffer::iterator it = ie_tensor_buffer.begin(); it != ie_tensor_buffer.end(); it++)
- names.push_back(it->first);
-}
-
-template<typename T> void ImageSegmentation<T>::getOutputTensor(string target_name, vector<float> &tensor)
-{
- TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
-
- inference_engine_tensor_buffer *tensor_buffer = tensor_buffer_obj.getTensorBuffer(target_name);
- if (!tensor_buffer)
- throw InvalidOperation("Fail to get tensor buffer.");
-
- auto raw_buffer = static_cast<float *>(tensor_buffer->buffer);
-
- copy(&raw_buffer[0], &raw_buffer[tensor_buffer->size / sizeof(float)], back_inserter(tensor));
-}
-
-template class ImageSegmentation<unsigned char>;
-template class ImageSegmentation<float>;
-
-}
-}
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <dlfcn.h>
-#include <sys/stat.h>
-
-#include "mv_private.h"
-
-#include "machine_learning_exception.h"
-#include "image_segmentation_external.h"
-
-using namespace std;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-typedef IImageSegmentation *create_t(ImageSegmentationTaskType task_type);
-typedef void destroy_t(IImageSegmentation *);
-
-ImageSegmentationExternal::ImageSegmentationExternal(ImageSegmentationTaskType task_type, const string &plugin_name)
- : _plugin_handle(), _image_segmentation_plugin()
-{
- // Load external plugin library.
- LOGI("lib: %s", plugin_name.c_str());
- _plugin_handle = dlopen(plugin_name.c_str(), RTLD_NOW);
-
- if (!_plugin_handle)
- throw InvalidOperation("Fail to open plugin library.");
-
- create_t *createPlugin = (create_t *) dlsym(_plugin_handle, "createPlugin");
- if (createPlugin == NULL) {
- dlclose(_plugin_handle);
- createPlugin = NULL;
- throw InvalidOperation("Fail to get symbol from plugin library.");
- }
-
- _image_segmentation_plugin = createPlugin(task_type);
- if (_image_segmentation_plugin == NULL) {
- dlclose(_plugin_handle);
- _plugin_handle = NULL;
- throw InvalidOperation("Fail to create plugin module.");
- }
-}
-
-ImageSegmentationExternal::~ImageSegmentationExternal()
-{
- if (_plugin_handle) {
- destroy_t *destroyPlugin = (destroy_t *) dlsym(_plugin_handle, "destroyPlugin");
-
- destroyPlugin(_image_segmentation_plugin);
- dlclose(_plugin_handle);
- _image_segmentation_plugin = nullptr;
- _plugin_handle = nullptr;
- }
-}
-
-void ImageSegmentationExternal::preDestroy()
-{
- _image_segmentation_plugin->preDestroy();
-}
-
-void ImageSegmentationExternal::setUserModel(string model_file, string meta_file, string label_file)
-{
- _image_segmentation_plugin->setUserModel(model_file, meta_file, label_file);
-}
-
-void ImageSegmentationExternal::setEngineInfo(string engine_type, string device_type)
-{
- _image_segmentation_plugin->setEngineInfo(engine_type, device_type);
-}
-
-unsigned int ImageSegmentationExternal::getNumberOfEngines()
-{
- return _image_segmentation_plugin->getNumberOfEngines();
-}
-
-const string &ImageSegmentationExternal::getEngineType(unsigned int engine_index)
-{
- return _image_segmentation_plugin->getEngineType(engine_index);
-}
-
-unsigned int ImageSegmentationExternal::getNumberOfDevices(const string &engine_type)
-{
- return _image_segmentation_plugin->getNumberOfDevices(engine_type.c_str());
-}
-
-const string &ImageSegmentationExternal::getDeviceType(const string &engine_type, unsigned int device_index)
-{
- return _image_segmentation_plugin->getDeviceType(engine_type.c_str(), device_index);
-}
-
-void ImageSegmentationExternal::configure()
-{
- _image_segmentation_plugin->configure();
-}
-
-void ImageSegmentationExternal::prepare()
-{
- _image_segmentation_plugin->prepare();
-}
-
-void ImageSegmentationExternal::perform(mv_source_h &mv_src)
-{
- _image_segmentation_plugin->perform(mv_src);
-}
-
-void ImageSegmentationExternal::performAsync(ImageSegmentationInput &input)
-{
- _image_segmentation_plugin->performAsync(input);
-}
-
-ImageSegmentationResult &ImageSegmentationExternal::getOutput()
-{
- _current_result = _image_segmentation_plugin->getOutput();
-
- return _current_result;
-}
-
-ImageSegmentationResult &ImageSegmentationExternal::getOutputCache()
-{
- return _current_result;
-}
-
-}
-}
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include <memory>
-
-#include "machine_learning_exception.h"
-#include "image_segmentation_parser.h"
-#include "image_segmentation_type.h"
-
-using namespace std;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-ImageSegmentationParser::ImageSegmentationParser(int task_type)
-{
- LOGI("ENTER");
- LOGI("LEAVE");
-}
-
-ImageSegmentationParser::~ImageSegmentationParser()
-{
- LOGI("ENTER");
- LOGI("LEAVE");
-}
-
-void ImageSegmentationParser::parsePostprocess(shared_ptr<MetaInfo> meta_info, JsonObject *in_obj)
-{
- LOGI("ENTER");
- LOGI("LEAVE");
-}
-
-}
-}
\ No newline at end of file
#include "mv_private.h"
#include "mv_feature_key.h"
-#include "itask.h"
+#include "ITask.h"
#include "mv_selfie_segmentation_type.h"
#include "mv_selfie_segmentation_internal.h"
-#include "selfie_segmentation_adapter.h"
-#include "machine_learning_exception.h"
-#include "MachineLearningNative.h"
+#include "ImageSegmentationAdapter.h"
+#include "MvMlException.h"
+#include "native_capi.h"
#include "image_segmentation_type.h"
-#include "context.h"
+#include "Context.h"
#include <new>
#include <unistd.h>
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "machine_learning_exception.h"
-#include "selfie_segmentation_adapter.h"
-#include "image_segmentation_external.h"
-#include "mv_image_segmentation_config.h"
-
-using namespace std;
-using namespace MediaVision::Common;
-using namespace mediavision::machine_learning;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-ImageSegmentationAdapter::ImageSegmentationAdapter()
-{
- _config = make_shared<MachineLearningConfig>();
-
- // If the model type needs external plugin then bypass to load the meta file and just create the external plugin.
- // In this case, external plugin will use its own meta file approach regardless of Mediavision's one.
- _config->parsePluginConfigFile(_plugin_config_file_name);
- if (!_config->isPluginUsed())
- _config->parseConfigFile(_config_file_name);
-
- create(_config->getDefaultModelName());
-}
-
-ImageSegmentationAdapter::~ImageSegmentationAdapter()
-{
- _selfie_segmentation->preDestroy();
-}
-
-template<typename U> void ImageSegmentationAdapter::create(ImageSegmentationTaskType task_type)
-{
- // TODO. add switch-case statement here for Mediavision own task types.
-}
-
-void ImageSegmentationAdapter::create(std::string model_name)
-{
- if (model_name.empty())
- model_name = _config->getDefaultModelName();
-
- auto task_type = convertToTaskType(model_name);
-
- if (_config->isPluginUsed()) {
- const auto &plugin_name = _config->getPluginFileName();
-
- _selfie_segmentation = make_unique<ImageSegmentationExternal>(task_type, plugin_name.c_str());
- return;
- }
-
- _config->loadMetaFile(make_unique<ImageSegmentationParser>(static_cast<int>(task_type)));
- mv_inference_data_type_e dataType = _config->getInputMetaMap().begin()->second->dataType;
-
- switch (dataType) {
- case MV_INFERENCE_DATA_UINT8:
- create<unsigned char>(task_type);
- break;
- case MV_INFERENCE_DATA_FLOAT32:
- create<float>(task_type);
- break;
- default:
- throw InvalidOperation("Invalid image segmentation data type.");
- }
-}
-
-ImageSegmentationTaskType ImageSegmentationAdapter::convertToTaskType(string model_name)
-{
- if (model_name.empty())
- throw InvalidParameter("model name is empty.");
-
- transform(model_name.begin(), model_name.end(), model_name.begin(), ::toupper);
-
- if (model_name == "SELFIE_SEGMENTATION")
- return ImageSegmentationTaskType::SELFIE_SEGMENTATION;
-
- throw InvalidParameter("Invalid selfie segmentation model name.");
-}
-
-void ImageSegmentationAdapter::setModelInfo(const string &model_file, const string &meta_file, const string &label_file,
- const string &model_name)
-{
- try {
- _config->setUserModel(model_file, meta_file, label_file);
- create(model_name);
- } catch (const BaseException &e) {
- LOGW("A given model name is invalid so default task type will be used.");
- }
-
- if (model_file.empty() && meta_file.empty()) {
- LOGW("Given model info is invalid so default model info will be used instead.");
- return;
- }
-
- _selfie_segmentation->setUserModel(model_file, meta_file, label_file);
-}
-
-void ImageSegmentationAdapter::setEngineInfo(const string &engine_type, const string &device_type)
-{
- _selfie_segmentation->setEngineInfo(engine_type, device_type);
-}
-
-void ImageSegmentationAdapter::configure()
-{
- _selfie_segmentation->configure();
-}
-
-unsigned int ImageSegmentationAdapter::getNumberOfEngines()
-{
- return _selfie_segmentation->getNumberOfEngines();
-}
-
-const string &ImageSegmentationAdapter::getEngineType(unsigned int engine_index)
-{
- return _selfie_segmentation->getEngineType(engine_index);
-}
-
-unsigned int ImageSegmentationAdapter::getNumberOfDevices(const string &engine_type)
-{
- return _selfie_segmentation->getNumberOfDevices(engine_type);
-}
-
-const string &ImageSegmentationAdapter::getDeviceType(const string &engine_type, unsigned int device_index)
-{
- return _selfie_segmentation->getDeviceType(engine_type, device_index);
-}
-
-void ImageSegmentationAdapter::prepare()
-{
- _selfie_segmentation->prepare();
-}
-
-void ImageSegmentationAdapter::perform(InputBaseType &input)
-{
- _selfie_segmentation->perform(input.inference_src);
-}
-
-OutputBaseType &ImageSegmentationAdapter::getOutput()
-{
- return _selfie_segmentation->getOutput();
-}
-
-OutputBaseType &ImageSegmentationAdapter::getOutputCache()
-{
- return _selfie_segmentation->getOutputCache();
-}
-
-void ImageSegmentationAdapter::performAsync(InputBaseType &input)
-{
- _selfie_segmentation->performAsync(static_cast<ImageSegmentationInput &>(input));
-}
-
-}
-}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __FACE_LANDMARK_ADAPTER_H__
+#define __FACE_LANDMARK_ADAPTER_H__
+
+#include <dlog.h>
+
+#include "EngineConfig.h"
+#include "ITask.h"
+#include "MvMlConfig.h"
+#include "FldTweakCnn.h"
+#include "ILandmarkDetection.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+class FacialLandmarkAdapter : public mediavision::common::ITask
+{
+private:
+ std::unique_ptr<ILandmarkDetection> _landmark_detection;
+ std::shared_ptr<Config> _config;
+ const std::string _config_file_name = "facial_landmark.json";
+
+ void create(const std::string &model_name);
+ template<typename U> void create(LandmarkDetectionTaskType task_type);
+ LandmarkDetectionTaskType convertToTaskType(std::string model_name);
+
+public:
+ FacialLandmarkAdapter();
+ ~FacialLandmarkAdapter();
+
+ void setModelInfo(const std::string &model_file, const std::string &meta_file, const std::string &label_file,
+ const std::string &model_name) override;
+ void setEngineInfo(const std::string &engine_type, const std::string &device_type) override;
+ void configure() override;
+ unsigned int getNumberOfEngines() override;
+ const std::string &getEngineType(unsigned int engine_index) override;
+ unsigned int getNumberOfDevices(const std::string &engine_type) override;
+ const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
+ void prepare() override;
+ void perform(InputBaseType &input) override;
+ void performAsync(InputBaseType &input) override;
+ OutputBaseType &getOutput() override;
+ OutputBaseType &getOutputCache() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __FLD_TWEAK_CNN_H__
+#define __FLD_TWEAK_CNN_H__
+
+#include <string>
+#include <memory>
+#include <mv_common.h>
+#include "mv_private.h"
+
+#include "LandmarkDetection.h"
+#include <mv_inference_type.h>
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T> class FldTweakCnn : public LandmarkDetection<T>
+{
+ using LandmarkDetection<T>::_config;
+ using LandmarkDetection<T>::_preprocess;
+ using LandmarkDetection<T>::_inference;
+
+private:
+ LandmarkDetectionResult _result;
+
+public:
+ FldTweakCnn(LandmarkDetectionTaskType task_type, std::shared_ptr<Config> config);
+ ~FldTweakCnn();
+
+ LandmarkDetectionResult &result() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __ILANDMARK_DETECTION_H__
+#define __ILANDMARK_DETECTION_H__
+
+#include <mv_common.h>
+
+#include "landmark_detection_type.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+class ILandmarkDetection
+{
+public:
+ virtual ~ILandmarkDetection() {};
+
+ virtual void preDestroy() = 0;
+ virtual LandmarkDetectionTaskType getTaskType() = 0;
+ virtual void setEngineInfo(std::string engine_type, std::string device_type) = 0;
+ virtual unsigned int getNumberOfEngines() = 0;
+ virtual const std::string &getEngineType(unsigned int engine_index) = 0;
+ virtual unsigned int getNumberOfDevices(const std::string &engine_type) = 0;
+ virtual const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) = 0;
+ virtual void configure() = 0;
+ virtual void prepare() = 0;
+ virtual void perform(mv_source_h &mv_src) = 0;
+ virtual void performAsync(LandmarkDetectionInput &input) = 0;
+ virtual LandmarkDetectionResult &getOutput() = 0;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __LANDMARK_DETECTION_H__
+#define __LANDMARK_DETECTION_H__
+
+#include <mv_common.h>
+#include <mv_inference_type.h>
+#include "mv_private.h"
+
+#include "EngineConfig.h"
+#include "inference_engine_common_impl.h"
+#include "Inference.h"
+#include "landmark_detection_type.h"
+#include "MetaParser.h"
+#include "LandmarkDetectionParser.h"
+#include "MvMlConfig.h"
+#include "MvMlPreprocess.h"
+#include "AsyncManager.h"
+#include "ILandmarkDetection.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T> class LandmarkDetection : public ILandmarkDetection
+{
+private:
+ std::unique_ptr<AsyncManager<T, LandmarkDetectionResult> > _async_manager;
+ LandmarkDetectionResult _current_result {};
+ LandmarkDetectionTaskType _task_type;
+
+ void loadLabel();
+ void getEngineList();
+ void getDeviceList(const std::string &engine_type);
+
+ void preprocess(mv_source_h &mv_src, std::shared_ptr<MetaInfo> metaInfo, std::vector<T> &inputVector);
+ std::shared_ptr<MetaInfo> getInputMetaInfo();
+
+protected:
+ std::unique_ptr<mediavision::inference::Inference> _inference;
+ std::shared_ptr<Config> _config;
+ std::vector<std::string> _labels;
+ std::vector<std::string> _valid_backends;
+ std::vector<std::string> _valid_devices;
+ Preprocess _preprocess;
+
+ void getOutputNames(std::vector<std::string> &names);
+ void getOutputTensor(std::string target_name, std::vector<float> &tensor);
+ void inference(std::vector<std::vector<T> > &inputVectors);
+ virtual LandmarkDetectionResult &result() = 0;
+
+public:
+ LandmarkDetection(LandmarkDetectionTaskType task_type, std::shared_ptr<Config> config);
+ virtual ~LandmarkDetection() = default;
+ void preDestroy();
+ LandmarkDetectionTaskType getTaskType();
+ void setEngineInfo(std::string engine_type_name, std::string device_type_name);
+ unsigned int getNumberOfEngines();
+ const std::string &getEngineType(unsigned int engine_index);
+ unsigned int getNumberOfDevices(const std::string &engine_type);
+ const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index);
+ void configure();
+ void prepare();
+ void perform(mv_source_h &mv_src);
+ void performAsync(LandmarkDetectionInput &input);
+ LandmarkDetectionResult &getOutput();
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __PLD_CPM_H__
+#define __PLD_CPM_H__
+
+#include <string>
+#include <memory>
+#include <mv_common.h>
+#include "mv_private.h"
+
+#include "LandmarkDetection.h"
+#include <mv_inference_type.h>
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T> class PldCpm : public LandmarkDetection<T>
+{
+ using LandmarkDetection<T>::_config;
+ using LandmarkDetection<T>::_preprocess;
+
+private:
+ LandmarkDetectionResult _result;
+
+public:
+ PldCpm(LandmarkDetectionTaskType task_type, std::shared_ptr<Config> config);
+ ~PldCpm();
+
+ LandmarkDetectionResult &result() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __POSE_LANDMARK_ADAPTER_H__
+#define __POSE_LANDMARK_ADAPTER_H__
+
+#include <dlog.h>
+
+#include "EngineConfig.h"
+#include "ITask.h"
+#include "MvMlConfig.h"
+#include "PldCpm.h"
+#include "ILandmarkDetection.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+class PoseLandmarkAdapter : public mediavision::common::ITask
+{
+private:
+ std::unique_ptr<ILandmarkDetection> _landmark_detection;
+ std::shared_ptr<Config> _config;
+ const std::string _config_file_name = "pose_landmark.json";
+
+ void create(const std::string &model_name);
+ template<typename U> void create(LandmarkDetectionTaskType task_type);
+ LandmarkDetectionTaskType convertToTaskType(std::string model_name);
+
+public:
+ PoseLandmarkAdapter();
+ ~PoseLandmarkAdapter();
+
+ void setModelInfo(const std::string &model_file, const std::string &meta_file, const std::string &label_file,
+ const std::string &model_name) override;
+ void setEngineInfo(const std::string &engine_type, const std::string &device_type) override;
+ void configure() override;
+ unsigned int getNumberOfEngines() override;
+ const std::string &getEngineType(unsigned int engine_index) override;
+ unsigned int getNumberOfDevices(const std::string &engine_type) override;
+ const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
+ void prepare() override;
+ void perform(InputBaseType &input) override;
+ void performAsync(InputBaseType &input) override;
+ OutputBaseType &getOutput() override;
+ OutputBaseType &getOutputCache() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __FACE_LANDMARK_ADAPTER_H__
-#define __FACE_LANDMARK_ADAPTER_H__
-
-#include <dlog.h>
-
-#include "EngineConfig.h"
-#include "itask.h"
-#include "machine_learning_config.h"
-#include "fld_tweak_cnn.h"
-#include "ilandmark_detection.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-class FacialLandmarkAdapter : public mediavision::common::ITask
-{
-private:
- std::unique_ptr<ILandmarkDetection> _landmark_detection;
- std::shared_ptr<MachineLearningConfig> _config;
- const std::string _config_file_name = "facial_landmark.json";
-
- void create(const std::string &model_name);
- template<typename U> void create(LandmarkDetectionTaskType task_type);
- LandmarkDetectionTaskType convertToTaskType(std::string model_name);
-
-public:
- FacialLandmarkAdapter();
- ~FacialLandmarkAdapter();
-
- void setModelInfo(const std::string &model_file, const std::string &meta_file, const std::string &label_file,
- const std::string &model_name) override;
- void setEngineInfo(const std::string &engine_type, const std::string &device_type) override;
- void configure() override;
- unsigned int getNumberOfEngines() override;
- const std::string &getEngineType(unsigned int engine_index) override;
- unsigned int getNumberOfDevices(const std::string &engine_type) override;
- const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
- void prepare() override;
- void perform(InputBaseType &input) override;
- void performAsync(InputBaseType &input) override;
- OutputBaseType &getOutput() override;
- OutputBaseType &getOutputCache() override;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __FLD_TWEAK_CNN_H__
-#define __FLD_TWEAK_CNN_H__
-
-#include <string>
-#include <memory>
-#include <mv_common.h>
-#include "mv_private.h"
-
-#include "landmark_detection.h"
-#include <mv_inference_type.h>
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T> class FldTweakCnn : public LandmarkDetection<T>
-{
- using LandmarkDetection<T>::_config;
- using LandmarkDetection<T>::_preprocess;
- using LandmarkDetection<T>::_inference;
-
-private:
- LandmarkDetectionResult _result;
-
-public:
- FldTweakCnn(LandmarkDetectionTaskType task_type, std::shared_ptr<MachineLearningConfig> config);
- ~FldTweakCnn();
-
- LandmarkDetectionResult &result() override;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __ILANDMARK_DETECTION_H__
-#define __ILANDMARK_DETECTION_H__
-
-#include <mv_common.h>
-
-#include "landmark_detection_type.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-class ILandmarkDetection
-{
-public:
- virtual ~ILandmarkDetection() {};
-
- virtual void preDestroy() = 0;
- virtual LandmarkDetectionTaskType getTaskType() = 0;
- virtual void setEngineInfo(std::string engine_type, std::string device_type) = 0;
- virtual unsigned int getNumberOfEngines() = 0;
- virtual const std::string &getEngineType(unsigned int engine_index) = 0;
- virtual unsigned int getNumberOfDevices(const std::string &engine_type) = 0;
- virtual const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) = 0;
- virtual void configure() = 0;
- virtual void prepare() = 0;
- virtual void perform(mv_source_h &mv_src) = 0;
- virtual void performAsync(LandmarkDetectionInput &input) = 0;
- virtual LandmarkDetectionResult &getOutput() = 0;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __LANDMARK_DETECTION_H__
-#define __LANDMARK_DETECTION_H__
-
-#include <mv_common.h>
-#include <mv_inference_type.h>
-#include "mv_private.h"
-
-#include "EngineConfig.h"
-#include "inference_engine_common_impl.h"
-#include "Inference.h"
-#include "landmark_detection_type.h"
-#include "MetaParser.h"
-#include "LandmarkDetectionParser.h"
-#include "machine_learning_config.h"
-#include "machine_learning_preprocess.h"
-#include "async_manager.h"
-#include "ilandmark_detection.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T> class LandmarkDetection : public ILandmarkDetection
-{
-private:
- std::unique_ptr<AsyncManager<T, LandmarkDetectionResult> > _async_manager;
- LandmarkDetectionResult _current_result {};
- LandmarkDetectionTaskType _task_type;
-
- void loadLabel();
- void getEngineList();
- void getDeviceList(const std::string &engine_type);
-
- void preprocess(mv_source_h &mv_src, std::shared_ptr<MetaInfo> metaInfo, std::vector<T> &inputVector);
- std::shared_ptr<MetaInfo> getInputMetaInfo();
-
-protected:
- std::unique_ptr<mediavision::inference::Inference> _inference;
- std::shared_ptr<MachineLearningConfig> _config;
- std::vector<std::string> _labels;
- std::vector<std::string> _valid_backends;
- std::vector<std::string> _valid_devices;
- Preprocess _preprocess;
-
- void getOutputNames(std::vector<std::string> &names);
- void getOutputTensor(std::string target_name, std::vector<float> &tensor);
- void inference(std::vector<std::vector<T> > &inputVectors);
- virtual LandmarkDetectionResult &result() = 0;
-
-public:
- LandmarkDetection(LandmarkDetectionTaskType task_type, std::shared_ptr<MachineLearningConfig> config);
- virtual ~LandmarkDetection() = default;
- void preDestroy();
- LandmarkDetectionTaskType getTaskType();
- void setEngineInfo(std::string engine_type_name, std::string device_type_name);
- unsigned int getNumberOfEngines();
- const std::string &getEngineType(unsigned int engine_index);
- unsigned int getNumberOfDevices(const std::string &engine_type);
- const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index);
- void configure();
- void prepare();
- void perform(mv_source_h &mv_src);
- void performAsync(LandmarkDetectionInput &input);
- LandmarkDetectionResult &getOutput();
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
#include <mv_common.h>
#include <mv_inference_type.h>
-#include "MachineLearningType.h"
+#include "mv_ml_types.h"
namespace mediavision
{
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __PLD_CPM_H__
-#define __PLD_CPM_H__
-
-#include <string>
-#include <memory>
-#include <mv_common.h>
-#include "mv_private.h"
-
-#include "landmark_detection.h"
-#include <mv_inference_type.h>
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T> class PldCpm : public LandmarkDetection<T>
-{
- using LandmarkDetection<T>::_config;
- using LandmarkDetection<T>::_preprocess;
-
-private:
- LandmarkDetectionResult _result;
-
-public:
- PldCpm(LandmarkDetectionTaskType task_type, std::shared_ptr<MachineLearningConfig> config);
- ~PldCpm();
-
- LandmarkDetectionResult &result() override;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __POSE_LANDMARK_ADAPTER_H__
-#define __POSE_LANDMARK_ADAPTER_H__
-
-#include <dlog.h>
-
-#include "EngineConfig.h"
-#include "itask.h"
-#include "machine_learning_config.h"
-#include "pld_cpm.h"
-#include "ilandmark_detection.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-class PoseLandmarkAdapter : public mediavision::common::ITask
-{
-private:
- std::unique_ptr<ILandmarkDetection> _landmark_detection;
- std::shared_ptr<MachineLearningConfig> _config;
- const std::string _config_file_name = "pose_landmark.json";
-
- void create(const std::string &model_name);
- template<typename U> void create(LandmarkDetectionTaskType task_type);
- LandmarkDetectionTaskType convertToTaskType(std::string model_name);
-
-public:
- PoseLandmarkAdapter();
- ~PoseLandmarkAdapter();
-
- void setModelInfo(const std::string &model_file, const std::string &meta_file, const std::string &label_file,
- const std::string &model_name) override;
- void setEngineInfo(const std::string &engine_type, const std::string &device_type) override;
- void configure() override;
- unsigned int getNumberOfEngines() override;
- const std::string &getEngineType(unsigned int engine_index) override;
- unsigned int getNumberOfDevices(const std::string &engine_type) override;
- const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
- void prepare() override;
- void perform(InputBaseType &input) override;
- void performAsync(InputBaseType &input) override;
- OutputBaseType &getOutput() override;
- OutputBaseType &getOutputCache() override;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "MvMlException.h"
+#include "FacialLandmarkAdapter.h"
+#include "mv_landmark_detection_config.h"
+
+using namespace std;
+using namespace MediaVision::Common;
+using namespace mediavision::machine_learning;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+FacialLandmarkAdapter::FacialLandmarkAdapter()
+{
+ _config = make_shared<Config>();
+ _config->parseConfigFile(_config_file_name);
+
+ create(_config->getDefaultModelName());
+}
+
+FacialLandmarkAdapter::~FacialLandmarkAdapter()
+{
+ _landmark_detection->preDestroy();
+}
+
+template<typename U> void FacialLandmarkAdapter::create(LandmarkDetectionTaskType task_type)
+{
+ switch (task_type) {
+ case LandmarkDetectionTaskType::FLD_TWEAK_CNN:
+ _landmark_detection = make_unique<FldTweakCnn<U> >(task_type, _config);
+ break;
+ default:
+ throw InvalidOperation("Invalid landmark detection task type.");
+ }
+}
+
+void FacialLandmarkAdapter::create(const string &model_name)
+{
+ LandmarkDetectionTaskType task_type = convertToTaskType(model_name);
+ _config->loadMetaFile(make_unique<LandmarkDetectionParser>(static_cast<int>(task_type)));
+ mv_inference_data_type_e dataType = _config->getInputMetaMap().begin()->second->dataType;
+
+ switch (dataType) {
+ case MV_INFERENCE_DATA_UINT8:
+ create<unsigned char>(task_type);
+ break;
+ case MV_INFERENCE_DATA_FLOAT32:
+ create<float>(task_type);
+ break;
+ default:
+ throw InvalidOperation("Invalid landmark detection data type.");
+ }
+}
+
+LandmarkDetectionTaskType FacialLandmarkAdapter::convertToTaskType(string model_name)
+{
+ if (model_name.empty())
+ throw InvalidParameter("model name is empty.");
+
+ transform(model_name.begin(), model_name.end(), model_name.begin(), ::toupper);
+
+ if (model_name == "FLD_TWEAK_CNN")
+ return LandmarkDetectionTaskType::FLD_TWEAK_CNN;
+ // TODO.
+
+ throw InvalidParameter("Invalid facial detection model name.");
+}
+
+void FacialLandmarkAdapter::setModelInfo(const string &model_file, const string &meta_file, const string &label_file,
+ const string &model_name)
+{
+ try {
+ _config->setUserModel(model_file, meta_file, label_file);
+ create(model_name);
+ } catch (const BaseException &e) {
+ LOGW("A given model name is invalid so default task type will be used.");
+ }
+
+ if (model_file.empty() && meta_file.empty()) {
+ LOGW("Given model info is invalid so default model info will be used instead.");
+ return;
+ }
+}
+
+void FacialLandmarkAdapter::setEngineInfo(const string &engine_type, const string &device_type)
+{
+ _landmark_detection->setEngineInfo(string(engine_type), string(device_type));
+}
+
+void FacialLandmarkAdapter::configure()
+{
+ _landmark_detection->configure();
+}
+
+unsigned int FacialLandmarkAdapter::getNumberOfEngines()
+{
+ return _landmark_detection->getNumberOfEngines();
+}
+
+const string &FacialLandmarkAdapter::getEngineType(unsigned int engine_index)
+{
+ return _landmark_detection->getEngineType(engine_index);
+}
+
+unsigned int FacialLandmarkAdapter::getNumberOfDevices(const string &engine_type)
+{
+ return _landmark_detection->getNumberOfDevices(engine_type);
+}
+
+const string &FacialLandmarkAdapter::getDeviceType(const string &engine_type, unsigned int device_index)
+{
+ return _landmark_detection->getDeviceType(engine_type, device_index);
+}
+
+void FacialLandmarkAdapter::prepare()
+{
+ _landmark_detection->prepare();
+}
+
+void FacialLandmarkAdapter::perform(InputBaseType &input)
+{
+ _landmark_detection->perform(input.inference_src);
+}
+
+void FacialLandmarkAdapter::performAsync(InputBaseType &input)
+{
+ _landmark_detection->performAsync(static_cast<LandmarkDetectionInput &>(input));
+}
+
+OutputBaseType &FacialLandmarkAdapter::getOutput()
+{
+ return _landmark_detection->getOutput();
+}
+
+OutputBaseType &FacialLandmarkAdapter::getOutputCache()
+{
+ throw InvalidOperation("Not support yet.");
+}
+
+}
+}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <map>
+#include <algorithm>
+
+#include "MvMlException.h"
+#include "FldTweakCnn.h"
+#include "Postprocess.h"
+
+using namespace std;
+using namespace mediavision::inference;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T>
+FldTweakCnn<T>::FldTweakCnn(LandmarkDetectionTaskType task_type, std::shared_ptr<Config> config)
+ : LandmarkDetection<T>(task_type, config), _result()
+{}
+
+template<typename T> FldTweakCnn<T>::~FldTweakCnn()
+{}
+
+template<typename T> LandmarkDetectionResult &FldTweakCnn<T>::result()
+{
+ constexpr static unsigned int numberOfLandmarks = 5;
+
+ // Clear _result object because result() function can be called every time user wants
+ // so make sure to clear existing result data before getting the data again.
+ _result = LandmarkDetectionResult();
+
+ vector<string> names;
+
+ LandmarkDetection<T>::getOutputNames(names);
+
+ auto scoreMetaInfo = _config->getOutputMetaMap().at(names[0]);
+ auto decodingLandmark =
+ static_pointer_cast<DecodingLandmark>(scoreMetaInfo->decodingTypeMap[DecodingType::LANDMARK]);
+
+ if (decodingLandmark->decoding_type != LandmarkDecodingType::BYPASS)
+ throw InvalidOperation("decoding type not support.");
+
+ if (decodingLandmark->coordinate_type != LandmarkCoordinateType::RATIO)
+ throw InvalidOperation("coordinate type not support.");
+
+ if (decodingLandmark->landmark_type != LandmarkType::SINGLE_2D)
+ throw InvalidOperation("landmark type not support.");
+
+ auto ori_src_width = static_cast<double>(_preprocess.getImageWidth()[0]);
+ auto ori_src_height = static_cast<double>(_preprocess.getImageHeight()[0]);
+ auto input_tensor_width = static_cast<double>(_inference->getInputWidth());
+ auto input_tensor_height = static_cast<double>(_inference->getInputHeight());
+
+ _result.number_of_landmarks = numberOfLandmarks;
+
+ vector<float> score_tensor;
+
+ LandmarkDetection<T>::getOutputTensor(names[0], score_tensor);
+
+ // Calculate the ratio[A] between the original image size and the input tensor size.
+ double width_ratio = ori_src_width / input_tensor_width;
+ double height_ratio = ori_src_height / input_tensor_height;
+
+ // In case that landmark coordinate type is RATIO, output tensor buffer contains ratio values indicating
+ // the position of each landmark for the input tensor.
+ // Therefore, each landmark position for original image is as following,
+ // x = [width A] * width of input tensor * width ratio value of output tensor.
+ // y = [height A] * height of input tensor * height ratio value of output tensor.
+ for (unsigned int idx = 0; idx < numberOfLandmarks; ++idx) {
+ _result.x_pos.push_back(
+ static_cast<unsigned int>(width_ratio * input_tensor_width * score_tensor[idx + idx * 1]));
+ _result.y_pos.push_back(
+ static_cast<unsigned int>(height_ratio * input_tensor_height * score_tensor[idx + idx * 1 + 1]));
+ }
+
+ return _result;
+}
+
+template class FldTweakCnn<unsigned char>;
+template class FldTweakCnn<float>;
+
+}
+}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <fstream>
+#include <map>
+#include <memory>
+#include <algorithm>
+
+#include "MvMlException.h"
+#include "common.h"
+#include "LandmarkDetection.h"
+
+using namespace std;
+using namespace mediavision::inference;
+using namespace MediaVision::Common;
+using namespace mediavision::common;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T>
+LandmarkDetection<T>::LandmarkDetection(LandmarkDetectionTaskType task_type, shared_ptr<Config> config)
+ : _task_type(task_type), _config(config)
+{
+ _inference = make_unique<Inference>();
+}
+
+template<typename T> void LandmarkDetection<T>::preDestroy()
+{
+ if (!_async_manager)
+ return;
+
+ _async_manager->stop();
+}
+
+template<typename T> LandmarkDetectionTaskType LandmarkDetection<T>::getTaskType()
+{
+ return _task_type;
+}
+
+template<typename T> void LandmarkDetection<T>::getEngineList()
+{
+ for (auto idx = MV_INFERENCE_BACKEND_NONE + 1; idx < MV_INFERENCE_BACKEND_MAX; ++idx) {
+ auto backend = _inference->getSupportedInferenceBackend(idx);
+ // TODO. we need to describe what inference engines are supported by each Task API,
+ // and based on it, below inference engine types should be checked
+ // if a given type is supported by this Task API later. As of now, tflite only.
+ if (backend.second == true && backend.first.compare("tflite") == 0)
+ _valid_backends.push_back(backend.first);
+ }
+}
+
+template<typename T> void LandmarkDetection<T>::getDeviceList(const string &engine_type)
+{
+ // TODO. add device types available for a given engine type later.
+ // In default, cpu and gpu only.
+ _valid_devices.push_back("cpu");
+ _valid_devices.push_back("gpu");
+}
+
+template<typename T> void LandmarkDetection<T>::setEngineInfo(string engine_type_name, string device_type_name)
+{
+ if (engine_type_name.empty() || device_type_name.empty())
+ throw InvalidParameter("Invalid engine info.");
+
+ transform(engine_type_name.begin(), engine_type_name.end(), engine_type_name.begin(), ::toupper);
+ transform(device_type_name.begin(), device_type_name.end(), device_type_name.begin(), ::toupper);
+
+ int engine_type = GetBackendType(engine_type_name);
+ int device_type = GetDeviceType(device_type_name);
+
+ if (engine_type == MEDIA_VISION_ERROR_INVALID_PARAMETER || device_type == MEDIA_VISION_ERROR_INVALID_PARAMETER)
+ throw InvalidParameter("backend or target device type not found.");
+
+ _config->setBackendType(engine_type);
+ _config->setTargetDeviceType(device_type);
+
+ LOGI("Engine type : %s => %d, Device type : %s => %d", engine_type_name.c_str(), engine_type,
+ device_type_name.c_str(), device_type);
+}
+
+template<typename T> unsigned int LandmarkDetection<T>::getNumberOfEngines()
+{
+ if (!_valid_backends.empty()) {
+ return _valid_backends.size();
+ }
+
+ getEngineList();
+ return _valid_backends.size();
+}
+
+template<typename T> const string &LandmarkDetection<T>::getEngineType(unsigned int engine_index)
+{
+ if (!_valid_backends.empty()) {
+ if (_valid_backends.size() <= engine_index)
+ throw InvalidParameter("Invalid engine index.");
+
+ return _valid_backends[engine_index];
+ }
+
+ getEngineList();
+
+ if (_valid_backends.size() <= engine_index)
+ throw InvalidParameter("Invalid engine index.");
+
+ return _valid_backends[engine_index];
+}
+
+template<typename T> unsigned int LandmarkDetection<T>::getNumberOfDevices(const string &engine_type)
+{
+ if (!_valid_devices.empty()) {
+ return _valid_devices.size();
+ }
+
+ getDeviceList(engine_type);
+ return _valid_devices.size();
+}
+
+template<typename T>
+const string &LandmarkDetection<T>::getDeviceType(const string &engine_type, const unsigned int device_index)
+{
+ if (!_valid_devices.empty()) {
+ if (_valid_devices.size() <= device_index)
+ throw InvalidParameter("Invalid device index.");
+
+ return _valid_devices[device_index];
+ }
+
+ getDeviceList(engine_type);
+
+ if (_valid_devices.size() <= device_index)
+ throw InvalidParameter("Invalid device index.");
+
+ return _valid_devices[device_index];
+}
+
+template<typename T> void LandmarkDetection<T>::loadLabel()
+{
+ if (_config->getLabelFilePath().empty())
+ return;
+
+ ifstream readFile;
+
+ _labels.clear();
+ readFile.open(_config->getLabelFilePath().c_str());
+
+ if (readFile.fail())
+ throw InvalidOperation("Fail to open " + _config->getLabelFilePath() + " file.");
+
+ string line;
+
+ while (getline(readFile, line))
+ _labels.push_back(line);
+
+ readFile.close();
+}
+
+template<typename T> void LandmarkDetection<T>::configure()
+{
+ loadLabel();
+
+ int ret = _inference->bind(_config->getBackendType(), _config->getTargetDeviceType());
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to bind a backend engine.");
+}
+
+template<typename T> void LandmarkDetection<T>::prepare()
+{
+ int ret = _inference->configureInputMetaInfo(_config->getInputMetaMap());
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to configure input tensor info from meta file.");
+
+ ret = _inference->configureOutputMetaInfo(_config->getOutputMetaMap());
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to configure output tensor info from meta file.");
+
+ _inference->configureModelFiles("", _config->getModelFilePath(), "");
+
+ // Request to load model files to a backend engine.
+ ret = _inference->load();
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to load model files.");
+}
+
+template<typename T> shared_ptr<MetaInfo> LandmarkDetection<T>::getInputMetaInfo()
+{
+ TensorBuffer &tensor_buffer = _inference->getInputTensorBuffer();
+ IETensorBuffer &tensor_info_map = tensor_buffer.getIETensorBuffer();
+
+ // TODO. consider using multiple tensors later.
+ if (tensor_info_map.size() != 1)
+ throw InvalidOperation("Input tensor count not invalid.");
+
+ auto tensor_buffer_iter = tensor_info_map.begin();
+
+ // Get the meta information corresponding to a given input tensor name.
+ return _config->getInputMetaMap()[tensor_buffer_iter->first];
+}
+
+template<typename T>
+void LandmarkDetection<T>::preprocess(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo, vector<T> &inputVector)
+{
+ LOGI("ENTER");
+
+ PreprocessConfig config = { false,
+ metaInfo->colorSpace,
+ metaInfo->dataType,
+ metaInfo->getChannel(),
+ metaInfo->getWidth(),
+ metaInfo->getHeight() };
+
+ auto normalization = static_pointer_cast<DecodingNormal>(metaInfo->decodingTypeMap.at(DecodingType::NORMAL));
+ if (normalization) {
+ config.normalize = normalization->use;
+ config.mean = normalization->mean;
+ config.std = normalization->std;
+ }
+
+ auto quantization =
+ static_pointer_cast<DecodingQuantization>(metaInfo->decodingTypeMap.at(DecodingType::QUANTIZATION));
+ if (quantization) {
+ config.quantize = quantization->use;
+ config.scale = quantization->scale;
+ config.zeropoint = quantization->zeropoint;
+ }
+
+ _preprocess.setConfig(config);
+ _preprocess.run<T>(mv_src, inputVector);
+
+ LOGI("LEAVE");
+}
+
+template<typename T> void LandmarkDetection<T>::inference(vector<vector<T> > &inputVectors)
+{
+ LOGI("ENTER");
+
+ int ret = _inference->run<T>(inputVectors);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to run inference");
+
+ LOGI("LEAVE");
+}
+
+template<typename T> void LandmarkDetection<T>::perform(mv_source_h &mv_src)
+{
+ shared_ptr<MetaInfo> metaInfo = getInputMetaInfo();
+ vector<T> inputVector;
+
+ preprocess(mv_src, metaInfo, inputVector);
+
+ vector<vector<T> > inputVectors = { inputVector };
+ inference(inputVectors);
+}
+
+template<typename T> LandmarkDetectionResult &LandmarkDetection<T>::getOutput()
+{
+ if (_async_manager) {
+ if (!_async_manager->isWorking())
+ throw InvalidOperation("landmark detection has been already destroyed so invalid operation.");
+
+ _current_result = _async_manager->pop();
+ } else {
+ // TODO. Check if inference request is completed or not here.
+ // If not then throw an exception.
+ _current_result = result();
+ }
+
+ return _current_result;
+}
+
+template<typename T> void LandmarkDetection<T>::performAsync(LandmarkDetectionInput &input)
+{
+ if (!_async_manager) {
+ _async_manager = make_unique<AsyncManager<T, LandmarkDetectionResult> >([this]() {
+ AsyncInputQueue<T> inputQueue = _async_manager->popFromInput();
+
+ LOGD("Poped input frame number = %ld", inputQueue.frame_number);
+
+ inference(inputQueue.inputs);
+
+ LandmarkDetectionResult &resultQueue = result();
+
+ resultQueue.frame_number = inputQueue.frame_number;
+ _async_manager->pushToOutput(resultQueue);
+ });
+ }
+
+ shared_ptr<MetaInfo> metaInfo = getInputMetaInfo();
+ vector<T> inputVector;
+
+ preprocess(input.inference_src, metaInfo, inputVector);
+
+ vector<vector<T> > inputVectors = { inputVector };
+ _async_manager->push(inputVectors);
+}
+
+template<typename T> void LandmarkDetection<T>::getOutputNames(vector<string> &names)
+{
+ TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
+ IETensorBuffer &ie_tensor_buffer = tensor_buffer_obj.getIETensorBuffer();
+
+ for (IETensorBuffer::iterator it = ie_tensor_buffer.begin(); it != ie_tensor_buffer.end(); it++)
+ names.push_back(it->first);
+}
+
+template<typename T> void LandmarkDetection<T>::getOutputTensor(string target_name, vector<float> &tensor)
+{
+ TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
+
+ inference_engine_tensor_buffer *tensor_buffer = tensor_buffer_obj.getTensorBuffer(target_name);
+ if (!tensor_buffer)
+ throw InvalidOperation("Fail to get tensor buffer.");
+
+ auto raw_buffer = static_cast<float *>(tensor_buffer->buffer);
+
+ copy(&raw_buffer[0], &raw_buffer[tensor_buffer->size / sizeof(float)], back_inserter(tensor));
+}
+
+template class LandmarkDetection<float>;
+template class LandmarkDetection<unsigned char>;
+
+}
+}
\ No newline at end of file
*/
#include <memory>
-#include "machine_learning_exception.h"
+#include "MvMlException.h"
#include "LandmarkDetectionParser.h"
#include "landmark_detection_type.h"
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <map>
+#include <algorithm>
+
+#include "MvMlException.h"
+#include "mv_landmark_detection_config.h"
+#include "PldCpm.h"
+#include "Postprocess.h"
+
+using namespace std;
+using namespace mediavision::inference;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T>
+PldCpm<T>::PldCpm(LandmarkDetectionTaskType task_type, std::shared_ptr<Config> config)
+ : LandmarkDetection<T>(task_type, config), _result()
+{}
+
+template<typename T> PldCpm<T>::~PldCpm()
+{}
+
+template<typename T> LandmarkDetectionResult &PldCpm<T>::result()
+{
+ // Clear _result object because result() function can be called every time user wants
+ // so make sure to clear existing result data before getting the data again.
+ _result = LandmarkDetectionResult();
+
+ vector<string> names;
+
+ LandmarkDetection<T>::getOutputNames(names);
+
+ auto scoreMetaInfo = _config->getOutputMetaMap().at(names[0]);
+ auto decodingLandmark =
+ static_pointer_cast<DecodingLandmark>(scoreMetaInfo->decodingTypeMap[DecodingType::LANDMARK]);
+
+ if (decodingLandmark->decoding_type != LandmarkDecodingType::HEATMAP)
+ throw InvalidOperation("decoding type not support.");
+
+ if (decodingLandmark->coordinate_type != LandmarkCoordinateType::PIXEL)
+ throw InvalidOperation("coordinate type not support.");
+
+ if (decodingLandmark->landmark_type != LandmarkType::SINGLE_2D)
+ throw InvalidOperation("landmark type not support.");
+
+ auto heatMapWidth = scoreMetaInfo->dims[2];
+ auto heatMapHeight = scoreMetaInfo->dims[1];
+ auto heatMapChannel = scoreMetaInfo->dims[3];
+ vector<float> score_tensor;
+
+ _result.number_of_landmarks = heatMapChannel;
+
+ LandmarkDetection<T>::getOutputTensor(names[0], score_tensor);
+
+ auto ori_src_width = static_cast<double>(_preprocess.getImageWidth()[0]);
+ auto ori_src_height = static_cast<double>(_preprocess.getImageHeight()[0]);
+ auto width_ratio = ori_src_width / static_cast<double>(heatMapWidth);
+ auto height_ratio = ori_src_height / static_cast<double>(heatMapHeight);
+
+ for (auto c = 0; c < heatMapChannel; ++c) {
+ float max_score = 0.0f;
+ int max_x = 0;
+ int max_y = 0;
+
+ for (auto y = 0; y < heatMapHeight; ++y) {
+ for (auto x = 0; x < heatMapWidth; ++x) {
+ auto score = score_tensor[y * heatMapWidth * heatMapChannel + x * heatMapChannel + c];
+ if (score < _config->getConfidenceThreshold())
+ continue;
+
+ if (max_score < score) {
+ max_score = score;
+ max_x = x;
+ max_y = y;
+ }
+ }
+ }
+
+ if (max_score == 0.0f) {
+ // If max_score is 0 then it means that all score values of current heatmap is 0 so
+ // ignore the scores that do not meet the threshold.
+ _result.number_of_landmarks--;
+ continue;
+ }
+
+ _result.x_pos.push_back(static_cast<size_t>(static_cast<double>(max_x) * width_ratio));
+ _result.y_pos.push_back(static_cast<size_t>(static_cast<double>(max_y) * height_ratio));
+ _result.scores.push_back(max_score);
+ }
+
+ return _result;
+}
+
+template class PldCpm<unsigned char>;
+template class PldCpm<float>;
+
+}
+}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "MvMlException.h"
+#include "PoseLandmarkAdapter.h"
+#include "mv_landmark_detection_config.h"
+
+using namespace std;
+using namespace MediaVision::Common;
+using namespace mediavision::machine_learning;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+PoseLandmarkAdapter::PoseLandmarkAdapter()
+{
+ _config = make_shared<Config>();
+ _config->parseConfigFile(_config_file_name);
+
+ create(_config->getDefaultModelName());
+}
+
+PoseLandmarkAdapter::~PoseLandmarkAdapter()
+{
+ _landmark_detection->preDestroy();
+}
+
+template<typename U> void PoseLandmarkAdapter::create(LandmarkDetectionTaskType task_type)
+{
+ switch (task_type) {
+ case LandmarkDetectionTaskType::PLD_CPM:
+ _landmark_detection = make_unique<PldCpm<U> >(task_type, _config);
+ break;
+ default:
+ throw InvalidOperation("Invalid landmark detection task type.");
+ }
+}
+
+void PoseLandmarkAdapter::create(const string &model_name)
+{
+ LandmarkDetectionTaskType task_type = convertToTaskType(model_name);
+ _config->loadMetaFile(make_unique<LandmarkDetectionParser>(static_cast<int>(task_type)));
+ mv_inference_data_type_e dataType = _config->getInputMetaMap().begin()->second->dataType;
+
+ switch (dataType) {
+ case MV_INFERENCE_DATA_UINT8:
+ create<unsigned char>(task_type);
+ break;
+ case MV_INFERENCE_DATA_FLOAT32:
+ create<float>(task_type);
+ break;
+ default:
+ throw InvalidOperation("Invalid landmark detection data type.");
+ }
+}
+
+LandmarkDetectionTaskType PoseLandmarkAdapter::convertToTaskType(string model_name)
+{
+ if (model_name.empty())
+ throw InvalidParameter("model name is empty.");
+
+ transform(model_name.begin(), model_name.end(), model_name.begin(), ::toupper);
+
+ if (model_name == "PLD_CPM")
+ return LandmarkDetectionTaskType::PLD_CPM;
+
+ throw InvalidParameter("Invalid pose landmark model name.");
+}
+
+void PoseLandmarkAdapter::setModelInfo(const string &model_file, const string &meta_file, const string &label_file,
+ const string &model_name)
+{
+ try {
+ _config->setUserModel(model_file, meta_file, label_file);
+ create(model_name);
+ } catch (const BaseException &e) {
+ LOGW("A given model name is invalid so default task type will be used.");
+ }
+
+ if (model_file.empty() && meta_file.empty()) {
+ LOGW("Given model info is invalid so default model info will be used instead.");
+ return;
+ }
+}
+
+void PoseLandmarkAdapter::setEngineInfo(const string &engine_type, const string &device_type)
+{
+ _landmark_detection->setEngineInfo(engine_type, device_type);
+}
+
+void PoseLandmarkAdapter::configure()
+{
+ _landmark_detection->configure();
+}
+
+unsigned int PoseLandmarkAdapter::getNumberOfEngines()
+{
+ return _landmark_detection->getNumberOfEngines();
+}
+
+const string &PoseLandmarkAdapter::getEngineType(unsigned int engine_index)
+{
+ return _landmark_detection->getEngineType(engine_index);
+}
+
+unsigned int PoseLandmarkAdapter::getNumberOfDevices(const string &engine_type)
+{
+ return _landmark_detection->getNumberOfDevices(engine_type);
+}
+
+const string &PoseLandmarkAdapter::getDeviceType(const string &engine_type, unsigned int device_index)
+{
+ return _landmark_detection->getDeviceType(engine_type, device_index);
+}
+
+void PoseLandmarkAdapter::prepare()
+{
+ _landmark_detection->prepare();
+}
+
+void PoseLandmarkAdapter::perform(InputBaseType &input)
+{
+ _landmark_detection->perform(input.inference_src);
+}
+
+void PoseLandmarkAdapter::performAsync(InputBaseType &input)
+{
+ _landmark_detection->performAsync(static_cast<LandmarkDetectionInput &>(input));
+}
+
+OutputBaseType &PoseLandmarkAdapter::getOutput()
+{
+ return _landmark_detection->getOutput();
+}
+
+OutputBaseType &PoseLandmarkAdapter::getOutputCache()
+{
+ throw InvalidOperation("Not support yet.");
+}
+
+}
+}
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "machine_learning_exception.h"
-#include "facial_landmark_adapter.h"
-#include "mv_landmark_detection_config.h"
-
-using namespace std;
-using namespace MediaVision::Common;
-using namespace mediavision::machine_learning;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-FacialLandmarkAdapter::FacialLandmarkAdapter()
-{
- _config = make_shared<MachineLearningConfig>();
- _config->parseConfigFile(_config_file_name);
-
- create(_config->getDefaultModelName());
-}
-
-FacialLandmarkAdapter::~FacialLandmarkAdapter()
-{
- _landmark_detection->preDestroy();
-}
-
-template<typename U> void FacialLandmarkAdapter::create(LandmarkDetectionTaskType task_type)
-{
- switch (task_type) {
- case LandmarkDetectionTaskType::FLD_TWEAK_CNN:
- _landmark_detection = make_unique<FldTweakCnn<U> >(task_type, _config);
- break;
- default:
- throw InvalidOperation("Invalid landmark detection task type.");
- }
-}
-
-void FacialLandmarkAdapter::create(const string &model_name)
-{
- LandmarkDetectionTaskType task_type = convertToTaskType(model_name);
- _config->loadMetaFile(make_unique<LandmarkDetectionParser>(static_cast<int>(task_type)));
- mv_inference_data_type_e dataType = _config->getInputMetaMap().begin()->second->dataType;
-
- switch (dataType) {
- case MV_INFERENCE_DATA_UINT8:
- create<unsigned char>(task_type);
- break;
- case MV_INFERENCE_DATA_FLOAT32:
- create<float>(task_type);
- break;
- default:
- throw InvalidOperation("Invalid landmark detection data type.");
- }
-}
-
-LandmarkDetectionTaskType FacialLandmarkAdapter::convertToTaskType(string model_name)
-{
- if (model_name.empty())
- throw InvalidParameter("model name is empty.");
-
- transform(model_name.begin(), model_name.end(), model_name.begin(), ::toupper);
-
- if (model_name == "FLD_TWEAK_CNN")
- return LandmarkDetectionTaskType::FLD_TWEAK_CNN;
- // TODO.
-
- throw InvalidParameter("Invalid facial detection model name.");
-}
-
-void FacialLandmarkAdapter::setModelInfo(const string &model_file, const string &meta_file, const string &label_file,
- const string &model_name)
-{
- try {
- _config->setUserModel(model_file, meta_file, label_file);
- create(model_name);
- } catch (const BaseException &e) {
- LOGW("A given model name is invalid so default task type will be used.");
- }
-
- if (model_file.empty() && meta_file.empty()) {
- LOGW("Given model info is invalid so default model info will be used instead.");
- return;
- }
-}
-
-void FacialLandmarkAdapter::setEngineInfo(const string &engine_type, const string &device_type)
-{
- _landmark_detection->setEngineInfo(string(engine_type), string(device_type));
-}
-
-void FacialLandmarkAdapter::configure()
-{
- _landmark_detection->configure();
-}
-
-unsigned int FacialLandmarkAdapter::getNumberOfEngines()
-{
- return _landmark_detection->getNumberOfEngines();
-}
-
-const string &FacialLandmarkAdapter::getEngineType(unsigned int engine_index)
-{
- return _landmark_detection->getEngineType(engine_index);
-}
-
-unsigned int FacialLandmarkAdapter::getNumberOfDevices(const string &engine_type)
-{
- return _landmark_detection->getNumberOfDevices(engine_type);
-}
-
-const string &FacialLandmarkAdapter::getDeviceType(const string &engine_type, unsigned int device_index)
-{
- return _landmark_detection->getDeviceType(engine_type, device_index);
-}
-
-void FacialLandmarkAdapter::prepare()
-{
- _landmark_detection->prepare();
-}
-
-void FacialLandmarkAdapter::perform(InputBaseType &input)
-{
- _landmark_detection->perform(input.inference_src);
-}
-
-void FacialLandmarkAdapter::performAsync(InputBaseType &input)
-{
- _landmark_detection->performAsync(static_cast<LandmarkDetectionInput &>(input));
-}
-
-OutputBaseType &FacialLandmarkAdapter::getOutput()
-{
- return _landmark_detection->getOutput();
-}
-
-OutputBaseType &FacialLandmarkAdapter::getOutputCache()
-{
- throw InvalidOperation("Not support yet.");
-}
-
-}
-}
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <string.h>
-#include <map>
-#include <algorithm>
-
-#include "machine_learning_exception.h"
-#include "fld_tweak_cnn.h"
-#include "Postprocess.h"
-
-using namespace std;
-using namespace mediavision::inference;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T>
-FldTweakCnn<T>::FldTweakCnn(LandmarkDetectionTaskType task_type, std::shared_ptr<MachineLearningConfig> config)
- : LandmarkDetection<T>(task_type, config), _result()
-{}
-
-template<typename T> FldTweakCnn<T>::~FldTweakCnn()
-{}
-
-template<typename T> LandmarkDetectionResult &FldTweakCnn<T>::result()
-{
- constexpr static unsigned int numberOfLandmarks = 5;
-
- // Clear _result object because result() function can be called every time user wants
- // so make sure to clear existing result data before getting the data again.
- _result = LandmarkDetectionResult();
-
- vector<string> names;
-
- LandmarkDetection<T>::getOutputNames(names);
-
- auto scoreMetaInfo = _config->getOutputMetaMap().at(names[0]);
- auto decodingLandmark =
- static_pointer_cast<DecodingLandmark>(scoreMetaInfo->decodingTypeMap[DecodingType::LANDMARK]);
-
- if (decodingLandmark->decoding_type != LandmarkDecodingType::BYPASS)
- throw InvalidOperation("decoding type not support.");
-
- if (decodingLandmark->coordinate_type != LandmarkCoordinateType::RATIO)
- throw InvalidOperation("coordinate type not support.");
-
- if (decodingLandmark->landmark_type != LandmarkType::SINGLE_2D)
- throw InvalidOperation("landmark type not support.");
-
- auto ori_src_width = static_cast<double>(_preprocess.getImageWidth()[0]);
- auto ori_src_height = static_cast<double>(_preprocess.getImageHeight()[0]);
- auto input_tensor_width = static_cast<double>(_inference->getInputWidth());
- auto input_tensor_height = static_cast<double>(_inference->getInputHeight());
-
- _result.number_of_landmarks = numberOfLandmarks;
-
- vector<float> score_tensor;
-
- LandmarkDetection<T>::getOutputTensor(names[0], score_tensor);
-
- // Calculate the ratio[A] between the original image size and the input tensor size.
- double width_ratio = ori_src_width / input_tensor_width;
- double height_ratio = ori_src_height / input_tensor_height;
-
- // In case that landmark coordinate type is RATIO, output tensor buffer contains ratio values indicating
- // the position of each landmark for the input tensor.
- // Therefore, each landmark position for original image is as following,
- // x = [width A] * width of input tensor * width ratio value of output tensor.
- // y = [height A] * height of input tensor * height ratio value of output tensor.
- for (unsigned int idx = 0; idx < numberOfLandmarks; ++idx) {
- _result.x_pos.push_back(
- static_cast<unsigned int>(width_ratio * input_tensor_width * score_tensor[idx + idx * 1]));
- _result.y_pos.push_back(
- static_cast<unsigned int>(height_ratio * input_tensor_height * score_tensor[idx + idx * 1 + 1]));
- }
-
- return _result;
-}
-
-template class FldTweakCnn<unsigned char>;
-template class FldTweakCnn<float>;
-
-}
-}
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <string.h>
-#include <fstream>
-#include <map>
-#include <memory>
-#include <algorithm>
-
-#include "machine_learning_exception.h"
-#include "mv_machine_learning_common.h"
-#include "landmark_detection.h"
-
-using namespace std;
-using namespace mediavision::inference;
-using namespace MediaVision::Common;
-using namespace mediavision::common;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T>
-LandmarkDetection<T>::LandmarkDetection(LandmarkDetectionTaskType task_type, shared_ptr<MachineLearningConfig> config)
- : _task_type(task_type), _config(config)
-{
- _inference = make_unique<Inference>();
-}
-
-template<typename T> void LandmarkDetection<T>::preDestroy()
-{
- if (!_async_manager)
- return;
-
- _async_manager->stop();
-}
-
-template<typename T> LandmarkDetectionTaskType LandmarkDetection<T>::getTaskType()
-{
- return _task_type;
-}
-
-template<typename T> void LandmarkDetection<T>::getEngineList()
-{
- for (auto idx = MV_INFERENCE_BACKEND_NONE + 1; idx < MV_INFERENCE_BACKEND_MAX; ++idx) {
- auto backend = _inference->getSupportedInferenceBackend(idx);
- // TODO. we need to describe what inference engines are supported by each Task API,
- // and based on it, below inference engine types should be checked
- // if a given type is supported by this Task API later. As of now, tflite only.
- if (backend.second == true && backend.first.compare("tflite") == 0)
- _valid_backends.push_back(backend.first);
- }
-}
-
-template<typename T> void LandmarkDetection<T>::getDeviceList(const string &engine_type)
-{
- // TODO. add device types available for a given engine type later.
- // In default, cpu and gpu only.
- _valid_devices.push_back("cpu");
- _valid_devices.push_back("gpu");
-}
-
-template<typename T> void LandmarkDetection<T>::setEngineInfo(string engine_type_name, string device_type_name)
-{
- if (engine_type_name.empty() || device_type_name.empty())
- throw InvalidParameter("Invalid engine info.");
-
- transform(engine_type_name.begin(), engine_type_name.end(), engine_type_name.begin(), ::toupper);
- transform(device_type_name.begin(), device_type_name.end(), device_type_name.begin(), ::toupper);
-
- int engine_type = GetBackendType(engine_type_name);
- int device_type = GetDeviceType(device_type_name);
-
- if (engine_type == MEDIA_VISION_ERROR_INVALID_PARAMETER || device_type == MEDIA_VISION_ERROR_INVALID_PARAMETER)
- throw InvalidParameter("backend or target device type not found.");
-
- _config->setBackendType(engine_type);
- _config->setTargetDeviceType(device_type);
-
- LOGI("Engine type : %s => %d, Device type : %s => %d", engine_type_name.c_str(), engine_type,
- device_type_name.c_str(), device_type);
-}
-
-template<typename T> unsigned int LandmarkDetection<T>::getNumberOfEngines()
-{
- if (!_valid_backends.empty()) {
- return _valid_backends.size();
- }
-
- getEngineList();
- return _valid_backends.size();
-}
-
-template<typename T> const string &LandmarkDetection<T>::getEngineType(unsigned int engine_index)
-{
- if (!_valid_backends.empty()) {
- if (_valid_backends.size() <= engine_index)
- throw InvalidParameter("Invalid engine index.");
-
- return _valid_backends[engine_index];
- }
-
- getEngineList();
-
- if (_valid_backends.size() <= engine_index)
- throw InvalidParameter("Invalid engine index.");
-
- return _valid_backends[engine_index];
-}
-
-template<typename T> unsigned int LandmarkDetection<T>::getNumberOfDevices(const string &engine_type)
-{
- if (!_valid_devices.empty()) {
- return _valid_devices.size();
- }
-
- getDeviceList(engine_type);
- return _valid_devices.size();
-}
-
-template<typename T>
-const string &LandmarkDetection<T>::getDeviceType(const string &engine_type, const unsigned int device_index)
-{
- if (!_valid_devices.empty()) {
- if (_valid_devices.size() <= device_index)
- throw InvalidParameter("Invalid device index.");
-
- return _valid_devices[device_index];
- }
-
- getDeviceList(engine_type);
-
- if (_valid_devices.size() <= device_index)
- throw InvalidParameter("Invalid device index.");
-
- return _valid_devices[device_index];
-}
-
-template<typename T> void LandmarkDetection<T>::loadLabel()
-{
- if (_config->getLabelFilePath().empty())
- return;
-
- ifstream readFile;
-
- _labels.clear();
- readFile.open(_config->getLabelFilePath().c_str());
-
- if (readFile.fail())
- throw InvalidOperation("Fail to open " + _config->getLabelFilePath() + " file.");
-
- string line;
-
- while (getline(readFile, line))
- _labels.push_back(line);
-
- readFile.close();
-}
-
-template<typename T> void LandmarkDetection<T>::configure()
-{
- loadLabel();
-
- int ret = _inference->bind(_config->getBackendType(), _config->getTargetDeviceType());
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to bind a backend engine.");
-}
-
-template<typename T> void LandmarkDetection<T>::prepare()
-{
- int ret = _inference->configureInputMetaInfo(_config->getInputMetaMap());
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to configure input tensor info from meta file.");
-
- ret = _inference->configureOutputMetaInfo(_config->getOutputMetaMap());
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to configure output tensor info from meta file.");
-
- _inference->configureModelFiles("", _config->getModelFilePath(), "");
-
- // Request to load model files to a backend engine.
- ret = _inference->load();
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to load model files.");
-}
-
-template<typename T> shared_ptr<MetaInfo> LandmarkDetection<T>::getInputMetaInfo()
-{
- TensorBuffer &tensor_buffer = _inference->getInputTensorBuffer();
- IETensorBuffer &tensor_info_map = tensor_buffer.getIETensorBuffer();
-
- // TODO. consider using multiple tensors later.
- if (tensor_info_map.size() != 1)
- throw InvalidOperation("Input tensor count not invalid.");
-
- auto tensor_buffer_iter = tensor_info_map.begin();
-
- // Get the meta information corresponding to a given input tensor name.
- return _config->getInputMetaMap()[tensor_buffer_iter->first];
-}
-
-template<typename T>
-void LandmarkDetection<T>::preprocess(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo, vector<T> &inputVector)
-{
- LOGI("ENTER");
-
- PreprocessConfig config = { false,
- metaInfo->colorSpace,
- metaInfo->dataType,
- metaInfo->getChannel(),
- metaInfo->getWidth(),
- metaInfo->getHeight() };
-
- auto normalization = static_pointer_cast<DecodingNormal>(metaInfo->decodingTypeMap.at(DecodingType::NORMAL));
- if (normalization) {
- config.normalize = normalization->use;
- config.mean = normalization->mean;
- config.std = normalization->std;
- }
-
- auto quantization =
- static_pointer_cast<DecodingQuantization>(metaInfo->decodingTypeMap.at(DecodingType::QUANTIZATION));
- if (quantization) {
- config.quantize = quantization->use;
- config.scale = quantization->scale;
- config.zeropoint = quantization->zeropoint;
- }
-
- _preprocess.setConfig(config);
- _preprocess.run<T>(mv_src, inputVector);
-
- LOGI("LEAVE");
-}
-
-template<typename T> void LandmarkDetection<T>::inference(vector<vector<T> > &inputVectors)
-{
- LOGI("ENTER");
-
- int ret = _inference->run<T>(inputVectors);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to run inference");
-
- LOGI("LEAVE");
-}
-
-template<typename T> void LandmarkDetection<T>::perform(mv_source_h &mv_src)
-{
- shared_ptr<MetaInfo> metaInfo = getInputMetaInfo();
- vector<T> inputVector;
-
- preprocess(mv_src, metaInfo, inputVector);
-
- vector<vector<T> > inputVectors = { inputVector };
- inference(inputVectors);
-}
-
-template<typename T> LandmarkDetectionResult &LandmarkDetection<T>::getOutput()
-{
- if (_async_manager) {
- if (!_async_manager->isWorking())
- throw InvalidOperation("landmark detection has been already destroyed so invalid operation.");
-
- _current_result = _async_manager->pop();
- } else {
- // TODO. Check if inference request is completed or not here.
- // If not then throw an exception.
- _current_result = result();
- }
-
- return _current_result;
-}
-
-template<typename T> void LandmarkDetection<T>::performAsync(LandmarkDetectionInput &input)
-{
- if (!_async_manager) {
- _async_manager = make_unique<AsyncManager<T, LandmarkDetectionResult> >([this]() {
- AsyncInputQueue<T> inputQueue = _async_manager->popFromInput();
-
- LOGD("Poped input frame number = %ld", inputQueue.frame_number);
-
- inference(inputQueue.inputs);
-
- LandmarkDetectionResult &resultQueue = result();
-
- resultQueue.frame_number = inputQueue.frame_number;
- _async_manager->pushToOutput(resultQueue);
- });
- }
-
- shared_ptr<MetaInfo> metaInfo = getInputMetaInfo();
- vector<T> inputVector;
-
- preprocess(input.inference_src, metaInfo, inputVector);
-
- vector<vector<T> > inputVectors = { inputVector };
- _async_manager->push(inputVectors);
-}
-
-template<typename T> void LandmarkDetection<T>::getOutputNames(vector<string> &names)
-{
- TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
- IETensorBuffer &ie_tensor_buffer = tensor_buffer_obj.getIETensorBuffer();
-
- for (IETensorBuffer::iterator it = ie_tensor_buffer.begin(); it != ie_tensor_buffer.end(); it++)
- names.push_back(it->first);
-}
-
-template<typename T> void LandmarkDetection<T>::getOutputTensor(string target_name, vector<float> &tensor)
-{
- TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
-
- inference_engine_tensor_buffer *tensor_buffer = tensor_buffer_obj.getTensorBuffer(target_name);
- if (!tensor_buffer)
- throw InvalidOperation("Fail to get tensor buffer.");
-
- auto raw_buffer = static_cast<float *>(tensor_buffer->buffer);
-
- copy(&raw_buffer[0], &raw_buffer[tensor_buffer->size / sizeof(float)], back_inserter(tensor));
-}
-
-template class LandmarkDetection<float>;
-template class LandmarkDetection<unsigned char>;
-
-}
-}
\ No newline at end of file
#include "mv_private.h"
#include "mv_feature_key.h"
-#include "itask.h"
+#include "ITask.h"
#include "mv_facial_landmark_internal.h"
-#include "facial_landmark_adapter.h"
-#include "machine_learning_exception.h"
-#include "MachineLearningNative.h"
+#include "FacialLandmarkAdapter.h"
+#include "MvMlException.h"
+#include "native_capi.h"
#include "landmark_detection_type.h"
-#include "context.h"
+#include "Context.h"
#include <new>
#include <unistd.h>
#include "mv_private.h"
#include "mv_feature_key.h"
-#include "itask.h"
+#include "ITask.h"
#include "mv_pose_landmark_internal.h"
-#include "pose_landmark_adapter.h"
-#include "machine_learning_exception.h"
-#include "MachineLearningNative.h"
+#include "PoseLandmarkAdapter.h"
+#include "MvMlException.h"
+#include "native_capi.h"
#include "landmark_detection_type.h"
-#include "context.h"
+#include "Context.h"
#include <new>
#include <unistd.h>
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <string.h>
-#include <map>
-#include <algorithm>
-
-#include "machine_learning_exception.h"
-#include "mv_landmark_detection_config.h"
-#include "pld_cpm.h"
-#include "Postprocess.h"
-
-using namespace std;
-using namespace mediavision::inference;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T>
-PldCpm<T>::PldCpm(LandmarkDetectionTaskType task_type, std::shared_ptr<MachineLearningConfig> config)
- : LandmarkDetection<T>(task_type, config), _result()
-{}
-
-template<typename T> PldCpm<T>::~PldCpm()
-{}
-
-template<typename T> LandmarkDetectionResult &PldCpm<T>::result()
-{
- // Clear _result object because result() function can be called every time user wants
- // so make sure to clear existing result data before getting the data again.
- _result = LandmarkDetectionResult();
-
- vector<string> names;
-
- LandmarkDetection<T>::getOutputNames(names);
-
- auto scoreMetaInfo = _config->getOutputMetaMap().at(names[0]);
- auto decodingLandmark =
- static_pointer_cast<DecodingLandmark>(scoreMetaInfo->decodingTypeMap[DecodingType::LANDMARK]);
-
- if (decodingLandmark->decoding_type != LandmarkDecodingType::HEATMAP)
- throw InvalidOperation("decoding type not support.");
-
- if (decodingLandmark->coordinate_type != LandmarkCoordinateType::PIXEL)
- throw InvalidOperation("coordinate type not support.");
-
- if (decodingLandmark->landmark_type != LandmarkType::SINGLE_2D)
- throw InvalidOperation("landmark type not support.");
-
- auto heatMapWidth = scoreMetaInfo->dims[2];
- auto heatMapHeight = scoreMetaInfo->dims[1];
- auto heatMapChannel = scoreMetaInfo->dims[3];
- vector<float> score_tensor;
-
- _result.number_of_landmarks = heatMapChannel;
-
- LandmarkDetection<T>::getOutputTensor(names[0], score_tensor);
-
- auto ori_src_width = static_cast<double>(_preprocess.getImageWidth()[0]);
- auto ori_src_height = static_cast<double>(_preprocess.getImageHeight()[0]);
- auto width_ratio = ori_src_width / static_cast<double>(heatMapWidth);
- auto height_ratio = ori_src_height / static_cast<double>(heatMapHeight);
-
- for (auto c = 0; c < heatMapChannel; ++c) {
- float max_score = 0.0f;
- int max_x = 0;
- int max_y = 0;
-
- for (auto y = 0; y < heatMapHeight; ++y) {
- for (auto x = 0; x < heatMapWidth; ++x) {
- auto score = score_tensor[y * heatMapWidth * heatMapChannel + x * heatMapChannel + c];
- if (score < _config->getConfidenceThreshold())
- continue;
-
- if (max_score < score) {
- max_score = score;
- max_x = x;
- max_y = y;
- }
- }
- }
-
- if (max_score == 0.0f) {
- // If max_score is 0 then it means that all score values of current heatmap is 0 so
- // ignore the scores that do not meet the threshold.
- _result.number_of_landmarks--;
- continue;
- }
-
- _result.x_pos.push_back(static_cast<size_t>(static_cast<double>(max_x) * width_ratio));
- _result.y_pos.push_back(static_cast<size_t>(static_cast<double>(max_y) * height_ratio));
- _result.scores.push_back(max_score);
- }
-
- return _result;
-}
-
-template class PldCpm<unsigned char>;
-template class PldCpm<float>;
-
-}
-}
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "machine_learning_exception.h"
-#include "pose_landmark_adapter.h"
-#include "mv_landmark_detection_config.h"
-
-using namespace std;
-using namespace MediaVision::Common;
-using namespace mediavision::machine_learning;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-PoseLandmarkAdapter::PoseLandmarkAdapter()
-{
- _config = make_shared<MachineLearningConfig>();
- _config->parseConfigFile(_config_file_name);
-
- create(_config->getDefaultModelName());
-}
-
-PoseLandmarkAdapter::~PoseLandmarkAdapter()
-{
- _landmark_detection->preDestroy();
-}
-
-template<typename U> void PoseLandmarkAdapter::create(LandmarkDetectionTaskType task_type)
-{
- switch (task_type) {
- case LandmarkDetectionTaskType::PLD_CPM:
- _landmark_detection = make_unique<PldCpm<U> >(task_type, _config);
- break;
- default:
- throw InvalidOperation("Invalid landmark detection task type.");
- }
-}
-
-void PoseLandmarkAdapter::create(const string &model_name)
-{
- LandmarkDetectionTaskType task_type = convertToTaskType(model_name);
- _config->loadMetaFile(make_unique<LandmarkDetectionParser>(static_cast<int>(task_type)));
- mv_inference_data_type_e dataType = _config->getInputMetaMap().begin()->second->dataType;
-
- switch (dataType) {
- case MV_INFERENCE_DATA_UINT8:
- create<unsigned char>(task_type);
- break;
- case MV_INFERENCE_DATA_FLOAT32:
- create<float>(task_type);
- break;
- default:
- throw InvalidOperation("Invalid landmark detection data type.");
- }
-}
-
-LandmarkDetectionTaskType PoseLandmarkAdapter::convertToTaskType(string model_name)
-{
- if (model_name.empty())
- throw InvalidParameter("model name is empty.");
-
- transform(model_name.begin(), model_name.end(), model_name.begin(), ::toupper);
-
- if (model_name == "PLD_CPM")
- return LandmarkDetectionTaskType::PLD_CPM;
-
- throw InvalidParameter("Invalid pose landmark model name.");
-}
-
-void PoseLandmarkAdapter::setModelInfo(const string &model_file, const string &meta_file, const string &label_file,
- const string &model_name)
-{
- try {
- _config->setUserModel(model_file, meta_file, label_file);
- create(model_name);
- } catch (const BaseException &e) {
- LOGW("A given model name is invalid so default task type will be used.");
- }
-
- if (model_file.empty() && meta_file.empty()) {
- LOGW("Given model info is invalid so default model info will be used instead.");
- return;
- }
-}
-
-void PoseLandmarkAdapter::setEngineInfo(const string &engine_type, const string &device_type)
-{
- _landmark_detection->setEngineInfo(engine_type, device_type);
-}
-
-void PoseLandmarkAdapter::configure()
-{
- _landmark_detection->configure();
-}
-
-unsigned int PoseLandmarkAdapter::getNumberOfEngines()
-{
- return _landmark_detection->getNumberOfEngines();
-}
-
-const string &PoseLandmarkAdapter::getEngineType(unsigned int engine_index)
-{
- return _landmark_detection->getEngineType(engine_index);
-}
-
-unsigned int PoseLandmarkAdapter::getNumberOfDevices(const string &engine_type)
-{
- return _landmark_detection->getNumberOfDevices(engine_type);
-}
-
-const string &PoseLandmarkAdapter::getDeviceType(const string &engine_type, unsigned int device_index)
-{
- return _landmark_detection->getDeviceType(engine_type, device_index);
-}
-
-void PoseLandmarkAdapter::prepare()
-{
- _landmark_detection->prepare();
-}
-
-void PoseLandmarkAdapter::perform(InputBaseType &input)
-{
- _landmark_detection->perform(input.inference_src);
-}
-
-void PoseLandmarkAdapter::performAsync(InputBaseType &input)
-{
- _landmark_detection->performAsync(static_cast<LandmarkDetectionInput &>(input));
-}
-
-OutputBaseType &PoseLandmarkAdapter::getOutput()
-{
- return _landmark_detection->getOutput();
-}
-
-OutputBaseType &PoseLandmarkAdapter::getOutputCache()
-{
- throw InvalidOperation("Not support yet.");
-}
-
-}
-}
install(
DIRECTORY ${PROJECT_SOURCE_DIR}/include/ DESTINATION include/media
FILES_MATCHING
- PATTERN "iobject_detection.h"
+ PATTERN "IObjectDetection.h"
PATTERN "object_detection_type.h"
)
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __FACE_DETECTION_ADAPTER_H__
+#define __FACE_DETECTION_ADAPTER_H__
+
+#include <dlog.h>
+
+#include "EngineConfig.h"
+#include "ITask.h"
+#include "MvMlConfig.h"
+#include "MobilenetV1Ssd.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+class FaceDetectionAdapter : public mediavision::common::ITask
+{
+private:
+ std::unique_ptr<IObjectDetection> _object_detection;
+ std::shared_ptr<Config> _config;
+ const std::string _config_file_name = "face_detection.json";
+ const std::string _plugin_config_file_name = "face_detection_plugin.json";
+
+ void create(std::string model_name = "");
+ template<typename U> void create(ObjectDetectionTaskType task_type);
+ ObjectDetectionTaskType convertToTaskType(std::string model_name);
+
+public:
+ FaceDetectionAdapter();
+ ~FaceDetectionAdapter();
+
+ void setModelInfo(const std::string &model_file, const std::string &meta_file, const std::string &label_file,
+ const std::string &model_name) override;
+ void setEngineInfo(const std::string &engine_type, const std::string &device_type) override;
+ void configure() override;
+ unsigned int getNumberOfEngines() override;
+ const std::string &getEngineType(unsigned int engine_index) override;
+ unsigned int getNumberOfDevices(const std::string &engine_type) override;
+ const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
+ void prepare() override;
+ void perform(InputBaseType &input) override;
+ void performAsync(InputBaseType &input) override;
+ OutputBaseType &getOutput() override;
+ OutputBaseType &getOutputCache() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __IOBJECT_DETECTION_H__
+#define __IOBJECT_DETECTION_H__
+
+#include <mv_common.h>
+
+#include "object_detection_type.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+class IObjectDetection
+{
+public:
+ virtual ~IObjectDetection() {};
+
+ virtual void preDestroy() = 0;
+ virtual ObjectDetectionTaskType getTaskType() = 0;
+ virtual void setEngineInfo(std::string engine_type, std::string device_type) = 0;
+ virtual unsigned int getNumberOfEngines() = 0;
+ virtual const std::string &getEngineType(unsigned int engine_index) = 0;
+ virtual unsigned int getNumberOfDevices(const std::string &engine_type) = 0;
+ virtual const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) = 0;
+ virtual void configure() = 0;
+ virtual void prepare() = 0;
+ virtual void perform(mv_source_h &mv_src) = 0;
+ virtual void performAsync(ObjectDetectionInput &input) = 0;
+ virtual ObjectDetectionResult &getOutput() = 0;
+ virtual ObjectDetectionResult &getOutputCache() = 0;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __MOBILENET_V1_SSD_H__
+#define __MOBILENET_V1_SSD_H__
+
+#include <string>
+#include <memory>
+#include <mv_common.h>
+#include "mv_private.h"
+
+#include "ObjectDetection.h"
+#include <mv_inference_type.h>
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T> class MobilenetV1Ssd : public ObjectDetection<T>
+{
+ using ObjectDetection<T>::_config;
+ using ObjectDetection<T>::_preprocess;
+ using ObjectDetection<T>::_labels;
+
+private:
+ ObjectDetectionResult _result;
+
+public:
+ MobilenetV1Ssd(ObjectDetectionTaskType task_type, std::shared_ptr<Config> config);
+ ~MobilenetV1Ssd();
+
+ ObjectDetectionResult &result() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __MOBILENET_V2_SSD_H__
+#define __MOBILENET_V2_SSD_H__
+
+#include <string>
+#include <memory>
+#include <mv_common.h>
+#include "mv_private.h"
+
+#include "ObjectDetection.h"
+#include <mv_inference_type.h>
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T> class MobilenetV2Ssd : public ObjectDetection<T>
+{
+ using ObjectDetection<T>::_config;
+ using ObjectDetection<T>::_preprocess;
+ using ObjectDetection<T>::_labels;
+
+private:
+ ObjectDetectionResult _result;
+
+ void ApplyNms(std::vector<std::vector<Box> > &box_lists, BoxNmsMode mode, float threshold,
+ std::vector<Box> &box_vector);
+ Box decodeBox(const DecodingBox *decodingBox, std::vector<float> &bb_tensor, int idx, float score, int label,
+ int box_offset);
+ Box decodeBoxWithAnchor(const BoxAnchorParam *boxAnchorParam, Box &box, cv::Rect2f &anchor);
+
+public:
+ MobilenetV2Ssd(ObjectDetectionTaskType task_type, std::shared_ptr<Config> config);
+ ~MobilenetV2Ssd();
+
+ ObjectDetectionResult &result() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __OBJECT_DETECTION_H__
+#define __OBJECT_DETECTION_H__
+
+#include <queue>
+#include <thread>
+#include <mutex>
+#include <atomic>
+
+#include <mv_common.h>
+#include <mv_inference_type.h>
+#include "mv_private.h"
+
+#include "EngineConfig.h"
+#include "inference_engine_common_impl.h"
+#include "Inference.h"
+#include "object_detection_type.h"
+#include "MetaParser.h"
+#include "ObjectDetectionParser.h"
+#include "MvMlConfig.h"
+#include "MvMlPreprocess.h"
+#include "IObjectDetection.h"
+#include "AsyncManager.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T> class ObjectDetection : public IObjectDetection
+{
+private:
+ ObjectDetectionTaskType _task_type { ObjectDetectionTaskType::OBJECT_DETECTION_TASK_NONE };
+ std::unique_ptr<AsyncManager<T, ObjectDetectionResult> > _async_manager;
+ ObjectDetectionResult _current_result;
+
+ void loadLabel();
+ void getEngineList();
+ void getDeviceList(const std::string &engine_type);
+ void preprocess(mv_source_h &mv_src, std::shared_ptr<MetaInfo> metaInfo, std::vector<T> &inputVector);
+ std::shared_ptr<MetaInfo> getInputMetaInfo();
+
+protected:
+ std::unique_ptr<mediavision::inference::Inference> _inference;
+ std::shared_ptr<Config> _config;
+ std::vector<std::string> _labels;
+ std::vector<std::string> _valid_backends;
+ std::vector<std::string> _valid_devices;
+ Preprocess _preprocess;
+
+ void getOutputNames(std::vector<std::string> &names);
+ void getOutputTensor(std::string target_name, std::vector<float> &tensor);
+ void inference(std::vector<std::vector<T> > &inputVectors);
+ virtual ObjectDetectionResult &result() = 0;
+
+public:
+ explicit ObjectDetection(ObjectDetectionTaskType task_type, std::shared_ptr<Config> config);
+ virtual ~ObjectDetection() = default;
+
+ void preDestroy() override;
+ ObjectDetectionTaskType getTaskType() override;
+ void setUserModel(std::string model_file, std::string meta_file, std::string label_file);
+ void setEngineInfo(std::string engine_type_name, std::string device_type_name) override;
+ unsigned int getNumberOfEngines() override;
+ const std::string &getEngineType(unsigned int engine_index) override;
+ unsigned int getNumberOfDevices(const std::string &engine_type) override;
+ const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
+ void configure() override;
+ void prepare() override;
+ void perform(mv_source_h &mv_src) override;
+ void performAsync(ObjectDetectionInput &input) override;
+ ObjectDetectionResult &getOutput() override;
+ ObjectDetectionResult &getOutputCache() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __OBJECT_DETECTION_ADAPTER_H__
+#define __OBJECT_DETECTION_ADAPTER_H__
+
+#include <dlog.h>
+
+#include "EngineConfig.h"
+#include "ITask.h"
+#include "MvMlConfig.h"
+#include "MobilenetV1Ssd.h"
+#include "MobilenetV2Ssd.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+class ObjectDetectionAdapter : public mediavision::common::ITask
+{
+private:
+ std::unique_ptr<IObjectDetection> _object_detection;
+ std::shared_ptr<Config> _config;
+ const std::string _config_file_name = "object_detection.json";
+ const std::string _plugin_config_file_name = "object_detection_plugin.json";
+
+ void create(std::string model_name = "");
+ template<typename U> void create(ObjectDetectionTaskType task_type);
+ ObjectDetectionTaskType convertToTaskType(std::string model_name);
+
+public:
+ ObjectDetectionAdapter();
+ ~ObjectDetectionAdapter();
+
+ void setModelInfo(const std::string &model_file, const std::string &meta_file, const std::string &label_file,
+ const std::string &model_name) override;
+ void setEngineInfo(const std::string &engine_type, const std::string &device_type) override;
+ void configure() override;
+ unsigned int getNumberOfEngines() override;
+ const std::string &getEngineType(unsigned int engine_index) override;
+ unsigned int getNumberOfDevices(const std::string &engine_type) override;
+ const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
+ void prepare() override;
+ void perform(InputBaseType &input) override;
+ void performAsync(InputBaseType &input) override;
+ OutputBaseType &getOutput() override;
+ OutputBaseType &getOutputCache() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __OBJECT_DETECTION_EXTERNAL_H__
+#define __OBJECT_DETECTION_EXTERNAL_H__
+
+#include <mv_common.h>
+#include <mv_inference_type.h>
+
+#include <opencv2/core.hpp>
+#include <opencv2/imgproc.hpp>
+
+#include "object_detection_type.h"
+#include "IObjectDetection.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+class ObjectDetectionExternal : public IObjectDetection
+{
+private:
+ void *_plugin_handle {};
+ IObjectDetection *_object_detection_plugin {};
+ ObjectDetectionTaskType _task_type { ObjectDetectionTaskType::OBJECT_DETECTION_TASK_NONE };
+ ObjectDetectionResult _current_result;
+
+public:
+ ObjectDetectionExternal(ObjectDetectionTaskType task_type, const std::string &plugin_name);
+ virtual ~ObjectDetectionExternal();
+
+ void preDestroy() override;
+ ObjectDetectionTaskType getTaskType() override;
+ void setEngineInfo(std::string engine_type, std::string device_type) override;
+ unsigned int getNumberOfEngines() override;
+ const std::string &getEngineType(unsigned int engine_index) override;
+ unsigned int getNumberOfDevices(const std::string &engine_type) override;
+ const std::string &getDeviceType(const std::string &engine_type, const unsigned int device_index) override;
+ void configure() override;
+ void prepare() override;
+ void perform(mv_source_h &mv_src) override;
+ void performAsync(ObjectDetectionInput &input) override;
+ ObjectDetectionResult &getOutput() override;
+ ObjectDetectionResult &getOutputCache() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __FACE_DETECTION_ADAPTER_H__
-#define __FACE_DETECTION_ADAPTER_H__
-
-#include <dlog.h>
-
-#include "EngineConfig.h"
-#include "itask.h"
-#include "machine_learning_config.h"
-#include "mobilenet_v1_ssd.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-class FaceDetectionAdapter : public mediavision::common::ITask
-{
-private:
- std::unique_ptr<IObjectDetection> _object_detection;
- std::shared_ptr<MachineLearningConfig> _config;
- const std::string _config_file_name = "face_detection.json";
- const std::string _plugin_config_file_name = "face_detection_plugin.json";
-
- void create(std::string model_name = "");
- template<typename U> void create(ObjectDetectionTaskType task_type);
- ObjectDetectionTaskType convertToTaskType(std::string model_name);
-
-public:
- FaceDetectionAdapter();
- ~FaceDetectionAdapter();
-
- void setModelInfo(const std::string &model_file, const std::string &meta_file, const std::string &label_file,
- const std::string &model_name) override;
- void setEngineInfo(const std::string &engine_type, const std::string &device_type) override;
- void configure() override;
- unsigned int getNumberOfEngines() override;
- const std::string &getEngineType(unsigned int engine_index) override;
- unsigned int getNumberOfDevices(const std::string &engine_type) override;
- const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
- void prepare() override;
- void perform(InputBaseType &input) override;
- void performAsync(InputBaseType &input) override;
- OutputBaseType &getOutput() override;
- OutputBaseType &getOutputCache() override;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __IOBJECT_DETECTION_H__
-#define __IOBJECT_DETECTION_H__
-
-#include <mv_common.h>
-
-#include "object_detection_type.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-class IObjectDetection
-{
-public:
- virtual ~IObjectDetection() {};
-
- virtual void preDestroy() = 0;
- virtual ObjectDetectionTaskType getTaskType() = 0;
- virtual void setEngineInfo(std::string engine_type, std::string device_type) = 0;
- virtual unsigned int getNumberOfEngines() = 0;
- virtual const std::string &getEngineType(unsigned int engine_index) = 0;
- virtual unsigned int getNumberOfDevices(const std::string &engine_type) = 0;
- virtual const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) = 0;
- virtual void configure() = 0;
- virtual void prepare() = 0;
- virtual void perform(mv_source_h &mv_src) = 0;
- virtual void performAsync(ObjectDetectionInput &input) = 0;
- virtual ObjectDetectionResult &getOutput() = 0;
- virtual ObjectDetectionResult &getOutputCache() = 0;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __MOBILENET_V1_SSD_H__
-#define __MOBILENET_V1_SSD_H__
-
-#include <string>
-#include <memory>
-#include <mv_common.h>
-#include "mv_private.h"
-
-#include "object_detection.h"
-#include <mv_inference_type.h>
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T> class MobilenetV1Ssd : public ObjectDetection<T>
-{
- using ObjectDetection<T>::_config;
- using ObjectDetection<T>::_preprocess;
- using ObjectDetection<T>::_labels;
-
-private:
- ObjectDetectionResult _result;
-
-public:
- MobilenetV1Ssd(ObjectDetectionTaskType task_type, std::shared_ptr<MachineLearningConfig> config);
- ~MobilenetV1Ssd();
-
- ObjectDetectionResult &result() override;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __MOBILENET_V2_SSD_H__
-#define __MOBILENET_V2_SSD_H__
-
-#include <string>
-#include <memory>
-#include <mv_common.h>
-#include "mv_private.h"
-
-#include "object_detection.h"
-#include <mv_inference_type.h>
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T> class MobilenetV2Ssd : public ObjectDetection<T>
-{
- using ObjectDetection<T>::_config;
- using ObjectDetection<T>::_preprocess;
- using ObjectDetection<T>::_labels;
-
-private:
- ObjectDetectionResult _result;
-
- void ApplyNms(std::vector<std::vector<Box> > &box_lists, BoxNmsMode mode, float threshold,
- std::vector<Box> &box_vector);
- Box decodeBox(const DecodingBox *decodingBox, std::vector<float> &bb_tensor, int idx, float score, int label,
- int box_offset);
- Box decodeBoxWithAnchor(const BoxAnchorParam *boxAnchorParam, Box &box, cv::Rect2f &anchor);
-
-public:
- MobilenetV2Ssd(ObjectDetectionTaskType task_type, std::shared_ptr<MachineLearningConfig> config);
- ~MobilenetV2Ssd();
-
- ObjectDetectionResult &result() override;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __OBJECT_DETECTION_H__
-#define __OBJECT_DETECTION_H__
-
-#include <queue>
-#include <thread>
-#include <mutex>
-#include <atomic>
-
-#include <mv_common.h>
-#include <mv_inference_type.h>
-#include "mv_private.h"
-
-#include "EngineConfig.h"
-#include "inference_engine_common_impl.h"
-#include "Inference.h"
-#include "object_detection_type.h"
-#include "MetaParser.h"
-#include "ObjectDetectionParser.h"
-#include "machine_learning_config.h"
-#include "machine_learning_preprocess.h"
-#include "iobject_detection.h"
-#include "async_manager.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T> class ObjectDetection : public IObjectDetection
-{
-private:
- ObjectDetectionTaskType _task_type { ObjectDetectionTaskType::OBJECT_DETECTION_TASK_NONE };
- std::unique_ptr<AsyncManager<T, ObjectDetectionResult> > _async_manager;
- ObjectDetectionResult _current_result;
-
- void loadLabel();
- void getEngineList();
- void getDeviceList(const std::string &engine_type);
- void preprocess(mv_source_h &mv_src, std::shared_ptr<MetaInfo> metaInfo, std::vector<T> &inputVector);
- std::shared_ptr<MetaInfo> getInputMetaInfo();
-
-protected:
- std::unique_ptr<mediavision::inference::Inference> _inference;
- std::shared_ptr<MachineLearningConfig> _config;
- std::vector<std::string> _labels;
- std::vector<std::string> _valid_backends;
- std::vector<std::string> _valid_devices;
- Preprocess _preprocess;
-
- void getOutputNames(std::vector<std::string> &names);
- void getOutputTensor(std::string target_name, std::vector<float> &tensor);
- void inference(std::vector<std::vector<T> > &inputVectors);
- virtual ObjectDetectionResult &result() = 0;
-
-public:
- explicit ObjectDetection(ObjectDetectionTaskType task_type, std::shared_ptr<MachineLearningConfig> config);
- virtual ~ObjectDetection() = default;
-
- void preDestroy() override;
- ObjectDetectionTaskType getTaskType() override;
- void setUserModel(std::string model_file, std::string meta_file, std::string label_file);
- void setEngineInfo(std::string engine_type_name, std::string device_type_name) override;
- unsigned int getNumberOfEngines() override;
- const std::string &getEngineType(unsigned int engine_index) override;
- unsigned int getNumberOfDevices(const std::string &engine_type) override;
- const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
- void configure() override;
- void prepare() override;
- void perform(mv_source_h &mv_src) override;
- void performAsync(ObjectDetectionInput &input) override;
- ObjectDetectionResult &getOutput() override;
- ObjectDetectionResult &getOutputCache() override;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __OBJECT_DETECTION_ADAPTER_H__
-#define __OBJECT_DETECTION_ADAPTER_H__
-
-#include <dlog.h>
-
-#include "EngineConfig.h"
-#include "itask.h"
-#include "machine_learning_config.h"
-#include "mobilenet_v1_ssd.h"
-#include "mobilenet_v2_ssd.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-class ObjectDetectionAdapter : public mediavision::common::ITask
-{
-private:
- std::unique_ptr<IObjectDetection> _object_detection;
- std::shared_ptr<MachineLearningConfig> _config;
- const std::string _config_file_name = "object_detection.json";
- const std::string _plugin_config_file_name = "object_detection_plugin.json";
-
- void create(std::string model_name = "");
- template<typename U> void create(ObjectDetectionTaskType task_type);
- ObjectDetectionTaskType convertToTaskType(std::string model_name);
-
-public:
- ObjectDetectionAdapter();
- ~ObjectDetectionAdapter();
-
- void setModelInfo(const std::string &model_file, const std::string &meta_file, const std::string &label_file,
- const std::string &model_name) override;
- void setEngineInfo(const std::string &engine_type, const std::string &device_type) override;
- void configure() override;
- unsigned int getNumberOfEngines() override;
- const std::string &getEngineType(unsigned int engine_index) override;
- unsigned int getNumberOfDevices(const std::string &engine_type) override;
- const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
- void prepare() override;
- void perform(InputBaseType &input) override;
- void performAsync(InputBaseType &input) override;
- OutputBaseType &getOutput() override;
- OutputBaseType &getOutputCache() override;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __OBJECT_DETECTION_EXTERNAL_H__
-#define __OBJECT_DETECTION_EXTERNAL_H__
-
-#include <mv_common.h>
-#include <mv_inference_type.h>
-
-#include <opencv2/core.hpp>
-#include <opencv2/imgproc.hpp>
-
-#include "object_detection_type.h"
-#include "iobject_detection.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-class ObjectDetectionExternal : public IObjectDetection
-{
-private:
- void *_plugin_handle {};
- IObjectDetection *_object_detection_plugin {};
- ObjectDetectionTaskType _task_type { ObjectDetectionTaskType::OBJECT_DETECTION_TASK_NONE };
- ObjectDetectionResult _current_result;
-
-public:
- ObjectDetectionExternal(ObjectDetectionTaskType task_type, const std::string &plugin_name);
- virtual ~ObjectDetectionExternal();
-
- void preDestroy() override;
- ObjectDetectionTaskType getTaskType() override;
- void setEngineInfo(std::string engine_type, std::string device_type) override;
- unsigned int getNumberOfEngines() override;
- const std::string &getEngineType(unsigned int engine_index) override;
- unsigned int getNumberOfDevices(const std::string &engine_type) override;
- const std::string &getDeviceType(const std::string &engine_type, const unsigned int device_index) override;
- void configure() override;
- void prepare() override;
- void perform(mv_source_h &mv_src) override;
- void performAsync(ObjectDetectionInput &input) override;
- ObjectDetectionResult &getOutput() override;
- ObjectDetectionResult &getOutputCache() override;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
#include <mv_common.h>
#include <mv_inference_type.h>
-#include "MachineLearningType.h"
+#include "mv_ml_types.h"
namespace mediavision
{
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "MvMlException.h"
+#include "FaceDetectionAdapter.h"
+#include "ObjectDetectionExternal.h"
+#include "mv_object_detection_config.h"
+
+using namespace std;
+using namespace MediaVision::Common;
+using namespace mediavision::machine_learning;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+FaceDetectionAdapter::FaceDetectionAdapter()
+{
+ _config = make_shared<Config>();
+
+ // If the model type needs external plugin then bypass to load the meta file and just create the external plugin.
+ // In this case, external plugin will use its own meta file approach regardless of Mediavision's one.
+ _config->parsePluginConfigFile(_plugin_config_file_name);
+ if (!_config->isPluginUsed())
+ _config->parseConfigFile(_config_file_name);
+
+ create(_config->getDefaultModelName());
+}
+
+FaceDetectionAdapter::~FaceDetectionAdapter()
+{
+ _object_detection->preDestroy();
+}
+
+template<typename U> void FaceDetectionAdapter::create(ObjectDetectionTaskType task_type)
+{
+ switch (task_type) {
+ case ObjectDetectionTaskType::FD_MOBILENET_V1_SSD:
+ _object_detection = make_unique<MobilenetV1Ssd<U> >(task_type, _config);
+ break;
+ default:
+ throw InvalidOperation("Invalid face detection task type.");
+ }
+ // TODO.
+}
+
+void FaceDetectionAdapter::create(string model_name)
+{
+ if (model_name.empty())
+ model_name = _config->getDefaultModelName();
+
+ auto task_type = convertToTaskType(model_name);
+
+ if (_config->isPluginUsed()) {
+ const auto &plugin_name = _config->getPluginFileName();
+
+ _object_detection = make_unique<ObjectDetectionExternal>(task_type, plugin_name.c_str());
+ return;
+ }
+
+ _config->loadMetaFile(make_unique<ObjectDetectionParser>(static_cast<int>(task_type)));
+ mv_inference_data_type_e dataType = _config->getInputMetaMap().begin()->second->dataType;
+
+ switch (dataType) {
+ case MV_INFERENCE_DATA_UINT8:
+ create<unsigned char>(task_type);
+ break;
+ case MV_INFERENCE_DATA_FLOAT32:
+ create<float>(task_type);
+ break;
+ default:
+ throw InvalidOperation("Invalid face detection data type.");
+ }
+}
+
+ObjectDetectionTaskType FaceDetectionAdapter::convertToTaskType(string model_name)
+{
+ if (model_name.empty())
+ throw InvalidParameter("model name is empty.");
+
+ transform(model_name.begin(), model_name.end(), model_name.begin(), ::toupper);
+
+ if (model_name == "FD_MOBILENET_V1_SSD")
+ return ObjectDetectionTaskType::FD_MOBILENET_V1_SSD;
+ if (model_name == "OD_TRIV2")
+ return ObjectDetectionTaskType::OD_TRIV2;
+ else if (model_name == "FD_TRIV2")
+ return ObjectDetectionTaskType::FD_TRIV2;
+ // TODO.
+
+ throw InvalidParameter("Invalid face detection model name.");
+}
+
+void FaceDetectionAdapter::setModelInfo(const string &model_file, const string &meta_file, const string &label_file,
+ const string &model_name)
+{
+ try {
+ _config->setUserModel(model_file, meta_file, label_file);
+ create(model_name);
+ } catch (const BaseException &e) {
+ LOGW("A given model name is invalid so default task type will be used.");
+ }
+
+ if (model_file.empty() && meta_file.empty()) {
+ LOGW("Given model info is invalid so default model info will be used instead.");
+ return;
+ }
+}
+
+void FaceDetectionAdapter::setEngineInfo(const string &engine_type, const string &device_type)
+{
+ _object_detection->setEngineInfo(string(engine_type), string(device_type));
+}
+
+void FaceDetectionAdapter::configure()
+{
+ _object_detection->configure();
+}
+
+unsigned int FaceDetectionAdapter::getNumberOfEngines()
+{
+ return _object_detection->getNumberOfEngines();
+}
+
+const string &FaceDetectionAdapter::getEngineType(unsigned int engine_index)
+{
+ return _object_detection->getEngineType(engine_index);
+}
+
+unsigned int FaceDetectionAdapter::getNumberOfDevices(const string &engine_type)
+{
+ return _object_detection->getNumberOfDevices(engine_type);
+}
+
+const string &FaceDetectionAdapter::getDeviceType(const string &engine_type, unsigned int device_index)
+{
+ return _object_detection->getDeviceType(engine_type, device_index);
+}
+
+void FaceDetectionAdapter::prepare()
+{
+ _object_detection->prepare();
+}
+
+void FaceDetectionAdapter::perform(InputBaseType &input)
+{
+ _object_detection->perform(input.inference_src);
+}
+
+void FaceDetectionAdapter::performAsync(InputBaseType &input)
+{
+ _object_detection->performAsync(static_cast<ObjectDetectionInput &>(input));
+}
+
+OutputBaseType &FaceDetectionAdapter::getOutput()
+{
+ return _object_detection->getOutput();
+}
+
+OutputBaseType &FaceDetectionAdapter::getOutputCache()
+{
+ return _object_detection->getOutputCache();
+}
+
+}
+}
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <map>
+#include <algorithm>
+
+#include "MvMlException.h"
+#include "mv_object_detection_config.h"
+#include "MobilenetV1Ssd.h"
+#include "Postprocess.h"
+
+using namespace std;
+using namespace mediavision::inference;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T>
+MobilenetV1Ssd<T>::MobilenetV1Ssd(ObjectDetectionTaskType task_type, std::shared_ptr<Config> config)
+ : ObjectDetection<T>(task_type, config), _result()
+{}
+
+template<typename T> MobilenetV1Ssd<T>::~MobilenetV1Ssd()
+{}
+
+template<typename T> ObjectDetectionResult &MobilenetV1Ssd<T>::result()
+{
+ // Clear _result object because result() function can be called every time user wants
+ // so make sure to clear existing result data before getting the data again.
+ _result = ObjectDetectionResult();
+
+ vector<string> names;
+
+ ObjectDetection<T>::getOutputNames(names);
+
+ vector<float> number_tensor;
+
+ // TFLite_Detection_PostProcess:3
+ ObjectDetection<T>::getOutputTensor(names[3], number_tensor);
+
+ vector<float> label_tensor;
+
+ // TFLite_Detection_PostProcess:1
+ ObjectDetection<T>::getOutputTensor(names[1], label_tensor);
+
+ vector<float> score_tensor;
+ map<float, unsigned int, std::greater<float> > sorted_score;
+
+ auto scoreMetaInfo = _config->getOutputMetaMap().at(names[2]);
+ auto decodingScore = static_pointer_cast<DecodingScore>(scoreMetaInfo->decodingTypeMap[DecodingType::SCORE]);
+
+ // TFLite_Detection_PostProcess:2
+ ObjectDetection<T>::getOutputTensor(names[2], score_tensor);
+ for (size_t idx = 0; idx < score_tensor.size(); ++idx) {
+ if (decodingScore->threshold > score_tensor[idx])
+ continue;
+
+ sorted_score[score_tensor[idx]] = idx;
+ }
+
+ auto boxMetaInfo = _config->getOutputMetaMap().at(names[0]);
+ auto decodingBox = static_pointer_cast<DecodingBox>(boxMetaInfo->decodingTypeMap[DecodingType::BOX]);
+ vector<float> box_tensor;
+
+ ObjectDetection<T>::getOutputTensor(names[0], box_tensor);
+
+ for (auto &score : sorted_score) {
+ _result.number_of_objects++;
+ // second is idx
+ _result.names.push_back(_labels[label_tensor[score.second]]);
+ _result.indices.push_back(_result.number_of_objects - 1);
+ _result.confidences.push_back(score.first);
+
+ vector<unsigned int> &order = decodingBox->order;
+
+ _result.left.push_back(
+ static_cast<int>(box_tensor[score.second * 4 + order[0]] * _preprocess.getImageWidth()[0]));
+ _result.top.push_back(
+ static_cast<int>(box_tensor[score.second * 4 + order[1]] * _preprocess.getImageHeight()[0]));
+ _result.right.push_back(
+ static_cast<int>(box_tensor[score.second * 4 + order[2]] * _preprocess.getImageWidth()[0]));
+ _result.bottom.push_back(
+ static_cast<int>(box_tensor[score.second * 4 + order[3]] * _preprocess.getImageHeight()[0]));
+
+ LOGI("idx = %d, name = %s, score = %f, %dx%d, %dx%d", score.second,
+ _result.names[_result.number_of_objects - 1].c_str(), _result.confidences[_result.number_of_objects - 1],
+ _result.left[_result.number_of_objects - 1], _result.top[_result.number_of_objects - 1],
+ _result.right[_result.number_of_objects - 1], _result.bottom[_result.number_of_objects - 1]);
+
+ if (decodingScore->topNumber == _result.number_of_objects)
+ break;
+ }
+
+ return _result;
+}
+
+template class MobilenetV1Ssd<float>;
+template class MobilenetV1Ssd<unsigned char>;
+
+}
+}
#include <string>
#include <queue>
#include <algorithm>
-#include "machine_learning_exception.h"
+#include "MvMlException.h"
#include "MobilenetV2AnchorParser.h"
-#include "common.h"
+#include "meta_common.h"
using namespace std;
using namespace mediavision::machine_learning::exception;
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <map>
+#include <algorithm>
+#include <iostream>
+
+#include "MvMlException.h"
+#include "mv_object_detection_config.h"
+#include "MobilenetV2Ssd.h"
+#include "Postprocess.h"
+
+using namespace std;
+using namespace mediavision::inference;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T>
+MobilenetV2Ssd<T>::MobilenetV2Ssd(ObjectDetectionTaskType task_type, std::shared_ptr<Config> config)
+ : ObjectDetection<T>(task_type, config), _result()
+{}
+
+template<typename T> MobilenetV2Ssd<T>::~MobilenetV2Ssd()
+{}
+
+static bool compareScore(Box box0, Box box1)
+{
+ return box0.score > box1.score;
+}
+
+static float calcIntersectionOverUnion(Box box0, Box box1)
+{
+ float area0 = box0.location.width * box0.location.height;
+ float area1 = box1.location.width * box1.location.height;
+
+ if (area0 <= 0.0f || area1 <= 0.0f)
+ return 0.0f;
+
+ float sx0 = box0.location.x - box0.location.width * 0.5f;
+ float sy0 = box0.location.y - box0.location.height * 0.5f;
+ float ex0 = box0.location.x + box0.location.width * 0.5f;
+ float ey0 = box0.location.y + box0.location.height * 0.5f;
+ float sx1 = box1.location.x - box1.location.width * 0.5f;
+ float sy1 = box1.location.y - box1.location.height * 0.5f;
+ float ex1 = box1.location.x + box1.location.width * 0.5f;
+ float ey1 = box1.location.y + box1.location.height * 0.5f;
+
+ float xmin0 = min(sx0, ex0);
+ float ymin0 = min(sy0, ey0);
+ float xmax0 = max(sx0, ex0);
+ float ymax0 = max(sy0, ey0);
+ float xmin1 = min(sx1, ex1);
+ float ymin1 = min(sy1, ey1);
+ float xmax1 = max(sx1, ex1);
+ float ymax1 = max(sy1, ey1);
+
+ float intersectXmin = max(xmin0, xmin1);
+ float intersectYmin = max(ymin0, ymin1);
+ float intersectXmax = min(xmax0, xmax1);
+ float intersectYmax = min(ymax0, ymax1);
+
+ float intersectArea = max((intersectYmax - intersectYmin), 0.0f) * max((intersectXmax - intersectXmin), 0.0f);
+
+ return intersectArea / (area0 + area1 - intersectArea);
+}
+
+template<typename T>
+void MobilenetV2Ssd<T>::ApplyNms(vector<vector<Box> > &box_lists, BoxNmsMode mode, float threshold,
+ vector<Box> &box_vector)
+{
+ LOGI("ENTER");
+
+ if (mode != BoxNmsMode::STANDARD) {
+ LOGI("Skip Nms");
+ return;
+ }
+
+ LOGI("threshold: %.3f", threshold);
+
+ bool isIgnore = false;
+ vector<Box> candidate_box_vec;
+
+ for (auto &box_list : box_lists) {
+ if (box_list.size() <= 0)
+ continue;
+
+ sort(box_list.begin(), box_list.end(), compareScore);
+ candidate_box_vec.clear();
+
+ for (auto &decoded_box : box_list) {
+ isIgnore = false;
+
+ for (auto candidate_box = candidate_box_vec.rbegin(); candidate_box != candidate_box_vec.rend();
+ ++candidate_box) {
+ // compare decoded_box with previous one
+ float iouValue = calcIntersectionOverUnion(decoded_box, (*candidate_box));
+
+ LOGI("iouValue: %.3f", iouValue);
+
+ if (iouValue >= threshold) {
+ isIgnore = true;
+ break;
+ }
+ }
+
+ if (!isIgnore)
+ candidate_box_vec.push_back(decoded_box);
+ }
+
+ if (candidate_box_vec.size() > 0)
+ box_vector.insert(box_vector.begin(), candidate_box_vec.begin(), candidate_box_vec.end());
+ }
+
+ LOGI("LEAVE");
+}
+
+template<typename T>
+Box MobilenetV2Ssd<T>::decodeBox(const DecodingBox *decodingBox, vector<float> &bb_tensor, int idx, float score,
+ int label, int box_offset)
+{
+ // assume type is (cx,cy,w,h)
+ // left or cx
+ float cx = bb_tensor[idx * box_offset + decodingBox->order[0]];
+ // top or cy
+ float cy = bb_tensor[idx * box_offset + decodingBox->order[1]];
+ // right or width
+ float cWidth = bb_tensor[idx * box_offset + decodingBox->order[2]];
+ // bottom or height
+ float cHeight = bb_tensor[idx * box_offset + decodingBox->order[3]];
+
+ LOGI("cx:%.2f, cy:%.2f, cW:%.2f, cH:%.2f", cx, cy, cWidth, cHeight);
+
+ Box box = { .index = label, .score = score, .location = cv::Rect2f(cx, cy, cWidth, cHeight) };
+
+ return box;
+}
+
+template<typename T>
+Box MobilenetV2Ssd<T>::decodeBoxWithAnchor(const BoxAnchorParam *boxAnchorParam, Box &box, cv::Rect2f &anchor)
+{
+ if (boxAnchorParam->isFixedAnchorSize) {
+ box.location.x += anchor.x;
+ box.location.y += anchor.y;
+ } else {
+ box.location.x = box.location.x / boxAnchorParam->xScale * anchor.width + anchor.x;
+ box.location.y = box.location.y / boxAnchorParam->yScale * anchor.height + anchor.y;
+ }
+
+ if (boxAnchorParam->isExponentialBoxScale) {
+ box.location.width = anchor.width * exp(box.location.width / boxAnchorParam->wScale);
+ box.location.height = anchor.height * exp(box.location.height / boxAnchorParam->hScale);
+ } else {
+ box.location.width = anchor.width * box.location.width / boxAnchorParam->wScale;
+ box.location.height = anchor.height * box.location.height / boxAnchorParam->hScale;
+ }
+
+ return box;
+}
+
+template<typename T> ObjectDetectionResult &MobilenetV2Ssd<T>::result()
+{
+ // Clear _result object because result() function can be called every time user wants
+ // so make sure to clear existing result data before getting the data again.
+ _result = ObjectDetectionResult();
+
+ vector<string> names;
+
+ ObjectDetection<T>::getOutputNames(names);
+
+ vector<float> score_tensor;
+
+ // raw_outputs/class_predictions
+ ObjectDetection<T>::getOutputTensor(names[1], score_tensor);
+
+ auto scoreMetaInfo = _config->getOutputMetaMap().at(names[1]);
+ auto decodingScore = static_pointer_cast<DecodingScore>(scoreMetaInfo->decodingTypeMap[DecodingType::SCORE]);
+
+ auto boxMetaInfo = _config->getOutputMetaMap().at(names[0]);
+ auto decodingBox = static_pointer_cast<DecodingBox>(boxMetaInfo->decodingTypeMap[DecodingType::BOX]);
+ auto anchorParam = static_pointer_cast<BoxAnchorParam>(decodingBox->decodingInfoMap[BoxDecodingType::SSD_ANCHOR]);
+ unsigned int number_of_objects = scoreMetaInfo->dims[2]; // Shape is 1 x 2034 x 91
+
+ vector<float> bb_tensor;
+
+ // raw_outputs/box_encodings
+ ObjectDetection<T>::getOutputTensor(names[0], bb_tensor);
+
+ vector<Box> box_vec;
+ vector<vector<Box> > box_list_vec;
+ int box_offset = boxMetaInfo->dims[2]; // Shape is 1 x 2034 x 4
+
+ for (unsigned int object_idx = 0; object_idx < number_of_objects; ++object_idx) {
+ int anchor_idx = -1;
+
+ box_vec.clear();
+
+ for (auto &anchor : anchorParam->anchorBox) {
+ anchor_idx++;
+
+ float score = score_tensor[anchor_idx * number_of_objects + object_idx];
+ if (score <= decodingScore->threshold)
+ continue;
+
+ Box box = decodeBox(decodingBox.get(), bb_tensor, anchor_idx, score, object_idx, box_offset);
+ box_vec.push_back(decodeBoxWithAnchor(anchorParam.get(), box, anchor));
+ }
+
+ box_list_vec.push_back(box_vec);
+ }
+
+ vector<Box> result_box_vec;
+
+ if (!box_list_vec.empty()) {
+ auto anchorNmsParam = static_pointer_cast<BoxNmsParam>(decodingBox->decodingInfoMap[BoxDecodingType::NMS]);
+ ApplyNms(box_list_vec, anchorNmsParam->mode, anchorNmsParam->iouThreshold, result_box_vec);
+ }
+
+ for (auto &box : result_box_vec) {
+ _result.number_of_objects++;
+ _result.names.push_back(_labels[box.index]);
+ _result.indices.push_back(_result.number_of_objects - 1);
+ _result.confidences.push_back(box.score);
+
+ auto src_width = static_cast<double>(_preprocess.getImageWidth()[0]);
+ auto src_height = static_cast<double>(_preprocess.getImageHeight()[0]);
+ auto half_width = box.location.x - box.location.width * 0.5f;
+ auto half_height = box.location.y - box.location.height * 0.5f;
+
+ _result.left.push_back(static_cast<int>(half_width * src_width));
+ _result.top.push_back(static_cast<int>(half_height * src_height));
+ _result.right.push_back(static_cast<int>(half_width * src_width) +
+ static_cast<int>(box.location.width * src_width));
+ _result.bottom.push_back(static_cast<int>(half_height * src_height) +
+ static_cast<int>(box.location.height * src_height));
+
+ LOGI("idx = %d, name = %s, score = %f, %dx%d, %dx%d", box.index,
+ _result.names[_result.number_of_objects - 1].c_str(), _result.confidences[_result.number_of_objects - 1],
+ _result.left[_result.number_of_objects - 1], _result.top[_result.number_of_objects - 1],
+ _result.right[_result.number_of_objects - 1], _result.bottom[_result.number_of_objects - 1]);
+
+ if (decodingScore->topNumber == _result.number_of_objects)
+ break;
+ }
+
+ return _result;
+}
+
+template class MobilenetV2Ssd<float>;
+template class MobilenetV2Ssd<unsigned char>;
+
+}
+}
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <fstream>
+#include <map>
+#include <memory>
+
+#include "MvMlException.h"
+#include "common.h"
+#include "mv_object_detection_config.h"
+#include "ObjectDetection.h"
+
+using namespace std;
+using namespace mediavision::inference;
+using namespace MediaVision::Common;
+using namespace mediavision::common;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T>
+ObjectDetection<T>::ObjectDetection(ObjectDetectionTaskType task_type, shared_ptr<Config> config)
+ : _task_type(task_type), _config(config)
+{
+ _inference = make_unique<Inference>();
+}
+
+template<typename T> void ObjectDetection<T>::preDestroy()
+{
+ if (!_async_manager)
+ return;
+
+ _async_manager->stop();
+}
+
+template<typename T> ObjectDetectionTaskType ObjectDetection<T>::getTaskType()
+{
+ return _task_type;
+}
+
+template<typename T> void ObjectDetection<T>::getEngineList()
+{
+ for (auto idx = MV_INFERENCE_BACKEND_NONE + 1; idx < MV_INFERENCE_BACKEND_MAX; ++idx) {
+ auto backend = _inference->getSupportedInferenceBackend(idx);
+ // TODO. we need to describe what inference engines are supported by each Task API,
+ // and based on it, below inference engine types should be checked
+ // if a given type is supported by this Task API later. As of now, tflite only.
+ if (backend.second == true && backend.first.compare("tflite") == 0)
+ _valid_backends.push_back(backend.first);
+ }
+}
+
+template<typename T> void ObjectDetection<T>::getDeviceList(const string &engine_type)
+{
+ // TODO. add device types available for a given engine type later.
+ // In default, cpu and gpu only.
+ _valid_devices.push_back("cpu");
+ _valid_devices.push_back("gpu");
+}
+
+template<typename T> void ObjectDetection<T>::setEngineInfo(std::string engine_type_name, std::string device_type_name)
+{
+ if (engine_type_name.empty() || device_type_name.empty())
+ throw InvalidParameter("Invalid engine info.");
+
+ transform(engine_type_name.begin(), engine_type_name.end(), engine_type_name.begin(), ::toupper);
+ transform(device_type_name.begin(), device_type_name.end(), device_type_name.begin(), ::toupper);
+
+ int engine_type = GetBackendType(engine_type_name);
+ int device_type = GetDeviceType(device_type_name);
+
+ if (engine_type == MEDIA_VISION_ERROR_INVALID_PARAMETER || device_type == MEDIA_VISION_ERROR_INVALID_PARAMETER)
+ throw InvalidParameter("backend or target device type not found.");
+
+ _config->setBackendType(engine_type);
+ _config->setTargetDeviceType(device_type);
+
+ LOGI("Engine type : %s => %d, Device type : %s => %d", engine_type_name.c_str(), engine_type,
+ device_type_name.c_str(), device_type);
+}
+
+template<typename T> unsigned int ObjectDetection<T>::getNumberOfEngines()
+{
+ if (!_valid_backends.empty()) {
+ return _valid_backends.size();
+ }
+
+ getEngineList();
+ return _valid_backends.size();
+}
+
+template<typename T> const string &ObjectDetection<T>::getEngineType(unsigned int engine_index)
+{
+ if (!_valid_backends.empty()) {
+ if (_valid_backends.size() <= engine_index)
+ throw InvalidParameter("Invalid engine index.");
+
+ return _valid_backends[engine_index];
+ }
+
+ getEngineList();
+
+ if (_valid_backends.size() <= engine_index)
+ throw InvalidParameter("Invalid engine index.");
+
+ return _valid_backends[engine_index];
+}
+
+template<typename T> unsigned int ObjectDetection<T>::getNumberOfDevices(const string &engine_type)
+{
+ if (!_valid_devices.empty()) {
+ return _valid_devices.size();
+ }
+
+ getDeviceList(engine_type);
+ return _valid_devices.size();
+}
+
+template<typename T>
+const string &ObjectDetection<T>::getDeviceType(const string &engine_type, unsigned int device_index)
+{
+ if (!_valid_devices.empty()) {
+ if (_valid_devices.size() <= device_index)
+ throw InvalidParameter("Invalid device index.");
+
+ return _valid_devices[device_index];
+ }
+
+ getDeviceList(engine_type);
+
+ if (_valid_devices.size() <= device_index)
+ throw InvalidParameter("Invalid device index.");
+
+ return _valid_devices[device_index];
+}
+
+template<typename T> void ObjectDetection<T>::loadLabel()
+{
+ if (_config->getLabelFilePath().empty())
+ return;
+
+ ifstream readFile;
+
+ _labels.clear();
+ readFile.open(_config->getLabelFilePath().c_str());
+
+ if (readFile.fail())
+ throw InvalidOperation("Fail to open " + _config->getLabelFilePath() + " file.");
+
+ string line;
+
+ while (getline(readFile, line))
+ _labels.push_back(line);
+
+ readFile.close();
+}
+
+template<typename T> void ObjectDetection<T>::configure()
+{
+ loadLabel();
+
+ int ret = _inference->bind(_config->getBackendType(), _config->getTargetDeviceType());
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to bind a backend engine.");
+}
+
+template<typename T> void ObjectDetection<T>::prepare()
+{
+ int ret = _inference->configureInputMetaInfo(_config->getInputMetaMap());
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to configure input tensor info from meta file.");
+
+ ret = _inference->configureOutputMetaInfo(_config->getOutputMetaMap());
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to configure output tensor info from meta file.");
+
+ _inference->configureModelFiles("", _config->getModelFilePath(), "");
+
+ // Request to load model files to a backend engine.
+ ret = _inference->load();
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to load model files.");
+}
+
+template<typename T> shared_ptr<MetaInfo> ObjectDetection<T>::getInputMetaInfo()
+{
+ TensorBuffer &tensor_buffer = _inference->getInputTensorBuffer();
+ IETensorBuffer &tensor_info_map = tensor_buffer.getIETensorBuffer();
+
+ // TODO. consider using multiple tensors later.
+ if (tensor_info_map.size() != 1)
+ throw InvalidOperation("Input tensor count not invalid.");
+
+ auto tensor_buffer_iter = tensor_info_map.begin();
+
+ // Get the meta information corresponding to a given input tensor name.
+ return _config->getInputMetaMap()[tensor_buffer_iter->first];
+}
+
+template<typename T>
+void ObjectDetection<T>::preprocess(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo, vector<T> &inputVector)
+{
+ LOGI("ENTER");
+
+ PreprocessConfig config = { false,
+ metaInfo->colorSpace,
+ metaInfo->dataType,
+ metaInfo->getChannel(),
+ metaInfo->getWidth(),
+ metaInfo->getHeight() };
+
+ auto normalization = static_pointer_cast<DecodingNormal>(metaInfo->decodingTypeMap.at(DecodingType::NORMAL));
+ if (normalization) {
+ config.normalize = normalization->use;
+ config.mean = normalization->mean;
+ config.std = normalization->std;
+ }
+
+ auto quantization =
+ static_pointer_cast<DecodingQuantization>(metaInfo->decodingTypeMap.at(DecodingType::QUANTIZATION));
+ if (quantization) {
+ config.quantize = quantization->use;
+ config.scale = quantization->scale;
+ config.zeropoint = quantization->zeropoint;
+ }
+
+ _preprocess.setConfig(config);
+ _preprocess.run<T>(mv_src, inputVector);
+
+ LOGI("LEAVE");
+}
+
+template<typename T> void ObjectDetection<T>::inference(vector<vector<T> > &inputVectors)
+{
+ LOGI("ENTER");
+
+ int ret = _inference->run<T>(inputVectors);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to run inference");
+
+ LOGI("LEAVE");
+}
+
+template<typename T> void ObjectDetection<T>::perform(mv_source_h &mv_src)
+{
+ shared_ptr<MetaInfo> metaInfo = getInputMetaInfo();
+ vector<T> inputVector;
+
+ preprocess(mv_src, metaInfo, inputVector);
+
+ vector<vector<T> > inputVectors = { inputVector };
+ inference(inputVectors);
+}
+
+template<typename T> void ObjectDetection<T>::performAsync(ObjectDetectionInput &input)
+{
+ if (!_async_manager) {
+ _async_manager = make_unique<AsyncManager<T, ObjectDetectionResult> >([this]() {
+ AsyncInputQueue<T> inputQueue = _async_manager->popFromInput();
+
+ inference(inputQueue.inputs);
+
+ ObjectDetectionResult &resultQueue = result();
+
+ resultQueue.frame_number = inputQueue.frame_number;
+ _async_manager->pushToOutput(resultQueue);
+ });
+ }
+
+ shared_ptr<MetaInfo> metaInfo = getInputMetaInfo();
+ vector<T> inputVector;
+
+ preprocess(input.inference_src, metaInfo, inputVector);
+
+ vector<vector<T> > inputVectors = { inputVector };
+ _async_manager->push(inputVectors);
+}
+
+template<typename T> ObjectDetectionResult &ObjectDetection<T>::getOutput()
+{
+ if (_async_manager) {
+ if (!_async_manager->isWorking())
+ throw InvalidOperation("Object detection has been already destroyed so invalid operation.");
+
+ _current_result = _async_manager->pop();
+ } else {
+ // TODO. Check if inference request is completed or not here.
+ // If not then throw an exception.
+ _current_result = result();
+ }
+
+ return _current_result;
+}
+
+template<typename T> ObjectDetectionResult &ObjectDetection<T>::getOutputCache()
+{
+ return _current_result;
+}
+
+template<typename T> void ObjectDetection<T>::getOutputNames(vector<string> &names)
+{
+ TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
+ IETensorBuffer &ie_tensor_buffer = tensor_buffer_obj.getIETensorBuffer();
+
+ for (IETensorBuffer::iterator it = ie_tensor_buffer.begin(); it != ie_tensor_buffer.end(); it++)
+ names.push_back(it->first);
+}
+
+template<typename T> void ObjectDetection<T>::getOutputTensor(string target_name, vector<float> &tensor)
+{
+ TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
+
+ inference_engine_tensor_buffer *tensor_buffer = tensor_buffer_obj.getTensorBuffer(target_name);
+ if (!tensor_buffer)
+ throw InvalidOperation("Fail to get tensor buffer.");
+
+ auto raw_buffer = static_cast<float *>(tensor_buffer->buffer);
+
+ copy(&raw_buffer[0], &raw_buffer[tensor_buffer->size / sizeof(float)], back_inserter(tensor));
+}
+
+template class ObjectDetection<float>;
+template class ObjectDetection<unsigned char>;
+
+}
+}
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "MvMlException.h"
+#include "ObjectDetectionAdapter.h"
+#include "ObjectDetectionExternal.h"
+#include "mv_object_detection_config.h"
+
+using namespace std;
+using namespace MediaVision::Common;
+using namespace mediavision::machine_learning;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+ObjectDetectionAdapter::ObjectDetectionAdapter()
+{
+ _config = make_shared<Config>();
+
+ // If the model type needs external plugin then bypass to load the meta file and just create the external plugin.
+ // In this case, external plugin will use its own meta file approach regardless of Mediavision's one.
+ _config->parsePluginConfigFile(_plugin_config_file_name);
+ if (!_config->isPluginUsed())
+ _config->parseConfigFile(_config_file_name);
+
+ create(_config->getDefaultModelName());
+}
+
+ObjectDetectionAdapter::~ObjectDetectionAdapter()
+{
+ _object_detection->preDestroy();
+}
+
+template<typename U> void ObjectDetectionAdapter::create(ObjectDetectionTaskType task_type)
+{
+ switch (task_type) {
+ case ObjectDetectionTaskType::MOBILENET_V1_SSD:
+ _object_detection = make_unique<MobilenetV1Ssd<U> >(task_type, _config);
+ break;
+ case ObjectDetectionTaskType::MOBILENET_V2_SSD:
+ _object_detection = make_unique<MobilenetV2Ssd<U> >(task_type, _config);
+ break;
+ default:
+ throw InvalidOperation("Invalid object detection task type.");
+ }
+ // TODO.
+}
+
+void ObjectDetectionAdapter::create(string model_name)
+{
+ if (model_name.empty())
+ model_name = _config->getDefaultModelName();
+
+ auto task_type = convertToTaskType(model_name);
+
+ if (_config->isPluginUsed()) {
+ const auto &plugin_name = _config->getPluginFileName();
+
+ _object_detection = make_unique<ObjectDetectionExternal>(task_type, plugin_name.c_str());
+ return;
+ }
+
+ _config->loadMetaFile(make_unique<ObjectDetectionParser>(static_cast<int>(task_type)));
+ mv_inference_data_type_e dataType = _config->getInputMetaMap().begin()->second->dataType;
+
+ switch (dataType) {
+ case MV_INFERENCE_DATA_UINT8:
+ create<unsigned char>(task_type);
+ break;
+ case MV_INFERENCE_DATA_FLOAT32:
+ create<float>(task_type);
+ break;
+ default:
+ throw InvalidOperation("Invalid object detection data type.");
+ }
+}
+
+ObjectDetectionTaskType ObjectDetectionAdapter::convertToTaskType(string model_name)
+{
+ if (model_name.empty())
+ throw InvalidParameter("model name is empty.");
+
+ transform(model_name.begin(), model_name.end(), model_name.begin(), ::toupper);
+
+ if (model_name == "OD_TRIV2")
+ return ObjectDetectionTaskType::OD_TRIV2;
+ else if (model_name == "FD_TRIV2")
+ return ObjectDetectionTaskType::FD_TRIV2;
+ else if (model_name == "MOBILENET_V1_SSD")
+ return ObjectDetectionTaskType::MOBILENET_V1_SSD;
+ else if (model_name == "MOBILENET_V2_SSD")
+ return ObjectDetectionTaskType::MOBILENET_V2_SSD;
+ // TODO.
+
+ throw InvalidParameter("Invalid object detection model name.");
+}
+
+void ObjectDetectionAdapter::setModelInfo(const string &model_file, const string &meta_file, const string &label_file,
+ const string &model_name)
+{
+ try {
+ _config->setUserModel(model_file, meta_file, label_file);
+ create(model_name);
+ } catch (const BaseException &e) {
+ LOGW("A given model name is invalid so default task type will be used.");
+ }
+
+ if (model_file.empty() && meta_file.empty()) {
+ LOGW("Given model info is invalid so default model info will be used instead.");
+ return;
+ }
+}
+
+void ObjectDetectionAdapter::setEngineInfo(const string &engine_type, const string &device_type)
+{
+ _object_detection->setEngineInfo(string(engine_type), string(device_type));
+}
+
+void ObjectDetectionAdapter::configure()
+{
+ _object_detection->configure();
+}
+
+unsigned int ObjectDetectionAdapter::getNumberOfEngines()
+{
+ return _object_detection->getNumberOfEngines();
+}
+
+const string &ObjectDetectionAdapter::getEngineType(unsigned int engine_index)
+{
+ return _object_detection->getEngineType(engine_index);
+}
+
+unsigned int ObjectDetectionAdapter::getNumberOfDevices(const string &engine_type)
+{
+ return _object_detection->getNumberOfDevices(engine_type);
+}
+
+const string &ObjectDetectionAdapter::getDeviceType(const string &engine_type, unsigned int device_index)
+{
+ return _object_detection->getDeviceType(engine_type, device_index);
+}
+
+void ObjectDetectionAdapter::prepare()
+{
+ _object_detection->prepare();
+}
+
+void ObjectDetectionAdapter::perform(InputBaseType &input)
+{
+ _object_detection->perform(input.inference_src);
+}
+
+OutputBaseType &ObjectDetectionAdapter::getOutput()
+{
+ return _object_detection->getOutput();
+}
+
+OutputBaseType &ObjectDetectionAdapter::getOutputCache()
+{
+ return _object_detection->getOutputCache();
+}
+
+void ObjectDetectionAdapter::performAsync(InputBaseType &input)
+{
+ _object_detection->performAsync(static_cast<ObjectDetectionInput &>(input));
+}
+
+}
+}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <dlfcn.h>
+#include <sys/stat.h>
+
+#include "mv_private.h"
+
+#include "MvMlException.h"
+#include "ObjectDetectionExternal.h"
+
+using namespace std;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+typedef IObjectDetection *create_t(ObjectDetectionTaskType task_type);
+typedef void destroy_t(IObjectDetection *);
+
+ObjectDetectionExternal::ObjectDetectionExternal(ObjectDetectionTaskType task_type, const string &plugin_name)
+ : _plugin_handle(), _object_detection_plugin()
+{
+ // Load external object detection library.
+ LOGI("lib: %s", plugin_name.c_str());
+ _plugin_handle = dlopen(plugin_name.c_str(), RTLD_NOW);
+
+ if (!_plugin_handle)
+ throw InvalidOperation("Fail to open plugin library.");
+
+ create_t *createPlugin = (create_t *) dlsym(_plugin_handle, "createPlugin");
+ if (createPlugin == NULL) {
+ dlclose(_plugin_handle);
+ createPlugin = NULL;
+ throw InvalidOperation("Fail to get symbol from plugin library.");
+ }
+
+ _object_detection_plugin = createPlugin(task_type);
+ if (_object_detection_plugin == NULL) {
+ dlclose(_plugin_handle);
+ _plugin_handle = NULL;
+ throw InvalidOperation("Fail to create plugin module.");
+ }
+}
+
+ObjectDetectionExternal::~ObjectDetectionExternal()
+{
+ if (_plugin_handle) {
+ destroy_t *destroyPlugin = (destroy_t *) dlsym(_plugin_handle, "destroyPlugin");
+
+ destroyPlugin(_object_detection_plugin);
+ dlclose(_plugin_handle);
+ _object_detection_plugin = nullptr;
+ _plugin_handle = nullptr;
+ }
+}
+
+void ObjectDetectionExternal::preDestroy()
+{
+ _object_detection_plugin->preDestroy();
+}
+
+ObjectDetectionTaskType ObjectDetectionExternal::getTaskType()
+{
+ return _object_detection_plugin->getTaskType();
+}
+
+void ObjectDetectionExternal::setEngineInfo(string engine_type, string device_type)
+{
+ _object_detection_plugin->setEngineInfo(engine_type, device_type);
+}
+
+unsigned int ObjectDetectionExternal::getNumberOfEngines()
+{
+ return _object_detection_plugin->getNumberOfEngines();
+}
+
+const string &ObjectDetectionExternal::getEngineType(unsigned int engine_index)
+{
+ return _object_detection_plugin->getEngineType(engine_index);
+}
+
+unsigned int ObjectDetectionExternal::getNumberOfDevices(const string &engine_type)
+{
+ return _object_detection_plugin->getNumberOfDevices(engine_type);
+}
+
+const string &ObjectDetectionExternal::getDeviceType(const string &engine_type, const unsigned int device_index)
+{
+ return _object_detection_plugin->getDeviceType(engine_type, device_index);
+}
+
+void ObjectDetectionExternal::configure()
+{
+ _object_detection_plugin->configure();
+}
+
+void ObjectDetectionExternal::prepare()
+{
+ _object_detection_plugin->prepare();
+}
+
+void ObjectDetectionExternal::perform(mv_source_h &mv_src)
+{
+ _object_detection_plugin->perform(mv_src);
+}
+
+void ObjectDetectionExternal::performAsync(ObjectDetectionInput &input)
+{
+ _object_detection_plugin->performAsync(input);
+}
+
+ObjectDetectionResult &ObjectDetectionExternal::getOutput()
+{
+ _current_result = _object_detection_plugin->getOutput();
+
+ return _current_result;
+}
+
+ObjectDetectionResult &ObjectDetectionExternal::getOutputCache()
+{
+ return _current_result;
+}
+
+}
+}
\ No newline at end of file
*/
#include <memory>
-#include "machine_learning_exception.h"
+#include "MvMlException.h"
#include "ObjectDetectionParser.h"
#include "object_detection_type.h"
#include "MobilenetV2AnchorParser.h"
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "machine_learning_exception.h"
-#include "face_detection_adapter.h"
-#include "object_detection_external.h"
-#include "mv_object_detection_config.h"
-
-using namespace std;
-using namespace MediaVision::Common;
-using namespace mediavision::machine_learning;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-FaceDetectionAdapter::FaceDetectionAdapter()
-{
- _config = make_shared<MachineLearningConfig>();
-
- // If the model type needs external plugin then bypass to load the meta file and just create the external plugin.
- // In this case, external plugin will use its own meta file approach regardless of Mediavision's one.
- _config->parsePluginConfigFile(_plugin_config_file_name);
- if (!_config->isPluginUsed())
- _config->parseConfigFile(_config_file_name);
-
- create(_config->getDefaultModelName());
-}
-
-FaceDetectionAdapter::~FaceDetectionAdapter()
-{
- _object_detection->preDestroy();
-}
-
-template<typename U> void FaceDetectionAdapter::create(ObjectDetectionTaskType task_type)
-{
- switch (task_type) {
- case ObjectDetectionTaskType::FD_MOBILENET_V1_SSD:
- _object_detection = make_unique<MobilenetV1Ssd<U> >(task_type, _config);
- break;
- default:
- throw InvalidOperation("Invalid face detection task type.");
- }
- // TODO.
-}
-
-void FaceDetectionAdapter::create(string model_name)
-{
- if (model_name.empty())
- model_name = _config->getDefaultModelName();
-
- auto task_type = convertToTaskType(model_name);
-
- if (_config->isPluginUsed()) {
- const auto &plugin_name = _config->getPluginFileName();
-
- _object_detection = make_unique<ObjectDetectionExternal>(task_type, plugin_name.c_str());
- return;
- }
-
- _config->loadMetaFile(make_unique<ObjectDetectionParser>(static_cast<int>(task_type)));
- mv_inference_data_type_e dataType = _config->getInputMetaMap().begin()->second->dataType;
-
- switch (dataType) {
- case MV_INFERENCE_DATA_UINT8:
- create<unsigned char>(task_type);
- break;
- case MV_INFERENCE_DATA_FLOAT32:
- create<float>(task_type);
- break;
- default:
- throw InvalidOperation("Invalid face detection data type.");
- }
-}
-
-ObjectDetectionTaskType FaceDetectionAdapter::convertToTaskType(string model_name)
-{
- if (model_name.empty())
- throw InvalidParameter("model name is empty.");
-
- transform(model_name.begin(), model_name.end(), model_name.begin(), ::toupper);
-
- if (model_name == "FD_MOBILENET_V1_SSD")
- return ObjectDetectionTaskType::FD_MOBILENET_V1_SSD;
- if (model_name == "OD_TRIV2")
- return ObjectDetectionTaskType::OD_TRIV2;
- else if (model_name == "FD_TRIV2")
- return ObjectDetectionTaskType::FD_TRIV2;
- // TODO.
-
- throw InvalidParameter("Invalid face detection model name.");
-}
-
-void FaceDetectionAdapter::setModelInfo(const string &model_file, const string &meta_file, const string &label_file,
- const string &model_name)
-{
- try {
- _config->setUserModel(model_file, meta_file, label_file);
- create(model_name);
- } catch (const BaseException &e) {
- LOGW("A given model name is invalid so default task type will be used.");
- }
-
- if (model_file.empty() && meta_file.empty()) {
- LOGW("Given model info is invalid so default model info will be used instead.");
- return;
- }
-}
-
-void FaceDetectionAdapter::setEngineInfo(const string &engine_type, const string &device_type)
-{
- _object_detection->setEngineInfo(string(engine_type), string(device_type));
-}
-
-void FaceDetectionAdapter::configure()
-{
- _object_detection->configure();
-}
-
-unsigned int FaceDetectionAdapter::getNumberOfEngines()
-{
- return _object_detection->getNumberOfEngines();
-}
-
-const string &FaceDetectionAdapter::getEngineType(unsigned int engine_index)
-{
- return _object_detection->getEngineType(engine_index);
-}
-
-unsigned int FaceDetectionAdapter::getNumberOfDevices(const string &engine_type)
-{
- return _object_detection->getNumberOfDevices(engine_type);
-}
-
-const string &FaceDetectionAdapter::getDeviceType(const string &engine_type, unsigned int device_index)
-{
- return _object_detection->getDeviceType(engine_type, device_index);
-}
-
-void FaceDetectionAdapter::prepare()
-{
- _object_detection->prepare();
-}
-
-void FaceDetectionAdapter::perform(InputBaseType &input)
-{
- _object_detection->perform(input.inference_src);
-}
-
-void FaceDetectionAdapter::performAsync(InputBaseType &input)
-{
- _object_detection->performAsync(static_cast<ObjectDetectionInput &>(input));
-}
-
-OutputBaseType &FaceDetectionAdapter::getOutput()
-{
- return _object_detection->getOutput();
-}
-
-OutputBaseType &FaceDetectionAdapter::getOutputCache()
-{
- return _object_detection->getOutputCache();
-}
-
-}
-}
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <string.h>
-#include <map>
-#include <algorithm>
-
-#include "machine_learning_exception.h"
-#include "mv_object_detection_config.h"
-#include "mobilenet_v1_ssd.h"
-#include "Postprocess.h"
-
-using namespace std;
-using namespace mediavision::inference;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T>
-MobilenetV1Ssd<T>::MobilenetV1Ssd(ObjectDetectionTaskType task_type, std::shared_ptr<MachineLearningConfig> config)
- : ObjectDetection<T>(task_type, config), _result()
-{}
-
-template<typename T> MobilenetV1Ssd<T>::~MobilenetV1Ssd()
-{}
-
-template<typename T> ObjectDetectionResult &MobilenetV1Ssd<T>::result()
-{
- // Clear _result object because result() function can be called every time user wants
- // so make sure to clear existing result data before getting the data again.
- _result = ObjectDetectionResult();
-
- vector<string> names;
-
- ObjectDetection<T>::getOutputNames(names);
-
- vector<float> number_tensor;
-
- // TFLite_Detection_PostProcess:3
- ObjectDetection<T>::getOutputTensor(names[3], number_tensor);
-
- vector<float> label_tensor;
-
- // TFLite_Detection_PostProcess:1
- ObjectDetection<T>::getOutputTensor(names[1], label_tensor);
-
- vector<float> score_tensor;
- map<float, unsigned int, std::greater<float> > sorted_score;
-
- auto scoreMetaInfo = _config->getOutputMetaMap().at(names[2]);
- auto decodingScore = static_pointer_cast<DecodingScore>(scoreMetaInfo->decodingTypeMap[DecodingType::SCORE]);
-
- // TFLite_Detection_PostProcess:2
- ObjectDetection<T>::getOutputTensor(names[2], score_tensor);
- for (size_t idx = 0; idx < score_tensor.size(); ++idx) {
- if (decodingScore->threshold > score_tensor[idx])
- continue;
-
- sorted_score[score_tensor[idx]] = idx;
- }
-
- auto boxMetaInfo = _config->getOutputMetaMap().at(names[0]);
- auto decodingBox = static_pointer_cast<DecodingBox>(boxMetaInfo->decodingTypeMap[DecodingType::BOX]);
- vector<float> box_tensor;
-
- ObjectDetection<T>::getOutputTensor(names[0], box_tensor);
-
- for (auto &score : sorted_score) {
- _result.number_of_objects++;
- // second is idx
- _result.names.push_back(_labels[label_tensor[score.second]]);
- _result.indices.push_back(_result.number_of_objects - 1);
- _result.confidences.push_back(score.first);
-
- vector<unsigned int> &order = decodingBox->order;
-
- _result.left.push_back(
- static_cast<int>(box_tensor[score.second * 4 + order[0]] * _preprocess.getImageWidth()[0]));
- _result.top.push_back(
- static_cast<int>(box_tensor[score.second * 4 + order[1]] * _preprocess.getImageHeight()[0]));
- _result.right.push_back(
- static_cast<int>(box_tensor[score.second * 4 + order[2]] * _preprocess.getImageWidth()[0]));
- _result.bottom.push_back(
- static_cast<int>(box_tensor[score.second * 4 + order[3]] * _preprocess.getImageHeight()[0]));
-
- LOGI("idx = %d, name = %s, score = %f, %dx%d, %dx%d", score.second,
- _result.names[_result.number_of_objects - 1].c_str(), _result.confidences[_result.number_of_objects - 1],
- _result.left[_result.number_of_objects - 1], _result.top[_result.number_of_objects - 1],
- _result.right[_result.number_of_objects - 1], _result.bottom[_result.number_of_objects - 1]);
-
- if (decodingScore->topNumber == _result.number_of_objects)
- break;
- }
-
- return _result;
-}
-
-template class MobilenetV1Ssd<float>;
-template class MobilenetV1Ssd<unsigned char>;
-
-}
-}
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <string.h>
-#include <map>
-#include <algorithm>
-#include <iostream>
-
-#include "machine_learning_exception.h"
-#include "mv_object_detection_config.h"
-#include "mobilenet_v2_ssd.h"
-#include "Postprocess.h"
-
-using namespace std;
-using namespace mediavision::inference;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T>
-MobilenetV2Ssd<T>::MobilenetV2Ssd(ObjectDetectionTaskType task_type, std::shared_ptr<MachineLearningConfig> config)
- : ObjectDetection<T>(task_type, config), _result()
-{}
-
-template<typename T> MobilenetV2Ssd<T>::~MobilenetV2Ssd()
-{}
-
-static bool compareScore(Box box0, Box box1)
-{
- return box0.score > box1.score;
-}
-
-static float calcIntersectionOverUnion(Box box0, Box box1)
-{
- float area0 = box0.location.width * box0.location.height;
- float area1 = box1.location.width * box1.location.height;
-
- if (area0 <= 0.0f || area1 <= 0.0f)
- return 0.0f;
-
- float sx0 = box0.location.x - box0.location.width * 0.5f;
- float sy0 = box0.location.y - box0.location.height * 0.5f;
- float ex0 = box0.location.x + box0.location.width * 0.5f;
- float ey0 = box0.location.y + box0.location.height * 0.5f;
- float sx1 = box1.location.x - box1.location.width * 0.5f;
- float sy1 = box1.location.y - box1.location.height * 0.5f;
- float ex1 = box1.location.x + box1.location.width * 0.5f;
- float ey1 = box1.location.y + box1.location.height * 0.5f;
-
- float xmin0 = min(sx0, ex0);
- float ymin0 = min(sy0, ey0);
- float xmax0 = max(sx0, ex0);
- float ymax0 = max(sy0, ey0);
- float xmin1 = min(sx1, ex1);
- float ymin1 = min(sy1, ey1);
- float xmax1 = max(sx1, ex1);
- float ymax1 = max(sy1, ey1);
-
- float intersectXmin = max(xmin0, xmin1);
- float intersectYmin = max(ymin0, ymin1);
- float intersectXmax = min(xmax0, xmax1);
- float intersectYmax = min(ymax0, ymax1);
-
- float intersectArea = max((intersectYmax - intersectYmin), 0.0f) * max((intersectXmax - intersectXmin), 0.0f);
-
- return intersectArea / (area0 + area1 - intersectArea);
-}
-
-template<typename T>
-void MobilenetV2Ssd<T>::ApplyNms(vector<vector<Box> > &box_lists, BoxNmsMode mode, float threshold,
- vector<Box> &box_vector)
-{
- LOGI("ENTER");
-
- if (mode != BoxNmsMode::STANDARD) {
- LOGI("Skip Nms");
- return;
- }
-
- LOGI("threshold: %.3f", threshold);
-
- bool isIgnore = false;
- vector<Box> candidate_box_vec;
-
- for (auto &box_list : box_lists) {
- if (box_list.size() <= 0)
- continue;
-
- sort(box_list.begin(), box_list.end(), compareScore);
- candidate_box_vec.clear();
-
- for (auto &decoded_box : box_list) {
- isIgnore = false;
-
- for (auto candidate_box = candidate_box_vec.rbegin(); candidate_box != candidate_box_vec.rend();
- ++candidate_box) {
- // compare decoded_box with previous one
- float iouValue = calcIntersectionOverUnion(decoded_box, (*candidate_box));
-
- LOGI("iouValue: %.3f", iouValue);
-
- if (iouValue >= threshold) {
- isIgnore = true;
- break;
- }
- }
-
- if (!isIgnore)
- candidate_box_vec.push_back(decoded_box);
- }
-
- if (candidate_box_vec.size() > 0)
- box_vector.insert(box_vector.begin(), candidate_box_vec.begin(), candidate_box_vec.end());
- }
-
- LOGI("LEAVE");
-}
-
-template<typename T>
-Box MobilenetV2Ssd<T>::decodeBox(const DecodingBox *decodingBox, vector<float> &bb_tensor, int idx, float score,
- int label, int box_offset)
-{
- // assume type is (cx,cy,w,h)
- // left or cx
- float cx = bb_tensor[idx * box_offset + decodingBox->order[0]];
- // top or cy
- float cy = bb_tensor[idx * box_offset + decodingBox->order[1]];
- // right or width
- float cWidth = bb_tensor[idx * box_offset + decodingBox->order[2]];
- // bottom or height
- float cHeight = bb_tensor[idx * box_offset + decodingBox->order[3]];
-
- LOGI("cx:%.2f, cy:%.2f, cW:%.2f, cH:%.2f", cx, cy, cWidth, cHeight);
-
- Box box = { .index = label, .score = score, .location = cv::Rect2f(cx, cy, cWidth, cHeight) };
-
- return box;
-}
-
-template<typename T>
-Box MobilenetV2Ssd<T>::decodeBoxWithAnchor(const BoxAnchorParam *boxAnchorParam, Box &box, cv::Rect2f &anchor)
-{
- if (boxAnchorParam->isFixedAnchorSize) {
- box.location.x += anchor.x;
- box.location.y += anchor.y;
- } else {
- box.location.x = box.location.x / boxAnchorParam->xScale * anchor.width + anchor.x;
- box.location.y = box.location.y / boxAnchorParam->yScale * anchor.height + anchor.y;
- }
-
- if (boxAnchorParam->isExponentialBoxScale) {
- box.location.width = anchor.width * exp(box.location.width / boxAnchorParam->wScale);
- box.location.height = anchor.height * exp(box.location.height / boxAnchorParam->hScale);
- } else {
- box.location.width = anchor.width * box.location.width / boxAnchorParam->wScale;
- box.location.height = anchor.height * box.location.height / boxAnchorParam->hScale;
- }
-
- return box;
-}
-
-template<typename T> ObjectDetectionResult &MobilenetV2Ssd<T>::result()
-{
- // Clear _result object because result() function can be called every time user wants
- // so make sure to clear existing result data before getting the data again.
- _result = ObjectDetectionResult();
-
- vector<string> names;
-
- ObjectDetection<T>::getOutputNames(names);
-
- vector<float> score_tensor;
-
- // raw_outputs/class_predictions
- ObjectDetection<T>::getOutputTensor(names[1], score_tensor);
-
- auto scoreMetaInfo = _config->getOutputMetaMap().at(names[1]);
- auto decodingScore = static_pointer_cast<DecodingScore>(scoreMetaInfo->decodingTypeMap[DecodingType::SCORE]);
-
- auto boxMetaInfo = _config->getOutputMetaMap().at(names[0]);
- auto decodingBox = static_pointer_cast<DecodingBox>(boxMetaInfo->decodingTypeMap[DecodingType::BOX]);
- auto anchorParam = static_pointer_cast<BoxAnchorParam>(decodingBox->decodingInfoMap[BoxDecodingType::SSD_ANCHOR]);
- unsigned int number_of_objects = scoreMetaInfo->dims[2]; // Shape is 1 x 2034 x 91
-
- vector<float> bb_tensor;
-
- // raw_outputs/box_encodings
- ObjectDetection<T>::getOutputTensor(names[0], bb_tensor);
-
- vector<Box> box_vec;
- vector<vector<Box> > box_list_vec;
- int box_offset = boxMetaInfo->dims[2]; // Shape is 1 x 2034 x 4
-
- for (unsigned int object_idx = 0; object_idx < number_of_objects; ++object_idx) {
- int anchor_idx = -1;
-
- box_vec.clear();
-
- for (auto &anchor : anchorParam->anchorBox) {
- anchor_idx++;
-
- float score = score_tensor[anchor_idx * number_of_objects + object_idx];
- if (score <= decodingScore->threshold)
- continue;
-
- Box box = decodeBox(decodingBox.get(), bb_tensor, anchor_idx, score, object_idx, box_offset);
- box_vec.push_back(decodeBoxWithAnchor(anchorParam.get(), box, anchor));
- }
-
- box_list_vec.push_back(box_vec);
- }
-
- vector<Box> result_box_vec;
-
- if (!box_list_vec.empty()) {
- auto anchorNmsParam = static_pointer_cast<BoxNmsParam>(decodingBox->decodingInfoMap[BoxDecodingType::NMS]);
- ApplyNms(box_list_vec, anchorNmsParam->mode, anchorNmsParam->iouThreshold, result_box_vec);
- }
-
- for (auto &box : result_box_vec) {
- _result.number_of_objects++;
- _result.names.push_back(_labels[box.index]);
- _result.indices.push_back(_result.number_of_objects - 1);
- _result.confidences.push_back(box.score);
-
- auto src_width = static_cast<double>(_preprocess.getImageWidth()[0]);
- auto src_height = static_cast<double>(_preprocess.getImageHeight()[0]);
- auto half_width = box.location.x - box.location.width * 0.5f;
- auto half_height = box.location.y - box.location.height * 0.5f;
-
- _result.left.push_back(static_cast<int>(half_width * src_width));
- _result.top.push_back(static_cast<int>(half_height * src_height));
- _result.right.push_back(static_cast<int>(half_width * src_width) +
- static_cast<int>(box.location.width * src_width));
- _result.bottom.push_back(static_cast<int>(half_height * src_height) +
- static_cast<int>(box.location.height * src_height));
-
- LOGI("idx = %d, name = %s, score = %f, %dx%d, %dx%d", box.index,
- _result.names[_result.number_of_objects - 1].c_str(), _result.confidences[_result.number_of_objects - 1],
- _result.left[_result.number_of_objects - 1], _result.top[_result.number_of_objects - 1],
- _result.right[_result.number_of_objects - 1], _result.bottom[_result.number_of_objects - 1]);
-
- if (decodingScore->topNumber == _result.number_of_objects)
- break;
- }
-
- return _result;
-}
-
-template class MobilenetV2Ssd<float>;
-template class MobilenetV2Ssd<unsigned char>;
-
-}
-}
#include "mv_private.h"
#include "mv_feature_key.h"
-#include "itask.h"
+#include "ITask.h"
#include "mv_face_detection_internal.h"
-#include "face_detection_adapter.h"
-#include "machine_learning_exception.h"
-#include "MachineLearningNative.h"
+#include "FaceDetectionAdapter.h"
+#include "MvMlException.h"
+#include "native_capi.h"
#include "object_detection_type.h"
-#include "context.h"
+#include "Context.h"
#include <new>
#include <unistd.h>
#include "mv_private.h"
#include "mv_feature_key.h"
-#include "itask.h"
+#include "ITask.h"
#include "mv_object_detection_internal.h"
-#include "object_detection_adapter.h"
-#include "machine_learning_exception.h"
-#include "MachineLearningNative.h"
+#include "ObjectDetectionAdapter.h"
+#include "MvMlException.h"
+#include "native_capi.h"
#include "object_detection_type.h"
-#include "context.h"
+#include "Context.h"
#include <new>
#include <unistd.h>
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <string.h>
-#include <fstream>
-#include <map>
-#include <memory>
-
-#include "machine_learning_exception.h"
-#include "mv_machine_learning_common.h"
-#include "mv_object_detection_config.h"
-#include "object_detection.h"
-
-using namespace std;
-using namespace mediavision::inference;
-using namespace MediaVision::Common;
-using namespace mediavision::common;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T>
-ObjectDetection<T>::ObjectDetection(ObjectDetectionTaskType task_type, shared_ptr<MachineLearningConfig> config)
- : _task_type(task_type), _config(config)
-{
- _inference = make_unique<Inference>();
-}
-
-template<typename T> void ObjectDetection<T>::preDestroy()
-{
- if (!_async_manager)
- return;
-
- _async_manager->stop();
-}
-
-template<typename T> ObjectDetectionTaskType ObjectDetection<T>::getTaskType()
-{
- return _task_type;
-}
-
-template<typename T> void ObjectDetection<T>::getEngineList()
-{
- for (auto idx = MV_INFERENCE_BACKEND_NONE + 1; idx < MV_INFERENCE_BACKEND_MAX; ++idx) {
- auto backend = _inference->getSupportedInferenceBackend(idx);
- // TODO. we need to describe what inference engines are supported by each Task API,
- // and based on it, below inference engine types should be checked
- // if a given type is supported by this Task API later. As of now, tflite only.
- if (backend.second == true && backend.first.compare("tflite") == 0)
- _valid_backends.push_back(backend.first);
- }
-}
-
-template<typename T> void ObjectDetection<T>::getDeviceList(const string &engine_type)
-{
- // TODO. add device types available for a given engine type later.
- // In default, cpu and gpu only.
- _valid_devices.push_back("cpu");
- _valid_devices.push_back("gpu");
-}
-
-template<typename T> void ObjectDetection<T>::setEngineInfo(std::string engine_type_name, std::string device_type_name)
-{
- if (engine_type_name.empty() || device_type_name.empty())
- throw InvalidParameter("Invalid engine info.");
-
- transform(engine_type_name.begin(), engine_type_name.end(), engine_type_name.begin(), ::toupper);
- transform(device_type_name.begin(), device_type_name.end(), device_type_name.begin(), ::toupper);
-
- int engine_type = GetBackendType(engine_type_name);
- int device_type = GetDeviceType(device_type_name);
-
- if (engine_type == MEDIA_VISION_ERROR_INVALID_PARAMETER || device_type == MEDIA_VISION_ERROR_INVALID_PARAMETER)
- throw InvalidParameter("backend or target device type not found.");
-
- _config->setBackendType(engine_type);
- _config->setTargetDeviceType(device_type);
-
- LOGI("Engine type : %s => %d, Device type : %s => %d", engine_type_name.c_str(), engine_type,
- device_type_name.c_str(), device_type);
-}
-
-template<typename T> unsigned int ObjectDetection<T>::getNumberOfEngines()
-{
- if (!_valid_backends.empty()) {
- return _valid_backends.size();
- }
-
- getEngineList();
- return _valid_backends.size();
-}
-
-template<typename T> const string &ObjectDetection<T>::getEngineType(unsigned int engine_index)
-{
- if (!_valid_backends.empty()) {
- if (_valid_backends.size() <= engine_index)
- throw InvalidParameter("Invalid engine index.");
-
- return _valid_backends[engine_index];
- }
-
- getEngineList();
-
- if (_valid_backends.size() <= engine_index)
- throw InvalidParameter("Invalid engine index.");
-
- return _valid_backends[engine_index];
-}
-
-template<typename T> unsigned int ObjectDetection<T>::getNumberOfDevices(const string &engine_type)
-{
- if (!_valid_devices.empty()) {
- return _valid_devices.size();
- }
-
- getDeviceList(engine_type);
- return _valid_devices.size();
-}
-
-template<typename T>
-const string &ObjectDetection<T>::getDeviceType(const string &engine_type, unsigned int device_index)
-{
- if (!_valid_devices.empty()) {
- if (_valid_devices.size() <= device_index)
- throw InvalidParameter("Invalid device index.");
-
- return _valid_devices[device_index];
- }
-
- getDeviceList(engine_type);
-
- if (_valid_devices.size() <= device_index)
- throw InvalidParameter("Invalid device index.");
-
- return _valid_devices[device_index];
-}
-
-template<typename T> void ObjectDetection<T>::loadLabel()
-{
- if (_config->getLabelFilePath().empty())
- return;
-
- ifstream readFile;
-
- _labels.clear();
- readFile.open(_config->getLabelFilePath().c_str());
-
- if (readFile.fail())
- throw InvalidOperation("Fail to open " + _config->getLabelFilePath() + " file.");
-
- string line;
-
- while (getline(readFile, line))
- _labels.push_back(line);
-
- readFile.close();
-}
-
-template<typename T> void ObjectDetection<T>::configure()
-{
- loadLabel();
-
- int ret = _inference->bind(_config->getBackendType(), _config->getTargetDeviceType());
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to bind a backend engine.");
-}
-
-template<typename T> void ObjectDetection<T>::prepare()
-{
- int ret = _inference->configureInputMetaInfo(_config->getInputMetaMap());
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to configure input tensor info from meta file.");
-
- ret = _inference->configureOutputMetaInfo(_config->getOutputMetaMap());
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to configure output tensor info from meta file.");
-
- _inference->configureModelFiles("", _config->getModelFilePath(), "");
-
- // Request to load model files to a backend engine.
- ret = _inference->load();
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to load model files.");
-}
-
-template<typename T> shared_ptr<MetaInfo> ObjectDetection<T>::getInputMetaInfo()
-{
- TensorBuffer &tensor_buffer = _inference->getInputTensorBuffer();
- IETensorBuffer &tensor_info_map = tensor_buffer.getIETensorBuffer();
-
- // TODO. consider using multiple tensors later.
- if (tensor_info_map.size() != 1)
- throw InvalidOperation("Input tensor count not invalid.");
-
- auto tensor_buffer_iter = tensor_info_map.begin();
-
- // Get the meta information corresponding to a given input tensor name.
- return _config->getInputMetaMap()[tensor_buffer_iter->first];
-}
-
-template<typename T>
-void ObjectDetection<T>::preprocess(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo, vector<T> &inputVector)
-{
- LOGI("ENTER");
-
- PreprocessConfig config = { false,
- metaInfo->colorSpace,
- metaInfo->dataType,
- metaInfo->getChannel(),
- metaInfo->getWidth(),
- metaInfo->getHeight() };
-
- auto normalization = static_pointer_cast<DecodingNormal>(metaInfo->decodingTypeMap.at(DecodingType::NORMAL));
- if (normalization) {
- config.normalize = normalization->use;
- config.mean = normalization->mean;
- config.std = normalization->std;
- }
-
- auto quantization =
- static_pointer_cast<DecodingQuantization>(metaInfo->decodingTypeMap.at(DecodingType::QUANTIZATION));
- if (quantization) {
- config.quantize = quantization->use;
- config.scale = quantization->scale;
- config.zeropoint = quantization->zeropoint;
- }
-
- _preprocess.setConfig(config);
- _preprocess.run<T>(mv_src, inputVector);
-
- LOGI("LEAVE");
-}
-
-template<typename T> void ObjectDetection<T>::inference(vector<vector<T> > &inputVectors)
-{
- LOGI("ENTER");
-
- int ret = _inference->run<T>(inputVectors);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to run inference");
-
- LOGI("LEAVE");
-}
-
-template<typename T> void ObjectDetection<T>::perform(mv_source_h &mv_src)
-{
- shared_ptr<MetaInfo> metaInfo = getInputMetaInfo();
- vector<T> inputVector;
-
- preprocess(mv_src, metaInfo, inputVector);
-
- vector<vector<T> > inputVectors = { inputVector };
- inference(inputVectors);
-}
-
-template<typename T> void ObjectDetection<T>::performAsync(ObjectDetectionInput &input)
-{
- if (!_async_manager) {
- _async_manager = make_unique<AsyncManager<T, ObjectDetectionResult> >([this]() {
- AsyncInputQueue<T> inputQueue = _async_manager->popFromInput();
-
- inference(inputQueue.inputs);
-
- ObjectDetectionResult &resultQueue = result();
-
- resultQueue.frame_number = inputQueue.frame_number;
- _async_manager->pushToOutput(resultQueue);
- });
- }
-
- shared_ptr<MetaInfo> metaInfo = getInputMetaInfo();
- vector<T> inputVector;
-
- preprocess(input.inference_src, metaInfo, inputVector);
-
- vector<vector<T> > inputVectors = { inputVector };
- _async_manager->push(inputVectors);
-}
-
-template<typename T> ObjectDetectionResult &ObjectDetection<T>::getOutput()
-{
- if (_async_manager) {
- if (!_async_manager->isWorking())
- throw InvalidOperation("Object detection has been already destroyed so invalid operation.");
-
- _current_result = _async_manager->pop();
- } else {
- // TODO. Check if inference request is completed or not here.
- // If not then throw an exception.
- _current_result = result();
- }
-
- return _current_result;
-}
-
-template<typename T> ObjectDetectionResult &ObjectDetection<T>::getOutputCache()
-{
- return _current_result;
-}
-
-template<typename T> void ObjectDetection<T>::getOutputNames(vector<string> &names)
-{
- TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
- IETensorBuffer &ie_tensor_buffer = tensor_buffer_obj.getIETensorBuffer();
-
- for (IETensorBuffer::iterator it = ie_tensor_buffer.begin(); it != ie_tensor_buffer.end(); it++)
- names.push_back(it->first);
-}
-
-template<typename T> void ObjectDetection<T>::getOutputTensor(string target_name, vector<float> &tensor)
-{
- TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
-
- inference_engine_tensor_buffer *tensor_buffer = tensor_buffer_obj.getTensorBuffer(target_name);
- if (!tensor_buffer)
- throw InvalidOperation("Fail to get tensor buffer.");
-
- auto raw_buffer = static_cast<float *>(tensor_buffer->buffer);
-
- copy(&raw_buffer[0], &raw_buffer[tensor_buffer->size / sizeof(float)], back_inserter(tensor));
-}
-
-template class ObjectDetection<float>;
-template class ObjectDetection<unsigned char>;
-
-}
-}
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "machine_learning_exception.h"
-#include "object_detection_adapter.h"
-#include "object_detection_external.h"
-#include "mv_object_detection_config.h"
-
-using namespace std;
-using namespace MediaVision::Common;
-using namespace mediavision::machine_learning;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-ObjectDetectionAdapter::ObjectDetectionAdapter()
-{
- _config = make_shared<MachineLearningConfig>();
-
- // If the model type needs external plugin then bypass to load the meta file and just create the external plugin.
- // In this case, external plugin will use its own meta file approach regardless of Mediavision's one.
- _config->parsePluginConfigFile(_plugin_config_file_name);
- if (!_config->isPluginUsed())
- _config->parseConfigFile(_config_file_name);
-
- create(_config->getDefaultModelName());
-}
-
-ObjectDetectionAdapter::~ObjectDetectionAdapter()
-{
- _object_detection->preDestroy();
-}
-
-template<typename U> void ObjectDetectionAdapter::create(ObjectDetectionTaskType task_type)
-{
- switch (task_type) {
- case ObjectDetectionTaskType::MOBILENET_V1_SSD:
- _object_detection = make_unique<MobilenetV1Ssd<U> >(task_type, _config);
- break;
- case ObjectDetectionTaskType::MOBILENET_V2_SSD:
- _object_detection = make_unique<MobilenetV2Ssd<U> >(task_type, _config);
- break;
- default:
- throw InvalidOperation("Invalid object detection task type.");
- }
- // TODO.
-}
-
-void ObjectDetectionAdapter::create(string model_name)
-{
- if (model_name.empty())
- model_name = _config->getDefaultModelName();
-
- auto task_type = convertToTaskType(model_name);
-
- if (_config->isPluginUsed()) {
- const auto &plugin_name = _config->getPluginFileName();
-
- _object_detection = make_unique<ObjectDetectionExternal>(task_type, plugin_name.c_str());
- return;
- }
-
- _config->loadMetaFile(make_unique<ObjectDetectionParser>(static_cast<int>(task_type)));
- mv_inference_data_type_e dataType = _config->getInputMetaMap().begin()->second->dataType;
-
- switch (dataType) {
- case MV_INFERENCE_DATA_UINT8:
- create<unsigned char>(task_type);
- break;
- case MV_INFERENCE_DATA_FLOAT32:
- create<float>(task_type);
- break;
- default:
- throw InvalidOperation("Invalid object detection data type.");
- }
-}
-
-ObjectDetectionTaskType ObjectDetectionAdapter::convertToTaskType(string model_name)
-{
- if (model_name.empty())
- throw InvalidParameter("model name is empty.");
-
- transform(model_name.begin(), model_name.end(), model_name.begin(), ::toupper);
-
- if (model_name == "OD_TRIV2")
- return ObjectDetectionTaskType::OD_TRIV2;
- else if (model_name == "FD_TRIV2")
- return ObjectDetectionTaskType::FD_TRIV2;
- else if (model_name == "MOBILENET_V1_SSD")
- return ObjectDetectionTaskType::MOBILENET_V1_SSD;
- else if (model_name == "MOBILENET_V2_SSD")
- return ObjectDetectionTaskType::MOBILENET_V2_SSD;
- // TODO.
-
- throw InvalidParameter("Invalid object detection model name.");
-}
-
-void ObjectDetectionAdapter::setModelInfo(const string &model_file, const string &meta_file, const string &label_file,
- const string &model_name)
-{
- try {
- _config->setUserModel(model_file, meta_file, label_file);
- create(model_name);
- } catch (const BaseException &e) {
- LOGW("A given model name is invalid so default task type will be used.");
- }
-
- if (model_file.empty() && meta_file.empty()) {
- LOGW("Given model info is invalid so default model info will be used instead.");
- return;
- }
-}
-
-void ObjectDetectionAdapter::setEngineInfo(const string &engine_type, const string &device_type)
-{
- _object_detection->setEngineInfo(string(engine_type), string(device_type));
-}
-
-void ObjectDetectionAdapter::configure()
-{
- _object_detection->configure();
-}
-
-unsigned int ObjectDetectionAdapter::getNumberOfEngines()
-{
- return _object_detection->getNumberOfEngines();
-}
-
-const string &ObjectDetectionAdapter::getEngineType(unsigned int engine_index)
-{
- return _object_detection->getEngineType(engine_index);
-}
-
-unsigned int ObjectDetectionAdapter::getNumberOfDevices(const string &engine_type)
-{
- return _object_detection->getNumberOfDevices(engine_type);
-}
-
-const string &ObjectDetectionAdapter::getDeviceType(const string &engine_type, unsigned int device_index)
-{
- return _object_detection->getDeviceType(engine_type, device_index);
-}
-
-void ObjectDetectionAdapter::prepare()
-{
- _object_detection->prepare();
-}
-
-void ObjectDetectionAdapter::perform(InputBaseType &input)
-{
- _object_detection->perform(input.inference_src);
-}
-
-OutputBaseType &ObjectDetectionAdapter::getOutput()
-{
- return _object_detection->getOutput();
-}
-
-OutputBaseType &ObjectDetectionAdapter::getOutputCache()
-{
- return _object_detection->getOutputCache();
-}
-
-void ObjectDetectionAdapter::performAsync(InputBaseType &input)
-{
- _object_detection->performAsync(static_cast<ObjectDetectionInput &>(input));
-}
-
-}
-}
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <dlfcn.h>
-#include <sys/stat.h>
-
-#include "mv_private.h"
-
-#include "machine_learning_exception.h"
-#include "object_detection_external.h"
-
-using namespace std;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-typedef IObjectDetection *create_t(ObjectDetectionTaskType task_type);
-typedef void destroy_t(IObjectDetection *);
-
-ObjectDetectionExternal::ObjectDetectionExternal(ObjectDetectionTaskType task_type, const string &plugin_name)
- : _plugin_handle(), _object_detection_plugin()
-{
- // Load external object detection library.
- LOGI("lib: %s", plugin_name.c_str());
- _plugin_handle = dlopen(plugin_name.c_str(), RTLD_NOW);
-
- if (!_plugin_handle)
- throw InvalidOperation("Fail to open plugin library.");
-
- create_t *createPlugin = (create_t *) dlsym(_plugin_handle, "createPlugin");
- if (createPlugin == NULL) {
- dlclose(_plugin_handle);
- createPlugin = NULL;
- throw InvalidOperation("Fail to get symbol from plugin library.");
- }
-
- _object_detection_plugin = createPlugin(task_type);
- if (_object_detection_plugin == NULL) {
- dlclose(_plugin_handle);
- _plugin_handle = NULL;
- throw InvalidOperation("Fail to create plugin module.");
- }
-}
-
-ObjectDetectionExternal::~ObjectDetectionExternal()
-{
- if (_plugin_handle) {
- destroy_t *destroyPlugin = (destroy_t *) dlsym(_plugin_handle, "destroyPlugin");
-
- destroyPlugin(_object_detection_plugin);
- dlclose(_plugin_handle);
- _object_detection_plugin = nullptr;
- _plugin_handle = nullptr;
- }
-}
-
-void ObjectDetectionExternal::preDestroy()
-{
- _object_detection_plugin->preDestroy();
-}
-
-ObjectDetectionTaskType ObjectDetectionExternal::getTaskType()
-{
- return _object_detection_plugin->getTaskType();
-}
-
-void ObjectDetectionExternal::setEngineInfo(string engine_type, string device_type)
-{
- _object_detection_plugin->setEngineInfo(engine_type, device_type);
-}
-
-unsigned int ObjectDetectionExternal::getNumberOfEngines()
-{
- return _object_detection_plugin->getNumberOfEngines();
-}
-
-const string &ObjectDetectionExternal::getEngineType(unsigned int engine_index)
-{
- return _object_detection_plugin->getEngineType(engine_index);
-}
-
-unsigned int ObjectDetectionExternal::getNumberOfDevices(const string &engine_type)
-{
- return _object_detection_plugin->getNumberOfDevices(engine_type);
-}
-
-const string &ObjectDetectionExternal::getDeviceType(const string &engine_type, const unsigned int device_index)
-{
- return _object_detection_plugin->getDeviceType(engine_type, device_index);
-}
-
-void ObjectDetectionExternal::configure()
-{
- _object_detection_plugin->configure();
-}
-
-void ObjectDetectionExternal::prepare()
-{
- _object_detection_plugin->prepare();
-}
-
-void ObjectDetectionExternal::perform(mv_source_h &mv_src)
-{
- _object_detection_plugin->perform(mv_src);
-}
-
-void ObjectDetectionExternal::performAsync(ObjectDetectionInput &input)
-{
- _object_detection_plugin->performAsync(input);
-}
-
-ObjectDetectionResult &ObjectDetectionExternal::getOutput()
-{
- _current_result = _object_detection_plugin->getOutput();
-
- return _current_result;
-}
-
-ObjectDetectionResult &ObjectDetectionExternal::getOutputCache()
-{
- return _current_result;
-}
-
-}
-}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __IOBJECT_DETECTION_3D_H__
+#define __IOBJECT_DETECTION_3D_H__
+
+#include <mv_common.h>
+
+#include "object_detection_3d_type.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+class IObjectDetection3d
+{
+public:
+ virtual ~IObjectDetection3d() {};
+
+ virtual ObjectDetection3dTaskType getTaskType() = 0;
+ virtual void setEngineInfo(std::string engine_type_name, std::string device_type_name) = 0;
+ virtual unsigned int getNumberOfEngines() = 0;
+ virtual const std::string &getEngineType(unsigned int engine_index) = 0;
+ virtual unsigned int getNumberOfDevices(const std::string &engine_type) = 0;
+ virtual const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) = 0;
+ virtual std::shared_ptr<MetaInfo> getInputMetaInfo() = 0;
+ virtual void configure() = 0;
+ virtual void prepare() = 0;
+ virtual void perform(mv_source_h &mv_src, std::shared_ptr<MetaInfo> metaInfo) = 0;
+ virtual ObjectDetection3dResult &result() = 0;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __OBJECT_DETECTION_3D_H__
+#define __OBJECT_DETECTION_3D_H__
+
+#include <mv_common.h>
+#include <mv_inference_type.h>
+#include "mv_private.h"
+
+#include "EngineConfig.h"
+#include "inference_engine_common_impl.h"
+#include "Inference.h"
+#include "object_detection_3d_type.h"
+#include "IObjectDetection3d.h"
+#include "MetaParser.h"
+#include "MvMlConfig.h"
+#include "ObjectDetection3dParser.h"
+#include "MvMlPreprocess.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T> class ObjectDetection3d : public IObjectDetection3d
+{
+private:
+ ObjectDetection3dTaskType _task_type;
+
+ void loadLabel();
+ void getEngineList();
+ void getDeviceList(const std::string &engine_type);
+
+protected:
+ std::unique_ptr<mediavision::inference::Inference> _inference;
+ std::shared_ptr<Config> _config;
+ std::vector<std::string> _labels;
+ std::vector<std::string> _valid_backends;
+ std::vector<std::string> _valid_devices;
+ Preprocess _preprocess;
+
+ void getOutputNames(std::vector<std::string> &names);
+ void getOutputTensor(std::string &target_name, std::vector<float> &tensor);
+ void preprocess(mv_source_h &mv_src, std::shared_ptr<MetaInfo> metaInfo, std::vector<T> &inputVector);
+ void inference(std::vector<std::vector<T> > &inputVectors);
+
+public:
+ ObjectDetection3d(ObjectDetection3dTaskType task_type, std::shared_ptr<Config> config);
+ virtual ~ObjectDetection3d() = default;
+
+ ObjectDetection3dTaskType getTaskType();
+ void setEngineInfo(std::string engine_type_name, std::string device_type_name);
+ unsigned int getNumberOfEngines();
+ const std::string &getEngineType(unsigned int engine_index);
+ unsigned int getNumberOfDevices(const std::string &engine_type);
+ const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index);
+ std::shared_ptr<MetaInfo> getInputMetaInfo();
+ void configure();
+ void prepare();
+ void perform(mv_source_h &mv_src, std::shared_ptr<MetaInfo> metaInfo);
+ virtual ObjectDetection3dResult &result() = 0;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __OBJECT_DETECTION_3D_ADAPTER_H__
+#define __OBJECT_DETECTION_3D_ADAPTER_H__
+
+#include <dlog.h>
+
+#include "EngineConfig.h"
+#include "ITask.h"
+#include "MvMlConfig.h"
+#include "Objectron.h"
+#include "IObjectDetection3d.h"
+
+namespace mediavision
+{
+namespace machine_learning
+{
+class ObjectDetection3dAdapter : public mediavision::common::ITask
+{
+private:
+ std::unique_ptr<IObjectDetection3d> _object_detection_3d;
+ std::shared_ptr<Config> _config;
+ const std::string _config_file_name = "object_detection_3d.json";
+
+ void create(const std::string &model_name);
+ template<typename U> void create(ObjectDetection3dTaskType task_type);
+ ObjectDetection3dTaskType convertToTaskType(std::string model_name);
+
+public:
+ ObjectDetection3dAdapter();
+ ~ObjectDetection3dAdapter();
+
+ void setModelInfo(const std::string &model_file, const std::string &meta_file, const std::string &label_file,
+ const std::string &model_name) override;
+ void setEngineInfo(const std::string &engine_type, const std::string &device_type) override;
+ unsigned int getNumberOfEngines() override;
+ const std::string &getEngineType(unsigned int engine_index) override;
+ unsigned int getNumberOfDevices(const std::string &engine_type) override;
+ const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
+ void configure() override;
+ void prepare() override;
+ void perform(InputBaseType &input) override;
+ void performAsync(InputBaseType &input) override;
+ OutputBaseType &getOutput() override;
+ OutputBaseType &getOutputCache() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __OBJECTRON_H__
+#define __OBJECTRON_H__
+
+#include <memory>
+#include <mv_common.h>
+#include "mv_private.h"
+#include "MvMlConfig.h"
+
+#include "ObjectDetection3d.h"
+#include <mv_inference_type.h>
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T> class Objectron : public ObjectDetection3d<T>
+{
+ using ObjectDetection3d<T>::_preprocess;
+ using ObjectDetection3d<T>::_inference;
+ using ObjectDetection3d<T>::_config;
+
+private:
+ ObjectDetection3dResult _result;
+
+public:
+ Objectron(ObjectDetection3dTaskType task_type, std::shared_ptr<Config> config);
+ ~Objectron();
+
+ ObjectDetection3dResult &result() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __IOBJECT_DETECTION_3D_H__
-#define __IOBJECT_DETECTION_3D_H__
-
-#include <mv_common.h>
-
-#include "object_detection_3d_type.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-class IObjectDetection3d
-{
-public:
- virtual ~IObjectDetection3d() {};
-
- virtual ObjectDetection3dTaskType getTaskType() = 0;
- virtual void setEngineInfo(std::string engine_type_name, std::string device_type_name) = 0;
- virtual unsigned int getNumberOfEngines() = 0;
- virtual const std::string &getEngineType(unsigned int engine_index) = 0;
- virtual unsigned int getNumberOfDevices(const std::string &engine_type) = 0;
- virtual const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) = 0;
- virtual std::shared_ptr<MetaInfo> getInputMetaInfo() = 0;
- virtual void configure() = 0;
- virtual void prepare() = 0;
- virtual void perform(mv_source_h &mv_src, std::shared_ptr<MetaInfo> metaInfo) = 0;
- virtual ObjectDetection3dResult &result() = 0;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __OBJECT_DETECTION_3D_H__
-#define __OBJECT_DETECTION_3D_H__
-
-#include <mv_common.h>
-#include <mv_inference_type.h>
-#include "mv_private.h"
-
-#include "EngineConfig.h"
-#include "inference_engine_common_impl.h"
-#include "Inference.h"
-#include "object_detection_3d_type.h"
-#include "iobject_detection_3d.h"
-#include "MetaParser.h"
-#include "machine_learning_config.h"
-#include "ObjectDetection3dParser.h"
-#include "machine_learning_preprocess.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T> class ObjectDetection3d : public IObjectDetection3d
-{
-private:
- ObjectDetection3dTaskType _task_type;
-
- void loadLabel();
- void getEngineList();
- void getDeviceList(const std::string &engine_type);
-
-protected:
- std::unique_ptr<mediavision::inference::Inference> _inference;
- std::shared_ptr<MachineLearningConfig> _config;
- std::vector<std::string> _labels;
- std::vector<std::string> _valid_backends;
- std::vector<std::string> _valid_devices;
- Preprocess _preprocess;
-
- void getOutputNames(std::vector<std::string> &names);
- void getOutputTensor(std::string &target_name, std::vector<float> &tensor);
- void preprocess(mv_source_h &mv_src, std::shared_ptr<MetaInfo> metaInfo, std::vector<T> &inputVector);
- void inference(std::vector<std::vector<T> > &inputVectors);
-
-public:
- ObjectDetection3d(ObjectDetection3dTaskType task_type, std::shared_ptr<MachineLearningConfig> config);
- virtual ~ObjectDetection3d() = default;
-
- ObjectDetection3dTaskType getTaskType();
- void setEngineInfo(std::string engine_type_name, std::string device_type_name);
- unsigned int getNumberOfEngines();
- const std::string &getEngineType(unsigned int engine_index);
- unsigned int getNumberOfDevices(const std::string &engine_type);
- const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index);
- std::shared_ptr<MetaInfo> getInputMetaInfo();
- void configure();
- void prepare();
- void perform(mv_source_h &mv_src, std::shared_ptr<MetaInfo> metaInfo);
- virtual ObjectDetection3dResult &result() = 0;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __OBJECT_DETECTION_3D_ADAPTER_H__
-#define __OBJECT_DETECTION_3D_ADAPTER_H__
-
-#include <dlog.h>
-
-#include "EngineConfig.h"
-#include "itask.h"
-#include "machine_learning_config.h"
-#include "objectron.h"
-#include "iobject_detection_3d.h"
-
-namespace mediavision
-{
-namespace machine_learning
-{
-class ObjectDetection3dAdapter : public mediavision::common::ITask
-{
-private:
- std::unique_ptr<IObjectDetection3d> _object_detection_3d;
- std::shared_ptr<MachineLearningConfig> _config;
- const std::string _config_file_name = "object_detection_3d.json";
-
- void create(const std::string &model_name);
- template<typename U> void create(ObjectDetection3dTaskType task_type);
- ObjectDetection3dTaskType convertToTaskType(std::string model_name);
-
-public:
- ObjectDetection3dAdapter();
- ~ObjectDetection3dAdapter();
-
- void setModelInfo(const std::string &model_file, const std::string &meta_file, const std::string &label_file,
- const std::string &model_name) override;
- void setEngineInfo(const std::string &engine_type, const std::string &device_type) override;
- unsigned int getNumberOfEngines() override;
- const std::string &getEngineType(unsigned int engine_index) override;
- unsigned int getNumberOfDevices(const std::string &engine_type) override;
- const std::string &getDeviceType(const std::string &engine_type, unsigned int device_index) override;
- void configure() override;
- void prepare() override;
- void perform(InputBaseType &input) override;
- void performAsync(InputBaseType &input) override;
- OutputBaseType &getOutput() override;
- OutputBaseType &getOutputCache() override;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
#include <mv_common.h>
#include <mv_inference_type.h>
-#include "MachineLearningType.h"
+#include "mv_ml_types.h"
namespace mediavision
{
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __OBJECTRON_H__
-#define __OBJECTRON_H__
-
-#include <memory>
-#include <mv_common.h>
-#include "mv_private.h"
-#include "machine_learning_config.h"
-
-#include "object_detection_3d.h"
-#include <mv_inference_type.h>
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T> class Objectron : public ObjectDetection3d<T>
-{
- using ObjectDetection3d<T>::_preprocess;
- using ObjectDetection3d<T>::_inference;
- using ObjectDetection3d<T>::_config;
-
-private:
- ObjectDetection3dResult _result;
-
-public:
- Objectron(ObjectDetection3dTaskType task_type, std::shared_ptr<MachineLearningConfig> config);
- ~Objectron();
-
- ObjectDetection3dResult &result() override;
-};
-
-} // machine_learning
-} // mediavision
-
-#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <fstream>
+#include <map>
+#include <memory>
+#include <algorithm>
+
+#include "MvMlException.h"
+#include "common.h"
+#include "ObjectDetection3d.h"
+
+using namespace std;
+using namespace mediavision::inference;
+using namespace MediaVision::Common;
+using namespace mediavision::common;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T>
+ObjectDetection3d<T>::ObjectDetection3d(ObjectDetection3dTaskType task_type, std::shared_ptr<Config> config)
+ : _task_type(task_type), _config(config)
+{
+ _inference = make_unique<Inference>();
+}
+
+template<typename T> ObjectDetection3dTaskType ObjectDetection3d<T>::getTaskType()
+{
+ return _task_type;
+}
+
+template<typename T> void ObjectDetection3d<T>::getEngineList()
+{
+ for (auto idx = MV_INFERENCE_BACKEND_NONE + 1; idx < MV_INFERENCE_BACKEND_MAX; ++idx) {
+ auto backend = _inference->getSupportedInferenceBackend(idx);
+ // TODO. we need to describe what inference engines are supported by each Task API,
+ // and based on it, below inference engine types should be checked
+ // if a given type is supported by this Task API later. As of now, tflite only.
+ if (backend.second == true && backend.first.compare("tflite") == 0)
+ _valid_backends.push_back(backend.first);
+ }
+}
+
+template<typename T> void ObjectDetection3d<T>::getDeviceList(const string &engine_type)
+{
+ // TODO. add device types available for a given engine type later.
+ // In default, cpu and gpu only.
+ _valid_devices.push_back("cpu");
+ _valid_devices.push_back("gpu");
+}
+
+template<typename T>
+void ObjectDetection3d<T>::setEngineInfo(std::string engine_type_name, std::string device_type_name)
+{
+ if (engine_type_name.empty() || device_type_name.empty())
+ throw InvalidParameter("Invalid engine info.");
+
+ transform(engine_type_name.begin(), engine_type_name.end(), engine_type_name.begin(), ::toupper);
+ transform(device_type_name.begin(), device_type_name.end(), device_type_name.begin(), ::toupper);
+
+ int engine_type = GetBackendType(engine_type_name);
+ int device_type = GetDeviceType(device_type_name);
+
+ if (engine_type == MEDIA_VISION_ERROR_INVALID_PARAMETER || device_type == MEDIA_VISION_ERROR_INVALID_PARAMETER)
+ throw InvalidParameter("backend or target device type not found.");
+
+ _config->setBackendType(engine_type);
+ _config->setTargetDeviceType(device_type);
+
+ LOGI("Engine type : %s => %d, Device type : %s => %d", engine_type_name.c_str(), engine_type,
+ device_type_name.c_str(), device_type);
+}
+
+template<typename T> unsigned int ObjectDetection3d<T>::getNumberOfEngines()
+{
+ if (!_valid_backends.empty()) {
+ return _valid_backends.size();
+ }
+
+ getEngineList();
+ return _valid_backends.size();
+}
+
+template<typename T> const string &ObjectDetection3d<T>::getEngineType(unsigned int engine_index)
+{
+ if (!_valid_backends.empty()) {
+ if (_valid_backends.size() <= engine_index)
+ throw InvalidParameter("Invalid engine index.");
+
+ return _valid_backends[engine_index];
+ }
+
+ getEngineList();
+
+ if (_valid_backends.size() <= engine_index)
+ throw InvalidParameter("Invalid engine index.");
+
+ return _valid_backends[engine_index];
+}
+
+template<typename T> unsigned int ObjectDetection3d<T>::getNumberOfDevices(const string &engine_type)
+{
+ if (!_valid_devices.empty()) {
+ return _valid_devices.size();
+ }
+
+ getDeviceList(engine_type);
+ return _valid_devices.size();
+}
+
+template<typename T>
+const string &ObjectDetection3d<T>::getDeviceType(const string &engine_type, const unsigned int device_index)
+{
+ if (!_valid_devices.empty()) {
+ if (_valid_devices.size() <= device_index)
+ throw InvalidParameter("Invalid device index.");
+
+ return _valid_devices[device_index];
+ }
+
+ getDeviceList(engine_type);
+
+ if (_valid_devices.size() <= device_index)
+ throw InvalidParameter("Invalid device index.");
+
+ return _valid_devices[device_index];
+}
+
+template<typename T> void ObjectDetection3d<T>::loadLabel()
+{
+ if (_config->getLabelFilePath().empty())
+ return;
+
+ ifstream readFile { _config->getLabelFilePath() };
+
+ _labels.clear();
+
+ if (readFile.fail())
+ throw InvalidOperation("Fail to open " + _config->getLabelFilePath() + " file.");
+
+ string line;
+
+ while (getline(readFile, line))
+ _labels.push_back(line);
+}
+
+template<typename T> void ObjectDetection3d<T>::configure()
+{
+ loadLabel();
+
+ int ret = _inference->bind(_config->getBackendType(), _config->getTargetDeviceType());
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to bind a backend engine.");
+}
+
+template<typename T> void ObjectDetection3d<T>::prepare()
+{
+ int ret = _inference->configureInputMetaInfo(_config->getInputMetaMap());
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to configure input tensor info from meta file.");
+
+ ret = _inference->configureOutputMetaInfo(_config->getOutputMetaMap());
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to configure output tensor info from meta file.");
+
+ _inference->configureModelFiles("", _config->getModelFilePath(), "");
+
+ // Request to load model files to a backend engine.
+ ret = _inference->load();
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to load model files.");
+}
+
+template<typename T> shared_ptr<MetaInfo> ObjectDetection3d<T>::getInputMetaInfo()
+{
+ TensorBuffer &tensor_buffer = _inference->getInputTensorBuffer();
+ IETensorBuffer &tensor_info_map = tensor_buffer.getIETensorBuffer();
+
+ // TODO. consider using multiple tensors later.
+ if (tensor_info_map.size() != 1)
+ throw InvalidOperation("Input tensor count not invalid.");
+
+ auto tensor_buffer_iter = tensor_info_map.begin();
+
+ // Get the meta information corresponding to a given input tensor name.
+ return _config->getInputMetaMap()[tensor_buffer_iter->first];
+}
+
+template<typename T>
+void ObjectDetection3d<T>::preprocess(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo, vector<T> &inputVector)
+{
+ LOGI("ENTER");
+
+ PreprocessConfig config = { false,
+ metaInfo->colorSpace,
+ metaInfo->dataType,
+ metaInfo->getChannel(),
+ metaInfo->getWidth(),
+ metaInfo->getHeight() };
+
+ auto normalization = static_pointer_cast<DecodingNormal>(metaInfo->decodingTypeMap.at(DecodingType::NORMAL));
+ if (normalization) {
+ config.normalize = normalization->use;
+ config.mean = normalization->mean;
+ config.std = normalization->std;
+ }
+
+ auto quantization =
+ static_pointer_cast<DecodingQuantization>(metaInfo->decodingTypeMap.at(DecodingType::QUANTIZATION));
+ if (quantization) {
+ config.quantize = quantization->use;
+ config.scale = quantization->scale;
+ config.zeropoint = quantization->zeropoint;
+ }
+
+ _preprocess.setConfig(config);
+ _preprocess.run<T>(mv_src, inputVector);
+
+ LOGI("LEAVE");
+}
+
+template<typename T> void ObjectDetection3d<T>::inference(vector<vector<T> > &inputVectors)
+{
+ LOGI("ENTER");
+
+ int ret = _inference->run<T>(inputVectors);
+ if (ret != MEDIA_VISION_ERROR_NONE)
+ throw InvalidOperation("Fail to run inference");
+
+ LOGI("LEAVE");
+}
+
+template<typename T> void ObjectDetection3d<T>::perform(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo)
+{
+ vector<T> inputVector;
+
+ preprocess(mv_src, metaInfo, inputVector);
+
+ vector<vector<T> > inputVectors = { inputVector };
+
+ inference(inputVectors);
+}
+
+template<typename T> void ObjectDetection3d<T>::getOutputNames(vector<string> &names)
+{
+ TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
+ IETensorBuffer &ie_tensor_buffer = tensor_buffer_obj.getIETensorBuffer();
+
+ for (IETensorBuffer::iterator it = ie_tensor_buffer.begin(); it != ie_tensor_buffer.end(); it++)
+ names.push_back(it->first);
+}
+
+template<typename T> void ObjectDetection3d<T>::getOutputTensor(string &target_name, vector<float> &tensor)
+{
+ LOGI("ENTER");
+
+ TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
+
+ inference_engine_tensor_buffer *tensor_buffer = tensor_buffer_obj.getTensorBuffer(target_name);
+ if (!tensor_buffer)
+ throw InvalidOperation("Fail to get tensor buffer.");
+
+ auto raw_buffer = static_cast<float *>(tensor_buffer->buffer);
+
+ copy(&raw_buffer[0], &raw_buffer[tensor_buffer->size / sizeof(float)], back_inserter(tensor));
+
+ LOGI("LEAVE");
+}
+
+template class ObjectDetection3d<float>;
+template class ObjectDetection3d<unsigned char>;
+
+}
+}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "MvMlException.h"
+#include "ObjectDetection3dAdapter.h"
+
+using namespace std;
+using namespace MediaVision::Common;
+using namespace mediavision::machine_learning;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+ObjectDetection3dAdapter::ObjectDetection3dAdapter()
+{
+ _config = make_shared<Config>();
+ _config->parseConfigFile(_config_file_name);
+
+ create(_config->getDefaultModelName());
+}
+
+ObjectDetection3dAdapter::~ObjectDetection3dAdapter()
+{}
+
+template<typename U> void ObjectDetection3dAdapter::create(ObjectDetection3dTaskType task_type)
+{
+ switch (task_type) {
+ case ObjectDetection3dTaskType::OBJECTRON:
+ _object_detection_3d = make_unique<Objectron<U> >(task_type, _config);
+ break;
+ default:
+ throw InvalidOperation("Invalid object detection 3d task type.");
+ }
+}
+
+void ObjectDetection3dAdapter::create(const string &model_name)
+{
+ ObjectDetection3dTaskType task_type = convertToTaskType(model_name);
+ _config->loadMetaFile(make_unique<ObjectDetection3dParser>(static_cast<int>(task_type)));
+ mv_inference_data_type_e dataType = _config->getInputMetaMap().begin()->second->dataType;
+
+ switch (dataType) {
+ case MV_INFERENCE_DATA_UINT8:
+ create<unsigned char>(task_type);
+ break;
+ case MV_INFERENCE_DATA_FLOAT32:
+ create<float>(task_type);
+ break;
+ default:
+ throw InvalidOperation("Invalid object detection 3d data type.");
+ }
+}
+
+ObjectDetection3dTaskType ObjectDetection3dAdapter::convertToTaskType(string model_name)
+{
+ if (model_name.empty())
+ throw InvalidParameter("model name is empty.");
+
+ transform(model_name.begin(), model_name.end(), model_name.begin(), ::toupper);
+
+ if (model_name == "OBJECTRON")
+ return ObjectDetection3dTaskType::OBJECTRON;
+
+ throw InvalidParameter("Invalid object detection 3d model name.");
+}
+
+void ObjectDetection3dAdapter::setModelInfo(const string &model_file, const string &meta_file, const string &label_file,
+ const string &model_name)
+{
+ try {
+ _config->setUserModel(model_file, meta_file, label_file);
+ create(model_name);
+ } catch (const BaseException &e) {
+ LOGW("A given model name is invalid so default task type will be used.");
+ }
+
+ if (model_file.empty() && meta_file.empty()) {
+ LOGW("Given model info is invalid so default model info will be used instead.");
+ return;
+ }
+}
+
+void ObjectDetection3dAdapter::setEngineInfo(const string &engine_type, const string &device_type)
+{
+ _object_detection_3d->setEngineInfo(string(engine_type), string(device_type));
+}
+
+void ObjectDetection3dAdapter::configure()
+{
+ _object_detection_3d->configure();
+}
+
+unsigned int ObjectDetection3dAdapter::getNumberOfEngines()
+{
+ return _object_detection_3d->getNumberOfEngines();
+}
+
+const string &ObjectDetection3dAdapter::getEngineType(unsigned int engine_index)
+{
+ return _object_detection_3d->getEngineType(engine_index);
+}
+
+unsigned int ObjectDetection3dAdapter::getNumberOfDevices(const string &engine_type)
+{
+ return _object_detection_3d->getNumberOfDevices(engine_type);
+}
+
+const string &ObjectDetection3dAdapter::getDeviceType(const string &engine_type, unsigned int device_index)
+{
+ return _object_detection_3d->getDeviceType(engine_type, device_index);
+}
+
+void ObjectDetection3dAdapter::prepare()
+{
+ _object_detection_3d->prepare();
+}
+
+void ObjectDetection3dAdapter::perform(InputBaseType &input)
+{
+ shared_ptr<MetaInfo> metaInfo = _object_detection_3d->getInputMetaInfo();
+ _object_detection_3d->perform(input.inference_src, metaInfo);
+}
+
+void ObjectDetection3dAdapter::performAsync(InputBaseType &input)
+{
+ throw InvalidOperation("Not support yet.");
+}
+
+OutputBaseType &ObjectDetection3dAdapter::getOutput()
+{
+ return _object_detection_3d->result();
+}
+
+OutputBaseType &ObjectDetection3dAdapter::getOutputCache()
+{
+ throw InvalidOperation("Not support yet.");
+}
+
+}
+}
\ No newline at end of file
*/
#include <iostream>
-#include "machine_learning_exception.h"
+#include "MvMlException.h"
#include "ObjectDetection3dParser.h"
using namespace std;
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <map>
+#include <algorithm>
+
+#include "MvMlException.h"
+#include "Objectron.h"
+#include "Postprocess.h"
+
+using namespace std;
+using namespace mediavision::inference;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T>
+Objectron<T>::Objectron(ObjectDetection3dTaskType task_type, std::shared_ptr<Config> config)
+ : ObjectDetection3d<T>(task_type, config), _result()
+{}
+
+template<typename T> Objectron<T>::~Objectron()
+{}
+
+template<typename T> ObjectDetection3dResult &Objectron<T>::result()
+{
+ vector<string> names;
+
+ ObjectDetection3d<T>::getOutputNames(names);
+
+ vector<float> keypoints;
+
+ ObjectDetection3d<T>::getOutputTensor(names[1], keypoints);
+
+ size_t output_size = keypoints.size();
+
+ Postprocess postprocess({ _preprocess.getImageWidth()[0], _preprocess.getImageHeight()[0],
+ _inference->getInputWidth(), _inference->getInputHeight() });
+
+ _result = ObjectDetection3dResult();
+
+ for (size_t idx = 0; idx < output_size; idx += 2) {
+ _result.x_vec.push_back(postprocess.getScaledX(keypoints[idx]));
+ _result.y_vec.push_back(postprocess.getScaledY(keypoints[idx + 1]));
+ }
+
+ _result.number_of_points = output_size / 2;
+
+ vector<float> probability_vec;
+
+ // names[0] is "Identity"
+ ObjectDetection3d<T>::getOutputTensor(names[0], probability_vec);
+
+ _result.probability = static_cast<unsigned int>(probability_vec[0] * 100);
+
+ try {
+ // names[1] is "Identity_1"
+ auto metaInfo = _config->getOutputMetaMap().at(names[1]);
+ auto decodingBox = static_pointer_cast<DecodingBox>(metaInfo->decodingTypeMap[DecodingType::BOX]);
+
+ for (size_t idx = 0; idx < decodingBox->edges.size(); idx += 2)
+ _result.edge_index_vec.push_back({ decodingBox->edges[idx], decodingBox->edges[idx + 1] });
+
+ _result.number_of_edges = decodingBox->edges.size();
+ } catch (const std::exception &e) {
+ throw InvalidOperation("Invalid meta info access.");
+ }
+
+ return _result;
+}
+
+template class Objectron<float>;
+template class Objectron<unsigned char>;
+
+}
+}
#include "mv_private.h"
#include "mv_feature_key.h"
-#include "itask.h"
+#include "ITask.h"
#include "mv_object_detection_3d_internal.h"
-#include "object_detection_3d_adapter.h"
-#include "machine_learning_exception.h"
-#include "MachineLearningNative.h"
+#include "ObjectDetection3dAdapter.h"
+#include "MvMlException.h"
+#include "native_capi.h"
#include "object_detection_3d_type.h"
-#include "context.h"
+#include "Context.h"
#include <new>
#include <unistd.h>
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <string.h>
-#include <fstream>
-#include <map>
-#include <memory>
-#include <algorithm>
-
-#include "machine_learning_exception.h"
-#include "mv_machine_learning_common.h"
-#include "object_detection_3d.h"
-
-using namespace std;
-using namespace mediavision::inference;
-using namespace MediaVision::Common;
-using namespace mediavision::common;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T>
-ObjectDetection3d<T>::ObjectDetection3d(ObjectDetection3dTaskType task_type,
- std::shared_ptr<MachineLearningConfig> config)
- : _task_type(task_type), _config(config)
-{
- _inference = make_unique<Inference>();
-}
-
-template<typename T> ObjectDetection3dTaskType ObjectDetection3d<T>::getTaskType()
-{
- return _task_type;
-}
-
-template<typename T> void ObjectDetection3d<T>::getEngineList()
-{
- for (auto idx = MV_INFERENCE_BACKEND_NONE + 1; idx < MV_INFERENCE_BACKEND_MAX; ++idx) {
- auto backend = _inference->getSupportedInferenceBackend(idx);
- // TODO. we need to describe what inference engines are supported by each Task API,
- // and based on it, below inference engine types should be checked
- // if a given type is supported by this Task API later. As of now, tflite only.
- if (backend.second == true && backend.first.compare("tflite") == 0)
- _valid_backends.push_back(backend.first);
- }
-}
-
-template<typename T> void ObjectDetection3d<T>::getDeviceList(const string &engine_type)
-{
- // TODO. add device types available for a given engine type later.
- // In default, cpu and gpu only.
- _valid_devices.push_back("cpu");
- _valid_devices.push_back("gpu");
-}
-
-template<typename T>
-void ObjectDetection3d<T>::setEngineInfo(std::string engine_type_name, std::string device_type_name)
-{
- if (engine_type_name.empty() || device_type_name.empty())
- throw InvalidParameter("Invalid engine info.");
-
- transform(engine_type_name.begin(), engine_type_name.end(), engine_type_name.begin(), ::toupper);
- transform(device_type_name.begin(), device_type_name.end(), device_type_name.begin(), ::toupper);
-
- int engine_type = GetBackendType(engine_type_name);
- int device_type = GetDeviceType(device_type_name);
-
- if (engine_type == MEDIA_VISION_ERROR_INVALID_PARAMETER || device_type == MEDIA_VISION_ERROR_INVALID_PARAMETER)
- throw InvalidParameter("backend or target device type not found.");
-
- _config->setBackendType(engine_type);
- _config->setTargetDeviceType(device_type);
-
- LOGI("Engine type : %s => %d, Device type : %s => %d", engine_type_name.c_str(), engine_type,
- device_type_name.c_str(), device_type);
-}
-
-template<typename T> unsigned int ObjectDetection3d<T>::getNumberOfEngines()
-{
- if (!_valid_backends.empty()) {
- return _valid_backends.size();
- }
-
- getEngineList();
- return _valid_backends.size();
-}
-
-template<typename T> const string &ObjectDetection3d<T>::getEngineType(unsigned int engine_index)
-{
- if (!_valid_backends.empty()) {
- if (_valid_backends.size() <= engine_index)
- throw InvalidParameter("Invalid engine index.");
-
- return _valid_backends[engine_index];
- }
-
- getEngineList();
-
- if (_valid_backends.size() <= engine_index)
- throw InvalidParameter("Invalid engine index.");
-
- return _valid_backends[engine_index];
-}
-
-template<typename T> unsigned int ObjectDetection3d<T>::getNumberOfDevices(const string &engine_type)
-{
- if (!_valid_devices.empty()) {
- return _valid_devices.size();
- }
-
- getDeviceList(engine_type);
- return _valid_devices.size();
-}
-
-template<typename T>
-const string &ObjectDetection3d<T>::getDeviceType(const string &engine_type, const unsigned int device_index)
-{
- if (!_valid_devices.empty()) {
- if (_valid_devices.size() <= device_index)
- throw InvalidParameter("Invalid device index.");
-
- return _valid_devices[device_index];
- }
-
- getDeviceList(engine_type);
-
- if (_valid_devices.size() <= device_index)
- throw InvalidParameter("Invalid device index.");
-
- return _valid_devices[device_index];
-}
-
-template<typename T> void ObjectDetection3d<T>::loadLabel()
-{
- if (_config->getLabelFilePath().empty())
- return;
-
- ifstream readFile { _config->getLabelFilePath() };
-
- _labels.clear();
-
- if (readFile.fail())
- throw InvalidOperation("Fail to open " + _config->getLabelFilePath() + " file.");
-
- string line;
-
- while (getline(readFile, line))
- _labels.push_back(line);
-}
-
-template<typename T> void ObjectDetection3d<T>::configure()
-{
- loadLabel();
-
- int ret = _inference->bind(_config->getBackendType(), _config->getTargetDeviceType());
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to bind a backend engine.");
-}
-
-template<typename T> void ObjectDetection3d<T>::prepare()
-{
- int ret = _inference->configureInputMetaInfo(_config->getInputMetaMap());
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to configure input tensor info from meta file.");
-
- ret = _inference->configureOutputMetaInfo(_config->getOutputMetaMap());
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to configure output tensor info from meta file.");
-
- _inference->configureModelFiles("", _config->getModelFilePath(), "");
-
- // Request to load model files to a backend engine.
- ret = _inference->load();
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to load model files.");
-}
-
-template<typename T> shared_ptr<MetaInfo> ObjectDetection3d<T>::getInputMetaInfo()
-{
- TensorBuffer &tensor_buffer = _inference->getInputTensorBuffer();
- IETensorBuffer &tensor_info_map = tensor_buffer.getIETensorBuffer();
-
- // TODO. consider using multiple tensors later.
- if (tensor_info_map.size() != 1)
- throw InvalidOperation("Input tensor count not invalid.");
-
- auto tensor_buffer_iter = tensor_info_map.begin();
-
- // Get the meta information corresponding to a given input tensor name.
- return _config->getInputMetaMap()[tensor_buffer_iter->first];
-}
-
-template<typename T>
-void ObjectDetection3d<T>::preprocess(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo, vector<T> &inputVector)
-{
- LOGI("ENTER");
-
- PreprocessConfig config = { false,
- metaInfo->colorSpace,
- metaInfo->dataType,
- metaInfo->getChannel(),
- metaInfo->getWidth(),
- metaInfo->getHeight() };
-
- auto normalization = static_pointer_cast<DecodingNormal>(metaInfo->decodingTypeMap.at(DecodingType::NORMAL));
- if (normalization) {
- config.normalize = normalization->use;
- config.mean = normalization->mean;
- config.std = normalization->std;
- }
-
- auto quantization =
- static_pointer_cast<DecodingQuantization>(metaInfo->decodingTypeMap.at(DecodingType::QUANTIZATION));
- if (quantization) {
- config.quantize = quantization->use;
- config.scale = quantization->scale;
- config.zeropoint = quantization->zeropoint;
- }
-
- _preprocess.setConfig(config);
- _preprocess.run<T>(mv_src, inputVector);
-
- LOGI("LEAVE");
-}
-
-template<typename T> void ObjectDetection3d<T>::inference(vector<vector<T> > &inputVectors)
-{
- LOGI("ENTER");
-
- int ret = _inference->run<T>(inputVectors);
- if (ret != MEDIA_VISION_ERROR_NONE)
- throw InvalidOperation("Fail to run inference");
-
- LOGI("LEAVE");
-}
-
-template<typename T> void ObjectDetection3d<T>::perform(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo)
-{
- vector<T> inputVector;
-
- preprocess(mv_src, metaInfo, inputVector);
-
- vector<vector<T> > inputVectors = { inputVector };
-
- inference(inputVectors);
-}
-
-template<typename T> void ObjectDetection3d<T>::getOutputNames(vector<string> &names)
-{
- TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
- IETensorBuffer &ie_tensor_buffer = tensor_buffer_obj.getIETensorBuffer();
-
- for (IETensorBuffer::iterator it = ie_tensor_buffer.begin(); it != ie_tensor_buffer.end(); it++)
- names.push_back(it->first);
-}
-
-template<typename T> void ObjectDetection3d<T>::getOutputTensor(string &target_name, vector<float> &tensor)
-{
- LOGI("ENTER");
-
- TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
-
- inference_engine_tensor_buffer *tensor_buffer = tensor_buffer_obj.getTensorBuffer(target_name);
- if (!tensor_buffer)
- throw InvalidOperation("Fail to get tensor buffer.");
-
- auto raw_buffer = static_cast<float *>(tensor_buffer->buffer);
-
- copy(&raw_buffer[0], &raw_buffer[tensor_buffer->size / sizeof(float)], back_inserter(tensor));
-
- LOGI("LEAVE");
-}
-
-template class ObjectDetection3d<float>;
-template class ObjectDetection3d<unsigned char>;
-
-}
-}
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "machine_learning_exception.h"
-#include "object_detection_3d_adapter.h"
-
-using namespace std;
-using namespace MediaVision::Common;
-using namespace mediavision::machine_learning;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-ObjectDetection3dAdapter::ObjectDetection3dAdapter()
-{
- _config = make_shared<MachineLearningConfig>();
- _config->parseConfigFile(_config_file_name);
-
- create(_config->getDefaultModelName());
-}
-
-ObjectDetection3dAdapter::~ObjectDetection3dAdapter()
-{}
-
-template<typename U> void ObjectDetection3dAdapter::create(ObjectDetection3dTaskType task_type)
-{
- switch (task_type) {
- case ObjectDetection3dTaskType::OBJECTRON:
- _object_detection_3d = make_unique<Objectron<U> >(task_type, _config);
- break;
- default:
- throw InvalidOperation("Invalid object detection 3d task type.");
- }
-}
-
-void ObjectDetection3dAdapter::create(const string &model_name)
-{
- ObjectDetection3dTaskType task_type = convertToTaskType(model_name);
- _config->loadMetaFile(make_unique<ObjectDetection3dParser>(static_cast<int>(task_type)));
- mv_inference_data_type_e dataType = _config->getInputMetaMap().begin()->second->dataType;
-
- switch (dataType) {
- case MV_INFERENCE_DATA_UINT8:
- create<unsigned char>(task_type);
- break;
- case MV_INFERENCE_DATA_FLOAT32:
- create<float>(task_type);
- break;
- default:
- throw InvalidOperation("Invalid object detection 3d data type.");
- }
-}
-
-ObjectDetection3dTaskType ObjectDetection3dAdapter::convertToTaskType(string model_name)
-{
- if (model_name.empty())
- throw InvalidParameter("model name is empty.");
-
- transform(model_name.begin(), model_name.end(), model_name.begin(), ::toupper);
-
- if (model_name == "OBJECTRON")
- return ObjectDetection3dTaskType::OBJECTRON;
-
- throw InvalidParameter("Invalid object detection 3d model name.");
-}
-
-void ObjectDetection3dAdapter::setModelInfo(const string &model_file, const string &meta_file, const string &label_file,
- const string &model_name)
-{
- try {
- _config->setUserModel(model_file, meta_file, label_file);
- create(model_name);
- } catch (const BaseException &e) {
- LOGW("A given model name is invalid so default task type will be used.");
- }
-
- if (model_file.empty() && meta_file.empty()) {
- LOGW("Given model info is invalid so default model info will be used instead.");
- return;
- }
-}
-
-void ObjectDetection3dAdapter::setEngineInfo(const string &engine_type, const string &device_type)
-{
- _object_detection_3d->setEngineInfo(string(engine_type), string(device_type));
-}
-
-void ObjectDetection3dAdapter::configure()
-{
- _object_detection_3d->configure();
-}
-
-unsigned int ObjectDetection3dAdapter::getNumberOfEngines()
-{
- return _object_detection_3d->getNumberOfEngines();
-}
-
-const string &ObjectDetection3dAdapter::getEngineType(unsigned int engine_index)
-{
- return _object_detection_3d->getEngineType(engine_index);
-}
-
-unsigned int ObjectDetection3dAdapter::getNumberOfDevices(const string &engine_type)
-{
- return _object_detection_3d->getNumberOfDevices(engine_type);
-}
-
-const string &ObjectDetection3dAdapter::getDeviceType(const string &engine_type, unsigned int device_index)
-{
- return _object_detection_3d->getDeviceType(engine_type, device_index);
-}
-
-void ObjectDetection3dAdapter::prepare()
-{
- _object_detection_3d->prepare();
-}
-
-void ObjectDetection3dAdapter::perform(InputBaseType &input)
-{
- shared_ptr<MetaInfo> metaInfo = _object_detection_3d->getInputMetaInfo();
- _object_detection_3d->perform(input.inference_src, metaInfo);
-}
-
-void ObjectDetection3dAdapter::performAsync(InputBaseType &input)
-{
- throw InvalidOperation("Not support yet.");
-}
-
-OutputBaseType &ObjectDetection3dAdapter::getOutput()
-{
- return _object_detection_3d->result();
-}
-
-OutputBaseType &ObjectDetection3dAdapter::getOutputCache()
-{
- throw InvalidOperation("Not support yet.");
-}
-
-}
-}
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <string.h>
-#include <map>
-#include <algorithm>
-
-#include "machine_learning_exception.h"
-#include "objectron.h"
-#include "Postprocess.h"
-
-using namespace std;
-using namespace mediavision::inference;
-using namespace mediavision::machine_learning::exception;
-
-namespace mediavision
-{
-namespace machine_learning
-{
-template<typename T>
-Objectron<T>::Objectron(ObjectDetection3dTaskType task_type, std::shared_ptr<MachineLearningConfig> config)
- : ObjectDetection3d<T>(task_type, config), _result()
-{}
-
-template<typename T> Objectron<T>::~Objectron()
-{}
-
-template<typename T> ObjectDetection3dResult &Objectron<T>::result()
-{
- vector<string> names;
-
- ObjectDetection3d<T>::getOutputNames(names);
-
- vector<float> keypoints;
-
- ObjectDetection3d<T>::getOutputTensor(names[1], keypoints);
-
- size_t output_size = keypoints.size();
-
- Postprocess postprocess({ _preprocess.getImageWidth()[0], _preprocess.getImageHeight()[0],
- _inference->getInputWidth(), _inference->getInputHeight() });
-
- _result = ObjectDetection3dResult();
-
- for (size_t idx = 0; idx < output_size; idx += 2) {
- _result.x_vec.push_back(postprocess.getScaledX(keypoints[idx]));
- _result.y_vec.push_back(postprocess.getScaledY(keypoints[idx + 1]));
- }
-
- _result.number_of_points = output_size / 2;
-
- vector<float> probability_vec;
-
- // names[0] is "Identity"
- ObjectDetection3d<T>::getOutputTensor(names[0], probability_vec);
-
- _result.probability = static_cast<unsigned int>(probability_vec[0] * 100);
-
- try {
- // names[1] is "Identity_1"
- auto metaInfo = _config->getOutputMetaMap().at(names[1]);
- auto decodingBox = static_pointer_cast<DecodingBox>(metaInfo->decodingTypeMap[DecodingType::BOX]);
-
- for (size_t idx = 0; idx < decodingBox->edges.size(); idx += 2)
- _result.edge_index_vec.push_back({ decodingBox->edges[idx], decodingBox->edges[idx + 1] });
-
- _result.number_of_edges = decodingBox->edges.size();
- } catch (const std::exception &e) {
- throw InvalidOperation("Invalid meta info access.");
- }
-
- return _result;
-}
-
-template class Objectron<float>;
-template class Objectron<unsigned char>;
-
-}
-}
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __DATA_SET_MANAGER_H__
+#define __DATA_SET_MANAGER_H__
+
+#include <fstream>
+#include <vector>
+#include <map>
+
+#include "FeatureVectorManager.h"
+
+class DataSetManager
+{
+protected:
+ std::map<unsigned int, unsigned int> _fv_cnt_per_label;
+ std::vector<std::vector<float> > _data;
+ std::vector<std::vector<float> > _labels;
+ std::vector<unsigned int> _label_index;
+ size_t _feature_vector_size {};
+ size_t _label_count {};
+
+public:
+ DataSetManager();
+ virtual ~DataSetManager();
+
+ void clear();
+ bool isFeatureVectorDuplicated(const std::vector<float> &vec);
+ std::vector<std::vector<float> > &getData(void);
+ std::vector<std::vector<float> > &getLabel(void);
+ size_t getFeaVecSize(void);
+ std::vector<unsigned int> &getLabelIdx(void);
+
+ virtual bool isFeatureVectorAllowed(unsigned int label_idx) = 0;
+ virtual void loadDataSet(const std::string &file_name, unsigned int new_label_cnt) = 0;
+ virtual void addDataSet(std::vector<float> &feature_vec, unsigned int label_idx, unsigned int label_cnt) = 0;
+};
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __FEATURE_VECTOR_MANAGER_H__
+#define __FEATURE_VECTOR_MANAGER_H__
+
+#include <string>
+#include <vector>
+
+#include <opencv2/opencv.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include "file_util.h"
+
+struct FeaVecHeader {
+ unsigned int signature;
+ size_t feature_size;
+ size_t label_cnt;
+ unsigned int data_set_cnt;
+};
+
+class FeatureVectorManager
+{
+protected:
+ std::string _feature_vector_file;
+
+public:
+ FeatureVectorManager(const std::string feature_vector_file = "feature_vector_file.dat");
+ virtual ~FeatureVectorManager() = default;
+
+ const std::string &getFileName();
+
+ static void getVecFromImg(const std::string image_file, std::vector<float> &vec, unsigned int width,
+ unsigned int height);
+ static void getVecFromRGB(unsigned char *in_data, std::vector<float> &vec, unsigned int width, unsigned int height,
+ size_t re_width, size_t re_height);
+ static void getVecFromXRGB(unsigned char *in_data, std::vector<float> &vec, unsigned int in_width,
+ unsigned int in_height, unsigned int re_width, unsigned int re_height);
+
+ virtual void updateHeader(size_t feature_size, size_t label_cnt, unsigned int data_set_cnt) = 0;
+ virtual void storeData(std::vector<std::vector<float> > &features_vec, std::vector<unsigned int> &label_index) = 0;
+ virtual void remove() = 0;
+
+ static constexpr unsigned int feature_vector_signature = 0xFEA09841;
+};
+
+#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __LABEL_MANAGER_H__
+#define __LABEL_MANAGER_H__
+
+#include <string.h>
+#include <iostream>
+#include <fstream>
+#include <istream>
+#include <algorithm>
+#include <vector>
+#include <map>
+
+#include "file_util.h"
+
+class LabelManager
+{
+private:
+ std::vector<std::string> _labels;
+ std::string _label_file;
+ float _decision_threshold;
+ static constexpr float _decision_weight = 0.01;
+
+public:
+ LabelManager(std::string label_file, double decision_threshold);
+ ~LabelManager();
+ float getDecisionThreshold();
+ float getDecisionWeight();
+ unsigned int getLabelIndex(const std::string &given_label);
+ bool isExist(const std::string given_label);
+ void removeLabel(const std::string given_label);
+ void getLabelString(std::string &label, const int idx);
+ void addLabelToFile(std::string given_label);
+ void importLabel(void);
+ const std::vector<std::string> &getLabels(void);
+ size_t getMaxLabel();
+ void removeFile();
+ bool isEmpty();
+};
+
+#endif
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __TRAINING_MODEL_H__
+#define __TRAINING_MODEL_H__
+
+#include <iostream>
+#include <map>
+
+#include <mv_inference_type.h>
+
+#include "training_engine_error.h"
+#include "training_engine_common_impl.h"
+#include "inference_engine_common_impl.h"
+#include "DataSetManager.h"
+#include "FeatureVectorManager.h"
+
+struct TrainingEngineBackendInfo {
+ std::vector<std::string> input_layer_names;
+ std::vector<std::string> output_layer_names;
+ std::vector<inference_engine_tensor_info> input_tensor_info;
+ std::vector<inference_engine_tensor_info> output_tensor_info;
+ training_engine_optimizer_property optimizer_property {};
+ training_engine_compile_property compile_property {};
+};
+
+class TrainingModel
+{
+private:
+ virtual void saveModel(const std::string file_path) = 0;
+ virtual void removeModel(const std::string file_path) = 0;
+
+protected:
+ std::unique_ptr<TrainingEngineInterface::Common::TrainingEngineCommon> _training;
+ std::unique_ptr<training_engine_model> _model;
+ std::unique_ptr<training_engine_dataset> _data_set;
+ std::string _internal_model_file;
+
+public:
+ TrainingModel(const training_backend_type_e backend_type, const training_target_type_e target_type,
+ const std::vector<size_t> input_tensor_shape, const std::string internal_model_file);
+ virtual ~TrainingModel();
+
+ void applyDataSet(std::unique_ptr<DataSetManager> &data_set);
+ void clearDataSet();
+ void compile();
+ void train();
+ void removeModel();
+ void getWeights(float **weights, size_t *size, std::string name);
+
+ virtual void configureModel(int num_of_class) = 0;
+ virtual TrainingEngineBackendInfo &getTrainingEngineInfo() = 0;
+};
+
+#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __DATA_SET_MANAGER_H__
-#define __DATA_SET_MANAGER_H__
-
-#include <fstream>
-#include <vector>
-#include <map>
-
-#include "feature_vector_manager.h"
-
-class DataSetManager
-{
-protected:
- std::map<unsigned int, unsigned int> _fv_cnt_per_label;
- std::vector<std::vector<float> > _data;
- std::vector<std::vector<float> > _labels;
- std::vector<unsigned int> _label_index;
- size_t _feature_vector_size {};
- size_t _label_count {};
-
-public:
- DataSetManager();
- virtual ~DataSetManager();
-
- void clear();
- bool isFeatureVectorDuplicated(const std::vector<float> &vec);
- std::vector<std::vector<float> > &getData(void);
- std::vector<std::vector<float> > &getLabel(void);
- size_t getFeaVecSize(void);
- std::vector<unsigned int> &getLabelIdx(void);
-
- virtual bool isFeatureVectorAllowed(unsigned int label_idx) = 0;
- virtual void loadDataSet(const std::string &file_name, unsigned int new_label_cnt) = 0;
- virtual void addDataSet(std::vector<float> &feature_vec, unsigned int label_idx, unsigned int label_cnt) = 0;
-};
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __FEATURE_VECTOR_MANAGER_H__
-#define __FEATURE_VECTOR_MANAGER_H__
-
-#include <string>
-#include <vector>
-
-#include <opencv2/opencv.hpp>
-#include <opencv2/imgproc/imgproc.hpp>
-
-#include "file_util.h"
-
-struct FeaVecHeader {
- unsigned int signature;
- size_t feature_size;
- size_t label_cnt;
- unsigned int data_set_cnt;
-};
-
-class FeatureVectorManager
-{
-protected:
- std::string _feature_vector_file;
-
-public:
- FeatureVectorManager(const std::string feature_vector_file = "feature_vector_file.dat");
- virtual ~FeatureVectorManager() = default;
-
- const std::string &getFileName();
-
- static void getVecFromImg(const std::string image_file, std::vector<float> &vec, unsigned int width,
- unsigned int height);
- static void getVecFromRGB(unsigned char *in_data, std::vector<float> &vec, unsigned int width, unsigned int height,
- size_t re_width, size_t re_height);
- static void getVecFromXRGB(unsigned char *in_data, std::vector<float> &vec, unsigned int in_width,
- unsigned int in_height, unsigned int re_width, unsigned int re_height);
-
- virtual void updateHeader(size_t feature_size, size_t label_cnt, unsigned int data_set_cnt) = 0;
- virtual void storeData(std::vector<std::vector<float> > &features_vec, std::vector<unsigned int> &label_index) = 0;
- virtual void remove() = 0;
-
- static constexpr unsigned int feature_vector_signature = 0xFEA09841;
-};
-
-#endif
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __LABEL_MANAGER_H__
-#define __LABEL_MANAGER_H__
-
-#include <string.h>
-#include <iostream>
-#include <fstream>
-#include <istream>
-#include <algorithm>
-#include <vector>
-#include <map>
-
-#include "file_util.h"
-
-class LabelManager
-{
-private:
- std::vector<std::string> _labels;
- std::string _label_file;
- float _decision_threshold;
- static constexpr float _decision_weight = 0.01;
-
-public:
- LabelManager(std::string label_file, double decision_threshold);
- ~LabelManager();
- float getDecisionThreshold();
- float getDecisionWeight();
- unsigned int getLabelIndex(const std::string &given_label);
- bool isExist(const std::string given_label);
- void removeLabel(const std::string given_label);
- void getLabelString(std::string &label, const int idx);
- void addLabelToFile(std::string given_label);
- void importLabel(void);
- const std::vector<std::string> &getLabels(void);
- size_t getMaxLabel();
- void removeFile();
- bool isEmpty();
-};
-
-#endif
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __TRAINING_MODEL_H__
-#define __TRAINING_MODEL_H__
-
-#include <iostream>
-#include <map>
-
-#include <mv_inference_type.h>
-
-#include "training_engine_error.h"
-#include "training_engine_common_impl.h"
-#include "inference_engine_common_impl.h"
-#include "data_set_manager.h"
-#include "feature_vector_manager.h"
-
-struct TrainingEngineBackendInfo {
- std::vector<std::string> input_layer_names;
- std::vector<std::string> output_layer_names;
- std::vector<inference_engine_tensor_info> input_tensor_info;
- std::vector<inference_engine_tensor_info> output_tensor_info;
- training_engine_optimizer_property optimizer_property {};
- training_engine_compile_property compile_property {};
-};
-
-class TrainingModel
-{
-private:
- virtual void saveModel(const std::string file_path) = 0;
- virtual void removeModel(const std::string file_path) = 0;
-
-protected:
- std::unique_ptr<TrainingEngineInterface::Common::TrainingEngineCommon> _training;
- std::unique_ptr<training_engine_model> _model;
- std::unique_ptr<training_engine_dataset> _data_set;
- std::string _internal_model_file;
-
-public:
- TrainingModel(const training_backend_type_e backend_type, const training_target_type_e target_type,
- const std::vector<size_t> input_tensor_shape, const std::string internal_model_file);
- virtual ~TrainingModel();
-
- void applyDataSet(std::unique_ptr<DataSetManager> &data_set);
- void clearDataSet();
- void compile();
- void train();
- void removeModel();
- void getWeights(float **weights, size_t *size, std::string name);
-
- virtual void configureModel(int num_of_class) = 0;
- virtual TrainingEngineBackendInfo &getTrainingEngineInfo() = 0;
-};
-
-#endif
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "DataSetManager.h"
+
+using namespace std;
+
+DataSetManager::DataSetManager() : _data(), _labels(), _label_index(), _feature_vector_size(), _label_count()
+{}
+
+DataSetManager::~DataSetManager()
+{
+ clear();
+}
+
+void DataSetManager::clear()
+{
+ for (auto &data : _data)
+ data.clear();
+
+ _data.clear();
+
+ for (auto &label : _labels)
+ label.clear();
+
+ _labels.clear();
+ _label_index.clear();
+ _fv_cnt_per_label.clear();
+}
+
+bool DataSetManager::isFeatureVectorDuplicated(const vector<float> &vec)
+{
+ if (_data.empty())
+ return false;
+
+ for (const auto &data : _data)
+ if (data == vec)
+ return true;
+
+ return false;
+}
+
+vector<vector<float> > &DataSetManager::getData(void)
+{
+ return _data;
+}
+
+vector<vector<float> > &DataSetManager::getLabel(void)
+{
+ return _labels;
+}
+
+size_t DataSetManager::getFeaVecSize(void)
+{
+ return _feature_vector_size;
+}
+
+vector<unsigned int> &DataSetManager::getLabelIdx(void)
+{
+ return _label_index;
+}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <opencv2/opencv.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include "MvMlException.h"
+#include "FeatureVectorManager.h"
+
+using namespace std;
+using namespace mediavision::machine_learning::exception;
+
+FeatureVectorManager::FeatureVectorManager(const string feature_vector_file) : _feature_vector_file(feature_vector_file)
+{}
+
+const string &FeatureVectorManager::getFileName()
+{
+ return _feature_vector_file;
+}
+
+void FeatureVectorManager::getVecFromImg(const string image_file, vector<float> &vec, unsigned int width,
+ unsigned int height)
+{
+ cv::Mat src, dst;
+
+ if (!FaceRecogUtil::isImageFile(image_file))
+ throw InvalidOperation("Invalid image file.");
+
+ src = cv::imread(image_file);
+
+ if (!src.data)
+ throw InvalidOperation("Invalid image data.");
+
+ cv::cvtColor(src, src, cv::COLOR_BGR2RGB);
+
+ cv::Mat resized;
+
+ resize(src, resized, cv::Size(width, height), 0, 0, cv::INTER_CUBIC);
+
+ cv::Mat floatSrc;
+
+ resized.convertTo(floatSrc, CV_32FC3);
+
+ cv::Mat meant = cv::Mat(floatSrc.size(), CV_32FC3, cv::Scalar(127.5f, 127.5f, 127.5f));
+
+ cv::subtract(floatSrc, meant, dst);
+ dst /= 127.5f;
+
+ vec.assign((float *) dst.data, (float *) dst.data + dst.total() * dst.channels());
+}
+
+void FeatureVectorManager::getVecFromRGB(unsigned char *in_data, vector<float> &vec, unsigned int width,
+ unsigned int height, size_t re_width, size_t re_height)
+{
+ cv::Mat cvSrc = cv::Mat(cv::Size(width, height), CV_MAKETYPE(CV_8U, 3), in_data).clone();
+
+ cv::Mat resized;
+
+ resize(cvSrc, resized, cv::Size(re_width, re_height), 0, 0, cv::INTER_CUBIC);
+
+ cv::Mat floatSrc;
+
+ resized.convertTo(floatSrc, CV_32FC3);
+
+ cv::Mat meant = cv::Mat(floatSrc.size(), CV_32FC3, cv::Scalar(127.5f, 127.5f, 127.5f));
+ cv::Mat dst;
+
+ cv::subtract(floatSrc, meant, dst);
+ dst /= 127.5f;
+
+ vec.assign((float *) dst.data, (float *) dst.data + dst.total() * dst.channels());
+}
+
+void FeatureVectorManager::getVecFromXRGB(unsigned char *in_data, vector<float> &vec, unsigned int in_width,
+ unsigned int in_height, unsigned int re_width, unsigned int re_height)
+{
+ cv::Mat argb(cv::Size(in_width, in_height), CV_8UC4, in_data);
+
+ cv::Mat split_rgbx[4];
+ cv::split(argb, split_rgbx);
+ cv::Mat split[] = { split_rgbx[0], split_rgbx[1], split_rgbx[2] };
+ cv::Mat rgb;
+ cv::merge(split, 3, rgb);
+
+ cv::Mat resized;
+
+ resize(rgb, resized, cv::Size(re_width, re_height), 0, 0, cv::INTER_CUBIC);
+
+ cv::Mat floatSrc;
+
+ resized.convertTo(floatSrc, CV_32FC3);
+
+ cv::Mat meant = cv::Mat(floatSrc.size(), CV_32FC3, cv::Scalar(127.5f, 127.5f, 127.5f));
+
+ cv::Mat dst;
+
+ cv::subtract(floatSrc, meant, dst);
+ dst /= 127.5f;
+
+ vec.assign((float *) dst.data, (float *) dst.data + dst.total() * dst.channels());
+}
\ No newline at end of file
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <dlog.h>
+#include <mv_private.h>
+
+#include "MvMlException.h"
+#include "LabelManager.h"
+
+using namespace std;
+using namespace mediavision::machine_learning::exception;
+
+LabelManager::LabelManager(string label_file, double decision_threshold) : _labels(), _label_file(label_file)
+{
+ _decision_threshold = decision_threshold;
+
+ LOGD("labal file path is %s", label_file.c_str());
+ LOGD("decision_threshold value is %lf", _decision_threshold);
+}
+
+LabelManager::~LabelManager()
+{}
+
+float LabelManager::getDecisionThreshold()
+{
+ return _decision_threshold;
+}
+
+float LabelManager::getDecisionWeight()
+{
+ return _decision_weight;
+}
+
+unsigned int LabelManager::getLabelIndex(const string &given_label)
+{
+ importLabel();
+
+ auto iter = find(_labels.begin(), _labels.end(), given_label);
+ if (iter == _labels.end())
+ throw InvalidOperation("Label index not found.");
+
+ return distance(_labels.begin(), iter);
+}
+
+void LabelManager::removeLabel(const string given_label)
+{
+ ifstream readFile { _label_file };
+
+ string new_label_file = _label_file + ".new";
+ ofstream writeFile { new_label_file, ios::out | ios::binary };
+
+ if (readFile.fail() || !writeFile.is_open()) {
+ throw InvalidOperation("Fail to open label file for read/write.");
+ }
+
+ std::streampos total_length = 0;
+ string line;
+
+ while (getline(readFile, line)) {
+ total_length = readFile.tellg();
+ if (line != given_label)
+ writeFile.write(line.c_str(), static_cast<streamsize>(line.size())).write("\n", 1);
+ }
+
+ readFile.close();
+ writeFile.close();
+
+ if (total_length == writeFile.tellp()) {
+ if (::remove(new_label_file.c_str()) == -1)
+ throw InvalidOperation("Fail to remove label file.");
+ }
+
+ if (::remove(_label_file.c_str()) == -1)
+ throw InvalidOperation("Fail to remove label file.");
+
+ if (::rename(new_label_file.c_str(), _label_file.c_str()) == -1)
+ throw InvalidOperation("Fail to rename new labal file to original one.");
+
+ // Update _labels because label file is changed.
+ importLabel();
+}
+
+void LabelManager::getLabelString(string &label, const int idx)
+{
+ if (idx < 0 || _labels.size() <= static_cast<size_t>(idx))
+ throw InvalidOperation("A given label index is invalid.");
+
+ label = _labels[idx];
+}
+
+void LabelManager::addLabelToFile(string given_label)
+{
+ ofstream writeFile;
+
+ writeFile.open(_label_file.c_str(), ios::out | ios::app);
+
+ if (!writeFile.is_open())
+ throw InvalidOperation("Fail to open " + _label_file + " file.");
+
+ given_label += "\n";
+ writeFile.write(given_label.c_str(), static_cast<streamsize>(given_label.size()));
+ writeFile.close();
+}
+
+void LabelManager::importLabel(void)
+{
+ // just return if label file doesn't exist.
+ if (!FaceRecogUtil::isFileExist(_label_file))
+ return;
+
+ ifstream readFile;
+
+ readFile.open(_label_file);
+
+ if (readFile.fail())
+ throw InvalidOperation("Fail to open " + _label_file + " file.");
+
+ string line;
+
+ _labels.clear();
+
+ while (getline(readFile, line)) {
+ // If a given label alreadys in the label file then just skip.
+ if (isExist(line))
+ continue;
+
+ _labels.push_back(line);
+ }
+
+ readFile.close();
+
+ LOGD("%zu labels have been imported", _labels.size());
+}
+
+const std::vector<std::string> &LabelManager::getLabels(void)
+{
+ return _labels;
+}
+
+bool LabelManager::isExist(const string given_label)
+{
+ // return false if label file doesn't exist.
+ if (!FaceRecogUtil::isFileExist(_label_file))
+ return false;
+
+ return (find(_labels.begin(), _labels.end(), given_label) != _labels.end());
+}
+
+size_t LabelManager::getMaxLabel()
+{
+ return _labels.size();
+}
+
+void LabelManager::removeFile()
+{
+ // Remove existing files forcely.
+ int ret = ::remove(_label_file.c_str());
+ if (ret)
+ throw InvalidOperation("Fail to remove label file.");
+}
+
+bool LabelManager::isEmpty()
+{
+ return _labels.empty();
+}
--- /dev/null
+/**
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <fstream>
+#include <istream>
+#include <tuple>
+#include <map>
+#include <algorithm>
+
+#include <sys/stat.h>
+
+#include <dlog.h>
+#include <mv_private.h>
+
+#include "MvMlException.h"
+#include "TrainingModel.h"
+
+using namespace std;
+using namespace TrainingEngineInterface::Common;
+using namespace mediavision::machine_learning::exception;
+
+TrainingModel::TrainingModel(const training_backend_type_e backend_type, const training_target_type_e target_type,
+ vector<size_t> input_tensor_shape, const string internal_model_file)
+{
+ _internal_model_file = internal_model_file;
+ _training = make_unique<TrainingEngineInterface::Common::TrainingEngineCommon>();
+
+ training_engine_config config = { "", backend_type, target_type };
+ int ret = _training->BindBackend(&config);
+ if (ret != TRAINING_ENGINE_ERROR_NONE)
+ throw InvalidOperation("Fail to bind backend engine.");
+
+ training_engine_capacity capacity = { TRAINING_TENSOR_SHAPE_MIN };
+ ret = _training->GetBackendCapacity(capacity);
+ if (ret != TRAINING_ENGINE_ERROR_NONE)
+ throw InvalidOperation("Fail to get backend capacity.");
+}
+
+TrainingModel::~TrainingModel()
+{
+ if (_training)
+ _training->UnbindBackend();
+}
+
+void TrainingModel::applyDataSet(unique_ptr<DataSetManager> &data_set)
+{
+ auto &values = data_set->getData();
+ auto &labels = data_set->getLabel();
+
+ LOGD("Generating feature vectors for training");
+
+ _data_set = _training->CreateDataset();
+ if (!_data_set)
+ throw InvalidOperation("Fail to create a dataset.");
+
+ int ret;
+
+ for (size_t idx = 0; idx < values.size(); ++idx) {
+ ret = _training->AddDataToDataset(_data_set.get(), values[idx], labels[idx], TRAINING_DATASET_TYPE_TRAIN);
+ if (ret != TRAINING_ENGINE_ERROR_NONE)
+ throw InvalidOperation("Fail to add data to dataset.", ret);
+ }
+
+ ret = _training->SetDataset(_model.get(), _data_set.get());
+ if (ret != TRAINING_ENGINE_ERROR_NONE)
+ throw InvalidOperation("Fail to set dataset to model.", ret);
+}
+
+void TrainingModel::clearDataSet()
+{
+ _training->DestroyDataset(_data_set.get());
+}
+
+void TrainingModel::compile()
+{
+ auto optimizer = _training->CreateOptimizer(TRAINING_OPTIMIZER_TYPE_SGD);
+ if (!optimizer)
+ throw InvalidOperation("Fail to create a optimizer.");
+
+ int ret = _training->SetOptimizerProperty(optimizer.get(), getTrainingEngineInfo().optimizer_property);
+ if (ret != TRAINING_ENGINE_ERROR_NONE)
+ throw InvalidOperation("Fail to set optimizer property.", ret);
+
+ ret = _training->AddOptimizer(_model.get(), optimizer.get());
+ if (ret != TRAINING_ENGINE_ERROR_NONE)
+ throw InvalidOperation("Fail to add optimizer to model.", ret);
+
+ ret = _training->CompileModel(_model.get(), getTrainingEngineInfo().compile_property);
+ if (ret != TRAINING_ENGINE_ERROR_NONE)
+ throw InvalidOperation("Fail to compile model.", ret);
+}
+
+void TrainingModel::train()
+{
+ training_engine_model_property model_property;
+ int ret = _training->TrainModel(_model.get(), model_property);
+ if (ret != TRAINING_ENGINE_ERROR_NONE)
+ throw InvalidOperation("Fail to train model.", ret);
+
+ // Save model file.
+ saveModel(_internal_model_file);
+}
+
+void TrainingModel::getWeights(float **weights, size_t *size, std::string name)
+{
+ // TODO.
+}
+
+void TrainingModel::removeModel()
+{
+ removeModel(_internal_model_file);
+}
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "data_set_manager.h"
-
-using namespace std;
-
-DataSetManager::DataSetManager() : _data(), _labels(), _label_index(), _feature_vector_size(), _label_count()
-{}
-
-DataSetManager::~DataSetManager()
-{
- clear();
-}
-
-void DataSetManager::clear()
-{
- for (auto &data : _data)
- data.clear();
-
- _data.clear();
-
- for (auto &label : _labels)
- label.clear();
-
- _labels.clear();
- _label_index.clear();
- _fv_cnt_per_label.clear();
-}
-
-bool DataSetManager::isFeatureVectorDuplicated(const vector<float> &vec)
-{
- if (_data.empty())
- return false;
-
- for (const auto &data : _data)
- if (data == vec)
- return true;
-
- return false;
-}
-
-vector<vector<float> > &DataSetManager::getData(void)
-{
- return _data;
-}
-
-vector<vector<float> > &DataSetManager::getLabel(void)
-{
- return _labels;
-}
-
-size_t DataSetManager::getFeaVecSize(void)
-{
- return _feature_vector_size;
-}
-
-vector<unsigned int> &DataSetManager::getLabelIdx(void)
-{
- return _label_index;
-}
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <opencv2/opencv.hpp>
-#include <opencv2/imgproc/imgproc.hpp>
-
-#include "machine_learning_exception.h"
-#include "feature_vector_manager.h"
-
-using namespace std;
-using namespace mediavision::machine_learning::exception;
-
-FeatureVectorManager::FeatureVectorManager(const string feature_vector_file) : _feature_vector_file(feature_vector_file)
-{}
-
-const string &FeatureVectorManager::getFileName()
-{
- return _feature_vector_file;
-}
-
-void FeatureVectorManager::getVecFromImg(const string image_file, vector<float> &vec, unsigned int width,
- unsigned int height)
-{
- cv::Mat src, dst;
-
- if (!FaceRecogUtil::isImageFile(image_file))
- throw InvalidOperation("Invalid image file.");
-
- src = cv::imread(image_file);
-
- if (!src.data)
- throw InvalidOperation("Invalid image data.");
-
- cv::cvtColor(src, src, cv::COLOR_BGR2RGB);
-
- cv::Mat resized;
-
- resize(src, resized, cv::Size(width, height), 0, 0, cv::INTER_CUBIC);
-
- cv::Mat floatSrc;
-
- resized.convertTo(floatSrc, CV_32FC3);
-
- cv::Mat meant = cv::Mat(floatSrc.size(), CV_32FC3, cv::Scalar(127.5f, 127.5f, 127.5f));
-
- cv::subtract(floatSrc, meant, dst);
- dst /= 127.5f;
-
- vec.assign((float *) dst.data, (float *) dst.data + dst.total() * dst.channels());
-}
-
-void FeatureVectorManager::getVecFromRGB(unsigned char *in_data, vector<float> &vec, unsigned int width,
- unsigned int height, size_t re_width, size_t re_height)
-{
- cv::Mat cvSrc = cv::Mat(cv::Size(width, height), CV_MAKETYPE(CV_8U, 3), in_data).clone();
-
- cv::Mat resized;
-
- resize(cvSrc, resized, cv::Size(re_width, re_height), 0, 0, cv::INTER_CUBIC);
-
- cv::Mat floatSrc;
-
- resized.convertTo(floatSrc, CV_32FC3);
-
- cv::Mat meant = cv::Mat(floatSrc.size(), CV_32FC3, cv::Scalar(127.5f, 127.5f, 127.5f));
- cv::Mat dst;
-
- cv::subtract(floatSrc, meant, dst);
- dst /= 127.5f;
-
- vec.assign((float *) dst.data, (float *) dst.data + dst.total() * dst.channels());
-}
-
-void FeatureVectorManager::getVecFromXRGB(unsigned char *in_data, vector<float> &vec, unsigned int in_width,
- unsigned int in_height, unsigned int re_width, unsigned int re_height)
-{
- cv::Mat argb(cv::Size(in_width, in_height), CV_8UC4, in_data);
-
- cv::Mat split_rgbx[4];
- cv::split(argb, split_rgbx);
- cv::Mat split[] = { split_rgbx[0], split_rgbx[1], split_rgbx[2] };
- cv::Mat rgb;
- cv::merge(split, 3, rgb);
-
- cv::Mat resized;
-
- resize(rgb, resized, cv::Size(re_width, re_height), 0, 0, cv::INTER_CUBIC);
-
- cv::Mat floatSrc;
-
- resized.convertTo(floatSrc, CV_32FC3);
-
- cv::Mat meant = cv::Mat(floatSrc.size(), CV_32FC3, cv::Scalar(127.5f, 127.5f, 127.5f));
-
- cv::Mat dst;
-
- cv::subtract(floatSrc, meant, dst);
- dst /= 127.5f;
-
- vec.assign((float *) dst.data, (float *) dst.data + dst.total() * dst.channels());
-}
\ No newline at end of file
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <dlog.h>
-#include <mv_private.h>
-
-#include "machine_learning_exception.h"
-#include "label_manager.h"
-
-using namespace std;
-using namespace mediavision::machine_learning::exception;
-
-LabelManager::LabelManager(string label_file, double decision_threshold) : _labels(), _label_file(label_file)
-{
- _decision_threshold = decision_threshold;
-
- LOGD("labal file path is %s", label_file.c_str());
- LOGD("decision_threshold value is %lf", _decision_threshold);
-}
-
-LabelManager::~LabelManager()
-{}
-
-float LabelManager::getDecisionThreshold()
-{
- return _decision_threshold;
-}
-
-float LabelManager::getDecisionWeight()
-{
- return _decision_weight;
-}
-
-unsigned int LabelManager::getLabelIndex(const string &given_label)
-{
- importLabel();
-
- auto iter = find(_labels.begin(), _labels.end(), given_label);
- if (iter == _labels.end())
- throw InvalidOperation("Label index not found.");
-
- return distance(_labels.begin(), iter);
-}
-
-void LabelManager::removeLabel(const string given_label)
-{
- ifstream readFile { _label_file };
-
- string new_label_file = _label_file + ".new";
- ofstream writeFile { new_label_file, ios::out | ios::binary };
-
- if (readFile.fail() || !writeFile.is_open()) {
- throw InvalidOperation("Fail to open label file for read/write.");
- }
-
- std::streampos total_length = 0;
- string line;
-
- while (getline(readFile, line)) {
- total_length = readFile.tellg();
- if (line != given_label)
- writeFile.write(line.c_str(), static_cast<streamsize>(line.size())).write("\n", 1);
- }
-
- readFile.close();
- writeFile.close();
-
- if (total_length == writeFile.tellp()) {
- if (::remove(new_label_file.c_str()) == -1)
- throw InvalidOperation("Fail to remove label file.");
- }
-
- if (::remove(_label_file.c_str()) == -1)
- throw InvalidOperation("Fail to remove label file.");
-
- if (::rename(new_label_file.c_str(), _label_file.c_str()) == -1)
- throw InvalidOperation("Fail to rename new labal file to original one.");
-
- // Update _labels because label file is changed.
- importLabel();
-}
-
-void LabelManager::getLabelString(string &label, const int idx)
-{
- if (idx < 0 || _labels.size() <= static_cast<size_t>(idx))
- throw InvalidOperation("A given label index is invalid.");
-
- label = _labels[idx];
-}
-
-void LabelManager::addLabelToFile(string given_label)
-{
- ofstream writeFile;
-
- writeFile.open(_label_file.c_str(), ios::out | ios::app);
-
- if (!writeFile.is_open())
- throw InvalidOperation("Fail to open " + _label_file + " file.");
-
- given_label += "\n";
- writeFile.write(given_label.c_str(), static_cast<streamsize>(given_label.size()));
- writeFile.close();
-}
-
-void LabelManager::importLabel(void)
-{
- // just return if label file doesn't exist.
- if (!FaceRecogUtil::isFileExist(_label_file))
- return;
-
- ifstream readFile;
-
- readFile.open(_label_file);
-
- if (readFile.fail())
- throw InvalidOperation("Fail to open " + _label_file + " file.");
-
- string line;
-
- _labels.clear();
-
- while (getline(readFile, line)) {
- // If a given label alreadys in the label file then just skip.
- if (isExist(line))
- continue;
-
- _labels.push_back(line);
- }
-
- readFile.close();
-
- LOGD("%zu labels have been imported", _labels.size());
-}
-
-const std::vector<std::string> &LabelManager::getLabels(void)
-{
- return _labels;
-}
-
-bool LabelManager::isExist(const string given_label)
-{
- // return false if label file doesn't exist.
- if (!FaceRecogUtil::isFileExist(_label_file))
- return false;
-
- return (find(_labels.begin(), _labels.end(), given_label) != _labels.end());
-}
-
-size_t LabelManager::getMaxLabel()
-{
- return _labels.size();
-}
-
-void LabelManager::removeFile()
-{
- // Remove existing files forcely.
- int ret = ::remove(_label_file.c_str());
- if (ret)
- throw InvalidOperation("Fail to remove label file.");
-}
-
-bool LabelManager::isEmpty()
-{
- return _labels.empty();
-}
+++ /dev/null
-/**
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <string.h>
-#include <fstream>
-#include <istream>
-#include <tuple>
-#include <map>
-#include <algorithm>
-
-#include <sys/stat.h>
-
-#include <dlog.h>
-#include <mv_private.h>
-
-#include "machine_learning_exception.h"
-#include "training_model.h"
-
-using namespace std;
-using namespace TrainingEngineInterface::Common;
-using namespace mediavision::machine_learning::exception;
-
-TrainingModel::TrainingModel(const training_backend_type_e backend_type, const training_target_type_e target_type,
- vector<size_t> input_tensor_shape, const string internal_model_file)
-{
- _internal_model_file = internal_model_file;
- _training = make_unique<TrainingEngineInterface::Common::TrainingEngineCommon>();
-
- training_engine_config config = { "", backend_type, target_type };
- int ret = _training->BindBackend(&config);
- if (ret != TRAINING_ENGINE_ERROR_NONE)
- throw InvalidOperation("Fail to bind backend engine.");
-
- training_engine_capacity capacity = { TRAINING_TENSOR_SHAPE_MIN };
- ret = _training->GetBackendCapacity(capacity);
- if (ret != TRAINING_ENGINE_ERROR_NONE)
- throw InvalidOperation("Fail to get backend capacity.");
-}
-
-TrainingModel::~TrainingModel()
-{
- if (_training)
- _training->UnbindBackend();
-}
-
-void TrainingModel::applyDataSet(unique_ptr<DataSetManager> &data_set)
-{
- auto &values = data_set->getData();
- auto &labels = data_set->getLabel();
-
- LOGD("Generating feature vectors for training");
-
- _data_set = _training->CreateDataset();
- if (!_data_set)
- throw InvalidOperation("Fail to create a dataset.");
-
- int ret;
-
- for (size_t idx = 0; idx < values.size(); ++idx) {
- ret = _training->AddDataToDataset(_data_set.get(), values[idx], labels[idx], TRAINING_DATASET_TYPE_TRAIN);
- if (ret != TRAINING_ENGINE_ERROR_NONE)
- throw InvalidOperation("Fail to add data to dataset.", ret);
- }
-
- ret = _training->SetDataset(_model.get(), _data_set.get());
- if (ret != TRAINING_ENGINE_ERROR_NONE)
- throw InvalidOperation("Fail to set dataset to model.", ret);
-}
-
-void TrainingModel::clearDataSet()
-{
- _training->DestroyDataset(_data_set.get());
-}
-
-void TrainingModel::compile()
-{
- auto optimizer = _training->CreateOptimizer(TRAINING_OPTIMIZER_TYPE_SGD);
- if (!optimizer)
- throw InvalidOperation("Fail to create a optimizer.");
-
- int ret = _training->SetOptimizerProperty(optimizer.get(), getTrainingEngineInfo().optimizer_property);
- if (ret != TRAINING_ENGINE_ERROR_NONE)
- throw InvalidOperation("Fail to set optimizer property.", ret);
-
- ret = _training->AddOptimizer(_model.get(), optimizer.get());
- if (ret != TRAINING_ENGINE_ERROR_NONE)
- throw InvalidOperation("Fail to add optimizer to model.", ret);
-
- ret = _training->CompileModel(_model.get(), getTrainingEngineInfo().compile_property);
- if (ret != TRAINING_ENGINE_ERROR_NONE)
- throw InvalidOperation("Fail to compile model.", ret);
-}
-
-void TrainingModel::train()
-{
- training_engine_model_property model_property;
- int ret = _training->TrainModel(_model.get(), model_property);
- if (ret != TRAINING_ENGINE_ERROR_NONE)
- throw InvalidOperation("Fail to train model.", ret);
-
- // Save model file.
- saveModel(_internal_model_file);
-}
-
-void TrainingModel::getWeights(float **weights, size_t *size, std::string name)
-{
- // TODO.
-}
-
-void TrainingModel::removeModel()
-{
- removeModel(_internal_model_file);
-}
\ No newline at end of file
%files machine_learning-devel
%{_includedir}/media/mv_infer*.h
-%{_includedir}/media/machine_learning_exception.h
-%{_includedir}/media/machine_learning_config.h
-%{_includedir}/media/MachineLearningType.h
-%{_includedir}/media/machine_learning_preprocess.h
+%{_includedir}/media/MvMlException.h
+%{_includedir}/media/MvMlConfig.h
+%{_includedir}/media/mv_ml_types.h
+%{_includedir}/media/MvMlPreprocess.h
%{_libdir}/pkgconfig/*inference.pc
%if "%{enable_ml_face_recognition}" == "1"
%{_includedir}/media/mv_face_recognition*.h
%{_includedir}/media/mv_object_detection_type.h
%{_includedir}/media/mv_face_detection_internal.h
%{_includedir}/media/mv_face_detection_type.h
-%{_includedir}/media/iobject_detection.h
+%{_includedir}/media/IObjectDetection.h
%{_includedir}/media/object_detection_type.h
%{_libdir}/pkgconfig/*object-detection.pc
%endif
%if "%{enable_ml_image_segmentation}" == "1"
%{_includedir}/media/mv_selfie_segmentation_internal.h
%{_includedir}/media/mv_selfie_segmentation_type.h
-%{_includedir}/media/iimage_segmentation.h
+%{_includedir}/media/IImageSegmentation.h
%{_includedir}/media/image_segmentation_type.h
%{_libdir}/pkgconfig/*image-segmentation.pc
%endif
projects / platform / core / api / mediavision.git / commitdiff