This patch adds a profiler support for inference engine.
For now, only latency profiling is support. For other, they
will be suppored later.
The profiler isn't built in inference engine framework so
other users can use the profiler anytime using some API.
And also, Inference engine framework provides some API for
profiler like below,
[How-to-use]
InferenceEngineCommon *engine = new InferenceEngineCOmmon(&config);
...
// Enable the use of profiler.
// Ps. Make sure to create InferenceEngineCommon object
// before enabling profiler.
ret = engine->EnableProfiler(true);
// Profile data will be stored to dump.txt file.
// If you want to just print out the profile data on console screen
// then call DumpProfileToConsole instead.
ret = engine->DumpProfileToFile("dump.txt");
All profile data will printed out to console screen or stored
to a given file in Markdown syntex[1] like below,
"
backend|target devices|model name|Function name|Latency(ms)|Memory Usage(kb)
--|--|--|--|--|--
armnn|CPU|/usr/share/capi-media-vision/models/IC/tflite/ic_tflite_model.tflite|Load|1799|
armnn|CPU|/usr/share/capi-media-vision/models/IC/tflite/ic_tflite_model.tflite|Run|95|
***
"
Please use Markdown syntax aware editer such as the edit box of github.
[1] https://daringfireball.net/projects/markdown/syntax
Change-Id: I2f643bb0b78410d48f94de2e39d986a2d646d97b
Signed-off-by: Inki Dae <inki.dae@samsung.com>
#include "inference_engine_common.h"
#include "inference_engine_type.h"
+#include "inference_engine_profiler.h"
+
+using namespace InferenceEngineInterface::Profiler;
namespace InferenceEngineInterface {
namespace Common {
std::vector<inference_engine_tensor_buffer> &output_buffers);
/**
+ * @brief Enable or disable Inference engine profiler.
+ * @details This function is used for inference engine user to control inference engine profiler itself.
+ * If inference engine user wants to collect profile data itself then disable inference engine profiler
+ * by calling EnableProfiler(false) and then call InferenceEngineProfiler interfaces properly.
+ *
+ * @since_tizen 6.0
+ * @param[in] enable whether using inference engine profiler or not, which can be true or false.
+ */
+ int EnableProfiler(bool enable);
+
+ /**
+ * @brief Dump profile data to console screen.
+ * @details This function is used to print out profile data on console screen.
+ *
+ * @since_tizen 6.0
+ */
+ int DumpProfileToConsole(void);
+
+ /**
+ * @brief Dump profile data to a file.
+ * @details This function is used to store profile data to file.
+ *
+ * @since_tizen 6.0
+ * @param[in] filename A file name for profile data to be stored in.
+ */
+ int DumpProfileToFile(const std::string filename = "dump.txt");
+
+ /**
* @brief Get inference results.
*
* @since_tizen 5.5
private:
std::string mBackendLibName;
inference_backend_type_e mSelectedBackendEngine;
+
+ // Profiler
+ InferenceEngineProfiler *mProfiler;
+ // In default, we use profiler.
+ bool mUseProfiler;
+ unsigned int mProfilerDumpType;
+
protected:
void *mBackendModule;
IInferenceEngineCommon *mBackendHandle;
--- /dev/null
+/**
+ * Copyright (c) 2020 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __INFERENCE_ENGINE_PROFILER_H__
+#define __INFERENCE_ENGINE_PROFILER_H__
+
+#include <vector>
+#include <map>
+#include <string>
+#include <time.h>
+
+#include "inference_engine_type.h"
+
+namespace InferenceEngineInterface {
+namespace Profiler {
+
+/**
+ * @brief Enumeration for how-to-dump to profile data.
+ *
+ * @since_tizen 6.0
+ *
+ */
+enum {
+ IR_PROFILER_DUMP_MIN,
+ // Profile data will be printed out on console screen.
+ IR_PROFILER_DUMP_CONSOLE,
+ // Profile data will be stored on a given file.
+ IR_PROFILER_DUMP_FILE,
+ IR_PROFILER_DUMP_MAX
+};
+
+/**
+ * @brief Enumeration for profile types.
+ *
+ * @since_tizen 6.0
+ *
+ */
+enum {
+ IR_PROFILER_MIN,
+ // Measure performance in millisecond.
+ IR_PROFILER_LATENCY,
+ // Measure physical memory usage.
+ IR_PROFILER_MEMORY,
+ IR_PROFILER_MAX
+};
+
+/**
+ * @brief Enumeration for dump format to profile data.
+ *
+ * @since_tizen 6.0
+ *
+ */
+enum {
+ IR_PROFILER_DUMP_FORMAT_MIN,
+ // Store profiling data to a given file in Markdown syntax[1]
+ // [1] https://daringfireball.net/projects/markdown/syntax
+ IR_PROFILER_DUMP_FORMAT_MARKDOWN,
+ IR_PROFILER_DUMP_FORMAT_MAX
+};
+
+/**
+ * @brief A structure of containg inference env.
+ * @details This structure contains inference information which says what env. the inference is being performed on.
+ *
+ * @since_tizen 6.0
+ */
+typedef struct _ProfileEnv {
+ std::string backend_name; /**< backend name such as armnn, tflite, opencv and dldt. */
+ std::string model_name; /**< model name which contains full path of a given model file. */
+ unsigned int target_devices; /**< Hardware type the inference will be performed on. */
+} ProfileEnv;
+
+/**
+ * @brief A structure of containg profiling data.
+ * @details This structure contains profiling data while in inference.
+ *
+ * @since_tizen 6.0
+ */
+typedef struct _ProileData {
+ unsigned int env_idx; /**< An index of v_mProfileEnv vector.*/
+ std::string function_name; /**< A function name targetd to profile. */
+ size_t memory_usage; /**< memory size consumed by a given function. */
+ unsigned int elapsed_time; /**< A latency to how long time a given function is performed. */
+} ProfileData;
+
+/**
+ * @brief A class of representing profiler.
+ * @details This class interfaces will be called by InferenceEngineCommon class properly.
+ *
+ * @since_tizen 6.0
+ */
+class InferenceEngineProfiler {
+public:
+ InferenceEngineProfiler();
+ ~InferenceEngineProfiler();
+
+ /**
+ * @brief Set backend name.
+ * @details It will be set in BindBackend callback of InferenceEngineCommon object
+ * to indicate which backend - armnn, opencv, tflite or dldt - inference will be performed by.
+ *
+ * @since_tizen 6.0
+ * @param[in] name A backend name.
+ */
+ void AddBackendName(std::string &name) { mProfileEnv.backend_name = name; }
+
+ /**
+ * @brief Set model name.
+ * @details It will be set in Load callback of InferenceEngineCommon object to indicate which pre-trained model
+ * the inference will be performed on.
+ *
+ * @since_tizen 6.0
+ * @param[in] name A full path to model file.
+ */
+ void AddModelName(std::string &name) { mProfileEnv.model_name = name; }
+
+ /**
+ * @brief Set taget devices the inference runs on.
+ * @details It will be set in SetTargetDevices callback of InferenceEngineCommon object to indicate
+ * which Hardware - CPU or GPU - the inference will be performed on.
+ *
+ * @since_tizen 6.0
+ * @param[in] name A target device type. Please refer to inference_target_type_e enumeration of inference_engine_type.h.
+ */
+ void AddTargetDevices(unsigned int devices) { mProfileEnv.target_devices = devices; }
+
+ /**
+ * @brief Add inference env. information to a vector member, v_mProfileEnv.
+ * @details It will be called in Load callback of InferenceEngineCommon object to add inference env. information
+ * updated already to the vector member, which will be used to get inference env. information
+ * when dumping profile data.
+ *
+ * @since_tizen 6.0
+ */
+ void PushEnv(void) { v_mProfileEnv.push_back(mProfileEnv); mEnvNum++; }
+
+ /**
+ * @brief Start profiling with a given profile type.
+ * @details It will be called at top of a callback function of InferenceEngineCommon object to collect profile data.
+ *
+ * @since_tizen 6.0
+ * @param[in] type Profile type which can be IR_PROFILER_LATENCY or IR_PROFILER_MEMORY for now.
+ */
+ void Start(const unsigned int type);
+
+ /**
+ * @brief Stop profiling to a given profile type.
+ * @details It will be called at bottom of a callback function of InferenceEngineCommon object to collect profile data.
+ *
+ * @since_tizen 6.0
+ * @param[in] type Profile type which can be IR_PROFILER_LATENCY or IR_PROFILER_MEMORY for now.
+ * @param[in] env_idx A index to v_mProfileEnv vector object.
+ * @param[in] func_name A function name to be profiled.
+ */
+ void Stop(const unsigned int type, const char *func_name);
+
+ /**
+ * @brief Dump profiled data to console or a given file.
+ * @details It will be called in deconstructor of InferenceEngineCommon object to dump all of collected profile data.
+ *
+ * @since_tizen 6.0
+ * @param[in] dump_type A dump type which can be IR_PROFILER_DUMP_TEXT or IR_PROFILER_DUMP_FILE for now.
+ */
+ void Dump(const unsigned int dump_type);
+
+ /**
+ * @brief Set user-given dump file name.
+ * @details If a file name is set using this function then profiled data will be stored to the given file.
+ *
+ * @since_tizen 6.0
+ * @param[in] filename A name to user-given dump file.
+ */
+ void SetDumpFilename(const std::string filename) { mDumpFilename = filename; }
+
+private:
+ void PushData(ProfileData &data);
+ struct timespec GetTimeDiff(struct timespec &start, struct timespec &end);
+ unsigned long ConvertMillisec(const struct timespec &time);
+ void DumpToConsole(void);
+ void DumpToFile(const unsigned int dump_type, std::string filename);
+
+ struct timespec mStartTime, mEndTime;
+ unsigned int mEnvNum;
+ ProfileEnv mProfileEnv;
+ std::vector<ProfileEnv> v_mProfileEnv;
+ std::vector<ProfileData> v_mProfileData;
+ std::map<const char *, const void *> m_mDataTable;
+ std::string mDumpFilename;
+};
+} /* Profiler */
+} /* InferenceEngineInterface */
+
+#endif /* __INFERENCE_ENGINE_COMMON_H__ */
{
LOGI("ENTER");
- // TODO.
+ mUseProfiler = false;
LOGI("LEAVE");
}
InferenceEngineCommon::~InferenceEngineCommon()
{
LOGW("ENTER");
+
+ if (mUseProfiler == true) {
+ mProfiler->Dump(mProfilerDumpType);
+ delete mProfiler;
+ }
+
LOGW("LEAVE");
}
+int InferenceEngineCommon::EnableProfiler(bool enable)
+{
+ if (enable != true && enable != false) {
+ return INFERENCE_ENGINE_ERROR_INVALID_PARAMETER;
+ }
+
+ mUseProfiler = enable;
+
+ if (mUseProfiler == true) {
+ mProfiler = new InferenceEngineProfiler();
+
+ // In default, profile data will be stored to a given file.
+ mProfilerDumpType = IR_PROFILER_DUMP_FILE;
+ }
+
+ return INFERENCE_ENGINE_ERROR_NONE;
+}
+
+int InferenceEngineCommon::DumpProfileToConsole(void)
+{
+ if (mUseProfiler == false) {
+ std::cout << "Enable Profiler." << "\n";
+ return INFERENCE_ENGINE_ERROR_INVALID_PARAMETER;
+ }
+
+ mProfilerDumpType = IR_PROFILER_DUMP_CONSOLE;
+ return INFERENCE_ENGINE_ERROR_NONE;
+}
+
+int InferenceEngineCommon::DumpProfileToFile(const std::string filename)
+{
+ if (mUseProfiler == false) {
+ std::cout << "Enable Profiler." << "\n";
+ return INFERENCE_ENGINE_ERROR_INVALID_PARAMETER;
+ }
+
+ mProfilerDumpType = IR_PROFILER_DUMP_FILE;
+ mProfiler->SetDumpFilename(filename);
+
+ return INFERENCE_ENGINE_ERROR_NONE;
+}
+
int InferenceEngineCommon::BindBackend(inference_engine_config *config)
{
LOGI("ENTER");
return INFERENCE_ENGINE_ERROR_INTERNAL;
}
+ if (mUseProfiler == true) {
+ mProfiler->AddBackendName(config->backend_name);
+ }
+
LOGI("LEAVE");
return INFERENCE_ENGINE_ERROR_NONE;
if (ret != INFERENCE_ENGINE_ERROR_NONE)
LOGE("Fail to SetTargetDevice");
+ if (mUseProfiler == true) {
+ mProfiler->AddTargetDevices(types);
+ }
+
return ret;
}
{
LOGI("ENTER");
+ if (mUseProfiler == true) {
+ mProfiler->AddModelName(model_paths[0]);
+ mProfiler->PushEnv();
+ mProfiler->Start(IR_PROFILER_LATENCY);
+ }
+
int ret = mBackendHandle->Load(model_paths, model_format);
if (ret != INFERENCE_ENGINE_ERROR_NONE)
LOGE("Fail to load InferenceEngineVision");
+ if (mUseProfiler == true) {
+ mProfiler->Stop(IR_PROFILER_LATENCY, "Load");
+ }
+
LOGI("LEAVE");
return INFERENCE_ENGINE_ERROR_NONE;
int InferenceEngineCommon::Run(std::vector<inference_engine_tensor_buffer> &input_buffers,
std::vector<inference_engine_tensor_buffer> &output_buffers)
{
- return mBackendHandle->Run(input_buffers, output_buffers);
+ if (mUseProfiler == true) {
+ mProfiler->Start(IR_PROFILER_LATENCY);
+ }
+
+ int ret = mBackendHandle->Run(input_buffers, output_buffers);
+
+ if (mUseProfiler == true) {
+ mProfiler->Stop(IR_PROFILER_LATENCY, "Run");
+ }
+
+ return ret;
}
int InferenceEngineCommon::SetLibraryPath(std::string path)
--- /dev/null
+/**
+ * Copyright (c) 2020 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "inference_engine_error.h"
+#include "inference_engine_profiler.h"
+#include <fstream>
+#include <iostream>
+#include <time.h>
+
+extern "C" {
+
+#include <dlog.h>
+
+#ifdef LOG_TAG
+#undef LOG_TAG
+#endif
+
+#define LOG_TAG "INFERENCE_ENGINE_PROFILER"
+}
+
+#define NANO_PER_SEC ((__clock_t) 1000000000)
+#define NANO_PER_MILLI ((__clock_t) 1000000)
+#define MILLI_PER_SEC ((__clock_t) 1000)
+
+namespace InferenceEngineInterface {
+namespace Profiler {
+
+// In default, we will use Markdown syntax to print out profile data.
+static const std::string sTitleMarkdown("backend|target devices|model name|Function name|Latency(ms)|Memory Usage(kb)\n--|--|--|--|--|--\n");
+
+InferenceEngineProfiler::InferenceEngineProfiler()
+{
+ mStartTime = { 0, };
+ mEndTime = { 0, };
+ mEnvNum = 0;
+
+ // In default. we will store profile data to dump.txt file.
+ // If you want to use other file then use SetDumpFilename function to change the filename.
+ mDumpFilename = "dump.txt";
+}
+
+InferenceEngineProfiler::~InferenceEngineProfiler()
+{
+ v_mProfileEnv.clear();
+ v_mProfileData.clear();
+ m_mDataTable.clear();
+}
+
+void InferenceEngineProfiler::PushData(ProfileData &data)
+{
+ std::string key = std::to_string(mEnvNum - 1) + data.function_name;
+
+ // In case of multiple 'Run' per one 'Load', update just average value of measured ones instead of adding new one.
+ if (!m_mDataTable.empty()) {
+ std::map<const char *, const void *>::iterator iter;
+ iter = m_mDataTable.find(key.c_str());
+ if (iter != m_mDataTable.end()) {
+ ProfileData *item = (ProfileData *)iter->second;
+ item->elapsed_time = (item->elapsed_time + data.elapsed_time) >> 1;
+ item->memory_usage = (item->memory_usage + data.memory_usage) >> 1;
+ return;
+ }
+ }
+
+ v_mProfileData.push_back(data);
+ m_mDataTable.insert(std::make_pair<const char *, const void *>(key.c_str(), &v_mProfileData.back()));
+}
+
+struct timespec InferenceEngineProfiler::GetTimeDiff(struct timespec &start,
+ struct timespec &end)
+{
+ struct timespec temp;
+
+ if ((end.tv_nsec - start.tv_nsec) < 0) {
+ temp.tv_sec = end.tv_sec - start.tv_sec - 1;
+ temp.tv_nsec = NANO_PER_SEC + end.tv_nsec - start.tv_nsec;
+ }
+ else {
+ temp.tv_sec = end.tv_sec - start.tv_sec;
+ temp.tv_nsec = end.tv_nsec - start.tv_nsec;
+ }
+
+ return temp;
+}
+
+unsigned long InferenceEngineProfiler::ConvertMillisec(const struct timespec &time)
+{
+ mStartTime.tv_nsec = 0;
+ mStartTime.tv_sec = 0;
+ mEndTime.tv_nsec = 0;
+ mEndTime.tv_sec = 0;
+
+ return (unsigned long)(time.tv_sec * MILLI_PER_SEC + time.tv_nsec / NANO_PER_MILLI);
+}
+
+void InferenceEngineProfiler::Start(const unsigned int type)
+{
+ if (IR_PROFILER_MIN >= type && IR_PROFILER_MAX <= type) {
+ LOGE("Invalid profiler type.");
+ return;
+ }
+
+ switch (type) {
+ case IR_PROFILER_LATENCY:
+ clock_gettime(CLOCK_MONOTONIC, &mStartTime);
+ break;
+ case IR_PROFILER_MEMORY:
+ break;
+ /* TODO */
+ }
+}
+
+void InferenceEngineProfiler::Stop(const unsigned int type, const char *func_name)
+{
+ if (IR_PROFILER_MIN >= type && IR_PROFILER_MAX <= type) {
+ LOGE("Invalid profiler type.");
+ return;
+ }
+
+ ProfileData data = { mEnvNum - 1, func_name, 0, 0 };
+
+ switch (type) {
+ case IR_PROFILER_LATENCY: {
+ clock_gettime(CLOCK_MONOTONIC, &mEndTime);
+ data.elapsed_time = ConvertMillisec(GetTimeDiff(mStartTime, mEndTime));
+ break;
+ }
+ case IR_PROFILER_MEMORY:
+ break;
+ /* TODO */
+ }
+
+ PushData(data);
+}
+
+void InferenceEngineProfiler::DumpToConsole(void)
+{
+ std::cout << sTitleMarkdown;
+
+ std::vector<ProfileData>::iterator iter;
+ for (iter = v_mProfileData.begin(); iter != v_mProfileData.end(); iter++) {
+ ProfileData data = *iter;
+ ProfileEnv env = v_mProfileEnv[data.env_idx];
+ std::cout << env.backend_name << "|" << env.target_devices << "|" << env.model_name << "|";
+ std::cout << data.function_name << "|" << data.elapsed_time << "|" << "\n";
+ }
+ std::cout << "***" << "\n";
+}
+
+void InferenceEngineProfiler::DumpToFile(const unsigned int dump_type, std::string filename)
+{
+ if (mDumpFilename.empty())
+ mDumpFilename = filename;
+
+ std::ofstream dump_file;
+
+ dump_file.open(mDumpFilename, std::ios::binary | std::ios::app);
+ if (dump_file.is_open()) {
+ dump_file.write(sTitleMarkdown.c_str(), sTitleMarkdown.length());
+
+ std::vector<ProfileData>::iterator iter;
+ for (iter = v_mProfileData.begin(); iter != v_mProfileData.end(); iter++) {
+ ProfileData data = *iter;
+ ProfileEnv env = v_mProfileEnv[data.env_idx];
+ dump_file.write(env.backend_name.c_str(), env.backend_name.length());
+ dump_file.write("|", 1);
+ if (env.target_devices & INFERENCE_TARGET_CPU)
+ dump_file.write("CPU", 3);
+ if (env.target_devices & INFERENCE_TARGET_GPU)
+ dump_file.write("GPU", 3);
+ dump_file.write("|", 1);
+ dump_file.write(env.model_name.c_str(), env.model_name.length());
+ dump_file.write("|", 1);
+ dump_file.write(data.function_name.c_str(), data.function_name.length());
+ dump_file.write("|", 1);
+ std::string sElapsedTime(std::to_string(data.elapsed_time));
+ dump_file.write(sElapsedTime.c_str(), sElapsedTime.length());
+ dump_file.write("|", 1);
+ dump_file.write("\n", 1);
+ }
+ dump_file.write("***\n", 4);
+ }
+
+ dump_file.close();
+}
+
+void InferenceEngineProfiler::Dump(const unsigned int dump_type)
+{
+ if (IR_PROFILER_DUMP_MIN >= dump_type && IR_PROFILER_DUMP_MAX <= dump_type) {
+ LOGE("Invalid profiler dump type.");
+ return;
+ }
+
+ if (dump_type == IR_PROFILER_DUMP_CONSOLE) {
+ DumpToConsole();
+ } else {
+ DumpToFile(IR_PROFILER_DUMP_FORMAT_MARKDOWN, mDumpFilename);
+ }
+}
+
+} /* Profiler */
+} /* InferenceEngineInterface */
projects / platform / core / multimedia / inference-engine-interface.git / commitdiff