From: Tae-Young Chung Date: Wed, 4 Mar 2020 05:49:12 +0000 (+0900) Subject: Refactoring code according to refactored inferenc-engine-interface X-Git-Tag: submit/tizen/20200423.063253~4 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=f448c615a8b94c8efacf4e371a4523399e51c40f;p=platform%2Fcore%2Fmultimedia%2Finference-engine-opencv.git Refactoring code according to refactored inferenc-engine-interface NOTE that OpenCV doesn't support to get information of input layer. The input should be set by calling cv::dnn::Net::setInput() API. OpenCV allocates memory on the first cv::dnn::Net::forward(). After that, it reuse the allocated memory. So, in this refactoring, Create mInputData as cv::Mat type for input tensors and returns their allocated memories. Allocate the memories of mOutputBlobs by calling forward() with a dummy tensors. Signed-off-by: Tae-Young Chung --- diff --git a/src/inference_engine_opencv.cpp b/src/inference_engine_opencv.cpp index d0cfdc4..924d634 100644 --- a/src/inference_engine_opencv.cpp +++ b/src/inference_engine_opencv.cpp @@ -27,11 +27,8 @@ namespace InferenceEngineImpl { namespace OpenCVImpl { -InferenceOpenCV::InferenceOpenCV(std::string protoFile, std::string weightFile) : - mNet(), - mConfigFile(protoFile), - mWeightFile(weightFile), - mInputData(cv::Mat()) +InferenceOpenCV::InferenceOpenCV(void) : + mNet() { LOGE("ENTER"); LOGE("LEAVE"); @@ -42,37 +39,12 @@ InferenceOpenCV::~InferenceOpenCV() ; } -int InferenceOpenCV::SetInputTensorParam() +int InferenceOpenCV::SetTargetDevices(int types) { - LOGE("Not supported"); - return INFERENCE_ENGINE_ERROR_NOT_SUPPORTED; -} - -int InferenceOpenCV::SetInputTensorParamNode(std::string node) -{ - mInputLayer = cv::String(node); - return INFERENCE_ENGINE_ERROR_NONE; -} + LOGI("ENTER"); -int InferenceOpenCV::SetOutputTensorParam() -{ - return INFERENCE_ENGINE_ERROR_NOT_SUPPORTED; -} - -int InferenceOpenCV::SetOutputTensorParamNodes(std::vector nodes) -{ - mOutputLayer.clear(); - for (std::vector::iterator iter = nodes.begin(); - iter != nodes.end(); ++iter) { - mOutputLayer.push_back(cv::String(*iter)); - } - - return INFERENCE_ENGINE_ERROR_NONE; -} - -int InferenceOpenCV::SetTargetDevice(inference_target_type_e type) -{ - switch (type) { + LOGI("Inferece targets are: "); + switch (types) { case INFERENCE_TARGET_CPU : mNet.setPreferableTarget(cv::dnn::DNN_TARGET_CPU); break; @@ -82,31 +54,41 @@ int InferenceOpenCV::SetTargetDevice(inference_target_type_e type) case INFERENCE_TARGET_CUSTOM: case INFERENCE_TARGET_NONE: default: - LOGE("Not supported device type [%d], Set CPU mode", (int)type); + LOGE("Not supported device type [%d], Set CPU mode", (int)types); mNet.setPreferableTarget(cv::dnn::DNN_TARGET_CPU); } + LOGI("LEAVE"); return INFERENCE_ENGINE_ERROR_NONE; } -int InferenceOpenCV::Load() +int InferenceOpenCV::Load(std::vector model_paths, inference_model_format_e model_format) { + LOGI("ENTER"); + int ret = INFERENCE_ENGINE_ERROR_NONE; - if (access(mConfigFile.c_str(), F_OK) || - access(mWeightFile.c_str(), F_OK)) { - LOGE("protofile in [%s] ", mConfigFile.c_str()); - LOGE("weightFilePath in [%s] ", mWeightFile.c_str()); - return INFERENCE_ENGINE_ERROR_INVALID_PATH; - } + std::string fileExt; + for (std::vector::iterator iter = model_paths.begin(); + iter != model_paths.end(); ++iter) { + if (access((*iter).c_str(), F_OK)) { + LOGE("model path in [%s] not exist", (*iter).c_str()); + return INFERENCE_ENGINE_ERROR_INVALID_PATH; + } + fileExt = (*iter).substr(((*iter).find_last_of("."))+1); + + if (fileExt.compare("caffemodel") == 0 || + fileExt.compare("pb") == 0) { + mWeightFile = (*iter); + } else { + mConfigFile = (*iter); + } + } // This call may be changed if OpenCV version would be upgraded - int nPos = mWeightFile.find_last_of("."); - std::string weightFileExt = mWeightFile.substr(nPos+1); - LOGI("%s", weightFileExt.c_str()); - if (weightFileExt.compare("caffemodel") == 0) { + if (model_format == INFERENCE_MODEL_CAFFE) { mNet = cv::dnn::readNetFromCaffe(mConfigFile, mWeightFile); - } else if (weightFileExt.compare("pb") == 0) { + } else if (model_format == INFERENCE_MODEL_TF) { mNet = cv::dnn::readNetFromTensorflow(mWeightFile, mConfigFile); } else { LOGE("Not supported model file!"); @@ -117,80 +99,225 @@ int InferenceOpenCV::Load() return INFERENCE_ENGINE_ERROR_INVALID_DATA; } + LOGI("LEAVE"); + return ret; } -int InferenceOpenCV::CreateInputLayerPassage() +int InferenceOpenCV::GetInputTensorBuffers(std::vector &buffers) { - /* Do Nothing - * In OpenCV, don't need to call this CreateInputLayerPassage() - * because that it can uses cv::Mat directly - */ + LOGI("ENTER"); + + mInputData.clear(); + + void * pBuff = NULL; + std::vector::iterator info_iter; + for (info_iter = mInputTensorInfo.begin(); + info_iter != mInputTensorInfo.end(); ++info_iter) { + cv::Mat inputBlob(cv::Size((*info_iter).shape[3], (*info_iter).shape[2]), CV_32FC3); + mInputData.push_back(inputBlob); + + pBuff = mInputData.back().ptr(0); + inference_engine_tensor_buffer buffer = { pBuff, TENSOR_DATA_TYPE_FLOAT32 }; + buffers.push_back(buffer); + } + + LOGI("LEAVE"); return INFERENCE_ENGINE_ERROR_NONE; } - -int InferenceOpenCV::GetInputLayerAttrType() +int InferenceOpenCV::GetOutputTensorBuffers(std::vector &buffers) { - return 1; + LOGI("ENTER"); + + + mInputBlobs = cv::dnn::blobFromImages(mInputData, 1.0, cv::Size(), cv::Scalar(), false, false); + + mNet.setInput(mInputBlobs, mInputLayers.front()); + + std::vector ouputLayers(mOutputLayers.begin(), mOutputLayers.end()); + mNet.forward(mOutputBlobs, ouputLayers); + + void *pBuff = NULL; + std::vector::iterator iter; + for (iter = mOutputBlobs.begin(); iter != mOutputBlobs.end(); ++iter) { + pBuff = (*iter).ptr(0); + inference_engine_tensor_buffer buffer = { pBuff, TENSOR_DATA_TYPE_FLOAT32 }; + buffers.push_back(buffer); + } + + LOGI("LEAVE"); + + return INFERENCE_ENGINE_ERROR_NONE; } -void * InferenceOpenCV::GetInputDataPtr() +int InferenceOpenCV::GetInputLayerProperty(inference_engine_layer_property &property) { - return static_cast(mInputData.data); + LOGI("ENTER"); + + if (mInputLayers.empty()) { + return INFERENCE_ENGINE_ERROR_INVALID_OPERATION; + } + + property.layer_names = mInputLayers; + property.tensor_infos = mInputTensorInfo; + + LOGI("LEAVE"); + + return INFERENCE_ENGINE_ERROR_NONE; } -int InferenceOpenCV::SetInputDataBuffer(tensor_t data) +int InferenceOpenCV::GetOutputLayerProperty(inference_engine_layer_property &property) { - mInputData = cv::Mat(data.dimInfo[0][2], data.dimInfo[0][3], data.dimInfo[0][1]); + LOGI("ENTER"); + + if (mOutputLayers.empty()) { + return INFERENCE_ENGINE_ERROR_INVALID_OPERATION; + } + + int lid = -1; + int idx = 0; + std::vector().swap(mOutputTensorInfo); + for (std::vector::iterator iter = mOutputLayers.begin(); + iter != mOutputLayers.end(); ++iter, ++idx) { + LOGI("output layer: %s", (*iter).c_str()); + lid = mNet.getLayerId((*iter)); + LOGI("output layer Id: %d", lid); + if(lid < 0) { + LOGE("Invalid output %s layer", (*iter).c_str()); + return INFERENCE_ENGINE_ERROR_INVALID_OPERATION; + } + + std::vector lInputShape, lOutputShape; + LOGI("%d, %d, %d, %d", mInputTensorInfo[idx].shape[0], + mInputTensorInfo[idx].shape[1], + mInputTensorInfo[idx].shape[2], + mInputTensorInfo[idx].shape[3]); + + mNet.getLayerShapes(mInputTensorInfo[idx].shape, + lid, + lInputShape, + lOutputShape); + inference_engine_tensor_info tensor_info; + tensor_info.data_type =TENSOR_DATA_TYPE_FLOAT32; + tensor_info.shape_type = TENSOR_SHAPE_NCHW; + // lOutputShape may have multiple tensors + // even though the output layer's name is only one + LOGI("size of OutputShape: %d", lOutputShape.size()); + tensor_info.shape = lOutputShape[0]; + + tensor_info.size = 1; + LOGE("tensor_info"); + for (std::vector::iterator iter2 = tensor_info.shape.begin(); + iter2 != tensor_info.shape.end(); ++iter2) { + LOGI("%d", (*iter2)); + tensor_info.size *= (*iter2); + } + mOutputTensorInfo.push_back(tensor_info); + } + + property.layer_names = mOutputLayers; + property.tensor_infos = mOutputTensorInfo; + + LOGI("LEAVE"); + return INFERENCE_ENGINE_ERROR_NONE; } -int InferenceOpenCV::Run() +int InferenceOpenCV::SetInputLayerProperty(inference_engine_layer_property &property) { - mInputBlob = cv::dnn::blobFromImage(mInputData, 1.0, cv::Size(), cv::Scalar(), false, false); + LOGI("ENTER"); + + std::vector::iterator iter; + for (iter = property.layer_names.begin(); iter != property.layer_names.end(); iter++) { + std::string name = *iter; + LOGI("input layer name = %s", name.c_str()); + } + + mInputLayers.clear(); + std::vector().swap(mInputLayers); + + mInputTensorInfo.clear(); + std::vector().swap(mInputTensorInfo); + + mInputLayers = property.layer_names; + mInputTensorInfo = property.tensor_infos; - mNet.setInput(mInputBlob, mInputLayer); - mNet.forward(mOutputProb, mOutputLayer); + LOGI("LEAVE"); - if (mOutputProb.empty()) { - LOGE("OutputProb is empty"); - return INFERENCE_ENGINE_ERROR_INTERNAL; + return INFERENCE_ENGINE_ERROR_NONE; +} + +int InferenceOpenCV::SetOutputLayerProperty(inference_engine_layer_property &property) +{ + std::vector::iterator iter; + for (iter = property.layer_names.begin(); iter != property.layer_names.end(); iter++) { + std::string name = *iter; + LOGI("output layer name = %s", name.c_str()); } + mOutputLayers.clear(); + std::vector().swap(mOutputLayers); + + mOutputLayers = property.layer_names; + + return INFERENCE_ENGINE_ERROR_NONE; } -int InferenceOpenCV::Run(std::vector tensor) +int InferenceOpenCV::GetBackendCapacity(inference_engine_capacity *capacity) { + LOGI("ENTER"); + + if (capacity == NULL) { + LOGE("Bad pointer."); + return INFERENCE_ENGINE_ERROR_INVALID_PARAMETER; + } + + capacity->supported_accel_devices = INFERENCE_TARGET_CPU; + + LOGI("LEAVE"); + return INFERENCE_ENGINE_ERROR_NONE; } -int InferenceOpenCV::GetInferenceResult(tensor_t& results) +int InferenceOpenCV::Run(std::vector &input_buffers, + std::vector &output_buffers) { - std::vector tmpDimInfo; - LOGE("outputProb size: %d", mOutputProb.size()); - for (std::vector::iterator iter = mOutputProb.begin(); - iter != mOutputProb.end(); ++iter) { - tmpDimInfo.clear(); - for (int d = 0; d < (*iter).dims; ++d) { - tmpDimInfo.push_back((*iter).size[d]); - } + LOGI("ENTER"); + + // need to check memoery + mInputBlobs = cv::dnn::blobFromImages(mInputData, 1.0, cv::Size(), cv::Scalar(), false, false); - results.dimInfo.push_back(tmpDimInfo); - results.data.push_back((void*)(*iter).ptr()); + // Currently it supports that one input layer with multiple input tensors. + // it doesn't support that mulitple input layer with multiple input tensors. + // To suppor that, setInput is called manually while we matching inputblobs + // and their corresponding input layer. + // Suppose a case that an input layer and mulitple input tensors are given. + mNet.setInput(mInputBlobs, mInputLayers.front()); + + int idx = 0; + + if (mOutputBlobs.size() != output_buffers.size()) { + LOGE("output_buffers size is %d but outputBlobs %d", output_buffers.size(), mOutputBlobs.size()); + return INFERENCE_ENGINE_ERROR_INVALID_PARAMETER; } + std::vector ouputLayers(mOutputLayers.begin(), mOutputLayers.end()); + + mNet.forward(mOutputBlobs, ouputLayers); + LOGI("LEAVE"); + return INFERENCE_ENGINE_ERROR_NONE; } extern "C" { -class IInferenceEngineCommon* EngineCommonInit(std::string protoFile, std::string weightFile) +class IInferenceEngineCommon* EngineCommonInit(void) { - InferenceOpenCV *engine = new InferenceOpenCV(protoFile, weightFile); + InferenceOpenCV *engine = new InferenceOpenCV(); return engine; } diff --git a/src/inference_engine_opencv_private.h b/src/inference_engine_opencv_private.h index c2992c7..1ab7645 100644 --- a/src/inference_engine_opencv_private.h +++ b/src/inference_engine_opencv_private.h @@ -19,10 +19,10 @@ #include +#include + #include #include - -#include /** * @file inference_engine_opencv_private.h * @brief This file contains the InferenceOpenCV class which @@ -42,52 +42,44 @@ namespace OpenCVImpl { class InferenceOpenCV : public IInferenceEngineCommon { public: - InferenceOpenCV(std::string protoFile, - std::string weightFile); - + InferenceOpenCV(); ~InferenceOpenCV(); - // Input Tensor Params - - int SetInputTensorParam() override; - - int SetInputTensorParamNode(std::string node = "input") override; + int SetTargetDevices(int types) override; - // Output Tensor Params - int SetOutputTensorParam() override; + int Load(std::vector model_paths, inference_model_format_e model_format) override; - int SetOutputTensorParamNodes(std::vector nodes) override; + int GetInputTensorBuffers(std::vector &buffers) override; - int SetTargetDevice(inference_target_type_e type) override; + int GetOutputTensorBuffers(std::vector &buffers) override; - int Load() override; + int GetInputLayerProperty(inference_engine_layer_property &property) override; - int CreateInputLayerPassage() override; + int GetOutputLayerProperty(inference_engine_layer_property &property) override; - int GetInputLayerAttrType() override; + int SetInputLayerProperty(inference_engine_layer_property &property) override; - void * GetInputDataPtr() override; + int SetOutputLayerProperty(inference_engine_layer_property &property) override; - int SetInputDataBuffer(tensor_t data) override; + int GetBackendCapacity(inference_engine_capacity *capacity) override; - int Run() override; - - int Run(std::vector tensor) override; - - int GetInferenceResult(tensor_t& data) override; + int Run(std::vector &input_buffers, + std::vector &output_buffers) override; private: - cv::Mat mInputBlob; - std::vector mOutputProb; + std::vector mInputData; + cv::Mat mInputBlobs; + + std::vector mInputTensorInfo; + std::vector mOutputTensorInfo; + std::vector mOutputBlobs; cv::dnn::Net mNet; /**< Network associated with a network model */ - cv::String mInputLayer; - std::vector mOutputLayer; + std::vector mInputLayers; + std::vector mOutputLayers; std::string mConfigFile; std::string mWeightFile; - - cv::Mat mInputData; }; } /* InferenceEngineImpl */