From 6a062af3e62703064891676c9734dc00ad16e822 Mon Sep 17 00:00:00 2001 From: Inki Dae Date: Thu, 1 Sep 2022 09:41:17 +0900 Subject: [PATCH] mv_machine_learning: Use Preprocess class instead of legacy one [Issue type] code refactoring Did code refactoring to Run function of Inference class by replacing existing internal Preprocess function with the one of Proprocess class. We can use Preprocess class instead of Inference class internal one. With this change, we can use Preprocess class commonly. Change-Id: Id22de1532ce352abb013ad6bb26075c74a835949 Signed-off-by: Inki Dae --- mv_machine_learning/inference/include/Inference.h | 2 +- mv_machine_learning/inference/src/Inference.cpp | 133 +++++++++------------- 2 files changed, 57 insertions(+), 78 deletions(-) diff --git a/mv_machine_learning/inference/include/Inference.h b/mv_machine_learning/inference/include/Inference.h index 7096094..84a63d2 100644 --- a/mv_machine_learning/inference/include/Inference.h +++ b/mv_machine_learning/inference/include/Inference.h @@ -351,8 +351,8 @@ namespace inference int ConvertTargetTypes(int given_types); int ConvertToCv(int given_type); int ConvertOutputDataTypeToFloat(); + int Preprocess(std::vector& mv_sources, std::vector& cv_sources); inference_tensor_data_type_e ConvertToIE(int given_type); - int Preprocess(cv::Mat cvImg, cv::Mat cvDst, int data_type); int PrepareTenosrBuffers(void); void CleanupTensorBuffers(void); int SetUserFile(std::string filename); diff --git a/mv_machine_learning/inference/src/Inference.cpp b/mv_machine_learning/inference/src/Inference.cpp index 4f426c7..6a5a6ca 100755 --- a/mv_machine_learning/inference/src/Inference.cpp +++ b/mv_machine_learning/inference/src/Inference.cpp @@ -279,51 +279,6 @@ namespace inference return type; } - int Inference::Preprocess(cv::Mat cvImg, cv::Mat cvDst, int data_type) - { - int width = mInputSize.width; - int height = mInputSize.height; - const int ch = mConfig.mTensorInfo.ch; - - cv::Mat sample; - if (cvImg.channels() == 3 && ch == 1) - cv::cvtColor(cvImg, sample, cv::COLOR_BGR2GRAY); - else - sample = cvImg; - - // size - cv::Mat sampleResized; - if (sample.size() != cv::Size(width, height)) - cv::resize(sample, sampleResized, cv::Size(width, height)); - else - sampleResized = sample; - - // type - cv::Mat sampleFloat; - if (ch == 3) - sampleResized.convertTo(sampleFloat, CV_32FC3); - else - sampleResized.convertTo(sampleFloat, CV_32FC1); - - // normalize - cv::Mat sampleNormalized; - cv::Mat meanMat; - auto mean = static_cast(mConfig.mMeanValue); - - if (ch == 3) - meanMat = cv::Mat(sampleFloat.size(), CV_32FC3, cv::Scalar(mean, mean, mean)); - else - meanMat = cv::Mat(sampleFloat.size(), CV_32FC1, cv::Scalar(mean)); - - cv::subtract(sampleFloat, meanMat, sampleNormalized); - - sampleNormalized /= static_cast(mConfig.mStdValue); - - sampleNormalized.convertTo(cvDst, data_type); - - return MEDIA_VISION_ERROR_NONE; - } - int Inference::SetUserFile(std::string filename) { std::ifstream fp(filename.c_str()); @@ -912,6 +867,58 @@ namespace inference return ConvertEngineErrorToVisionError(ret); } + int Inference::Preprocess(std::vector& mv_sources, std::vector& cv_sources) + { + unsigned int src_idx = 0; + + for (auto& buffer : mInputTensorBuffers.getIETensorBuffer()) { + inference_engine_tensor_buffer& tensor_buffer = buffer.second; + int data_type = ConvertToCv(tensor_buffer.data_type); + LayerInfo layerInfo; + Options opt; + mv_colorspace_e colorspace = MEDIA_VISION_COLORSPACE_INVALID; + + mv_source_get_colorspace(mv_sources[src_idx], &colorspace); + + if (mMetadata.GetInputMeta().IsParsed()) { + layerInfo = mMetadata.GetInputMeta().GetLayer().at(buffer.first); + + if (!mMetadata.GetInputMeta().GetOption().empty()) + opt = mMetadata.GetInputMeta().GetOption().at(buffer.first); + } else { + // Ps. in case of legacy way, there is no way to set model specific dequantization parameters - zero point and scale. + // TODO. find a proper way for it. + opt.normalization.use = true; + opt.normalization.mean.assign(3, mConfig.mMeanValue); + opt.normalization.std.assign(3, mConfig.mStdValue); + + layerInfo.name = buffer.first; + layerInfo.dims.push_back(mConfig.mTensorInfo.dim); + layerInfo.dims.push_back(mConfig.mTensorInfo.height); + layerInfo.dims.push_back(mConfig.mTensorInfo.width); + layerInfo.dims.push_back(mConfig.mTensorInfo.ch); + + // Ps. in case of legacy way, there is no way to use model specific color space but only fixed one. + // TODO. find a proper way for it. + layerInfo.colorSpace = MEDIA_VISION_COLORSPACE_RGB888; + layerInfo.dataType = mConfig.mDataType; + // TODO. find a proper way for setting the shape type. In case of legacy way, there is no way to change the shape type properly. + // According to a given inference engine, different shape type can be needed. + layerInfo.shapeType = INFERENCE_TENSOR_SHAPE_NHWC; + } + + // TODO: try-catch{} error handling + int ret = mPreProc.Run(cv_sources[src_idx++], colorspace, data_type, layerInfo, opt, tensor_buffer.buffer); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Fail to run pre-process."); + return ret; + } + } + + return MEDIA_VISION_ERROR_NONE; + + } + int Inference::Run(std::vector &mvSources, std::vector &rects) { @@ -955,38 +962,10 @@ namespace inference // TODO. consider multiple cv sources. mSourceSize = cvSources[0].size(); - unsigned int src_idx = 0; - - for (auto& buffer : mInputTensorBuffers.getIETensorBuffer()) { - inference_engine_tensor_buffer& tensor_buffer = buffer.second; - int data_type = ConvertToCv(tensor_buffer.data_type); - - if (mMetadata.GetInputMeta().IsParsed()) { - const LayerInfo& layerInfo = mMetadata.GetInputMeta().GetLayer().at(buffer.first); - Options opt; - - if (!mMetadata.GetInputMeta().GetOption().empty()) - opt = mMetadata.GetInputMeta().GetOption().at(buffer.first); - - mv_colorspace_e colorspace = MEDIA_VISION_COLORSPACE_INVALID; - - mv_source_get_colorspace(mvSources[src_idx], &colorspace); - - // TODO: try-catch{} error handling - ret = mPreProc.Run(cvSources[src_idx++], colorspace, data_type, layerInfo, opt, tensor_buffer.buffer); - if (ret != MEDIA_VISION_ERROR_NONE) { - LOGE("Fail to run pre-process."); - return ret; - } - } else { - // Convert color space of input tensor data and then normalize it. - - ret = Preprocess(cvSources[src_idx++], cv::Mat(mInputSize.height, mInputSize.width, data_type, tensor_buffer.buffer), data_type); - if (ret != MEDIA_VISION_ERROR_NONE) { - LOGE("Fail to preprocess input tensor data."); - return ret; - } - } + ret = Preprocess(mvSources, cvSources); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Fail to preprocess given input sources."); + return ret; } ret = mBackend->Run(mInputTensorBuffers.getIETensorBuffer(), -- 2.7.4