From e7007668a6bf3fe660628600e78e424f06de9773 Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Mon, 29 Mar 2021 23:59:55 +0300 Subject: [PATCH] Merge pull request #19792 from mpashchenkov:mp/ie-add-int32 G-API: IE. Adding support for INT32 type. * Added support for int32 * Added sample for semantic-segmentation-adas-0001 * Alignment * Alignment 2 * Rstrt build * Removed test for sem seg --- modules/gapi/samples/semantic_segmentation.cpp | 133 +++++++++++++++++++++++++ modules/gapi/src/backends/ie/giebackend.cpp | 4 + 2 files changed, 137 insertions(+) create mode 100644 modules/gapi/samples/semantic_segmentation.cpp diff --git a/modules/gapi/samples/semantic_segmentation.cpp b/modules/gapi/samples/semantic_segmentation.cpp new file mode 100644 index 0000000..0a6e723 --- /dev/null +++ b/modules/gapi/samples/semantic_segmentation.cpp @@ -0,0 +1,133 @@ +#include +#include +#include +#include +#include + +const std::string keys = + "{ h help | | Print this help message }" + "{ input | | Path to the input video file }" + "{ output | | Path to the output video file }" + "{ ssm | semantic-segmentation-adas-0001.xml | Path to OpenVINO IE semantic segmentation model (.xml) }"; + +// 20 colors for 20 classes of semantic-segmentation-adas-0001 +const std::vector colors = { + { 128, 64, 128 }, + { 232, 35, 244 }, + { 70, 70, 70 }, + { 156, 102, 102 }, + { 153, 153, 190 }, + { 153, 153, 153 }, + { 30, 170, 250 }, + { 0, 220, 220 }, + { 35, 142, 107 }, + { 152, 251, 152 }, + { 180, 130, 70 }, + { 60, 20, 220 }, + { 0, 0, 255 }, + { 142, 0, 0 }, + { 70, 0, 0 }, + { 100, 60, 0 }, + { 90, 0, 0 }, + { 230, 0, 0 }, + { 32, 11, 119 }, + { 0, 74, 111 }, +}; + +namespace { +std::string get_weights_path(const std::string &model_path) { + const auto EXT_LEN = 4u; + const auto sz = model_path.size(); + CV_Assert(sz > EXT_LEN); + + auto ext = model_path.substr(sz - EXT_LEN); + std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){ + return static_cast(std::tolower(c)); + }); + CV_Assert(ext == ".xml"); + return model_path.substr(0u, sz - EXT_LEN) + ".bin"; +} +} // anonymous namespace + +namespace custom { +G_API_OP(PostProcessing, , "sample.custom.post_processing") { + static cv::GMatDesc outMeta(const cv::GMatDesc &in, const cv::GMatDesc &) { + return in; + } +}; + +GAPI_OCV_KERNEL(OCVPostProcessing, PostProcessing) { + static void run(const cv::Mat &in, const cv::Mat &detected_classes, cv::Mat &out) { + // This kernel constructs output image by class table and colors vector + + // The semantic-segmentation-adas-0001 output a blob with the shape + // [B, C=1, H=1024, W=2048] + const int outHeight = 1024; + const int outWidth = 2048; + cv::Mat maskImg(outHeight, outWidth, CV_8UC3); + const int* const classes = detected_classes.ptr(); + for (int rowId = 0; rowId < outHeight; ++rowId) { + for (int colId = 0; colId < outWidth; ++colId) { + size_t classId = static_cast(classes[rowId * outWidth + colId]); + maskImg.at(rowId, colId) = + classId < colors.size() + ? colors[classId] + : cv::Vec3b{0, 0, 0}; // sample detects 20 classes + } + } + cv::resize(maskImg, out, in.size()); + const float blending = 0.3f; + out = in * blending + out * (1 - blending); + } +}; +} // namespace custom + +int main(int argc, char *argv[]) { + cv::CommandLineParser cmd(argc, argv, keys); + if (cmd.has("help")) { + cmd.printMessage(); + return 0; + } + + // Prepare parameters first + const std::string input = cmd.get("input"); + const std::string output = cmd.get("output"); + const auto model_path = cmd.get("ssm"); + const auto weights_path = get_weights_path(model_path); + const auto device = "CPU"; + G_API_NET(SemSegmNet, , "semantic-segmentation"); + const auto net = cv::gapi::ie::Params { + model_path, weights_path, device + }; + const auto kernels = cv::gapi::kernels(); + const auto networks = cv::gapi::networks(net); + + // Now build the graph + cv::GMat in; + cv::GMat detected_classes = cv::gapi::infer(in); + cv::GMat out = custom::PostProcessing::on(in, detected_classes); + + cv::GStreamingCompiled pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) + .compileStreaming(cv::compile_args(kernels, networks)); + auto inputs = cv::gin(cv::gapi::wip::make_src(input)); + + // The execution part + pipeline.setSource(std::move(inputs)); + pipeline.start(); + + cv::VideoWriter writer; + cv::Mat outMat; + while (pipeline.pull(cv::gout(outMat))) { + cv::imshow("Out", outMat); + cv::waitKey(1); + if (!output.empty()) { + if (!writer.isOpened()) { + const auto sz = cv::Size{outMat.cols, outMat.rows}; + writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz); + CV_Assert(writer.isOpened()); + } + writer << outMat; + } + } + return 0; +} diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index d4131c0..6cf77ac 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -104,6 +104,7 @@ inline IE::Layout toIELayout(const std::size_t ndims) { inline IE::Precision toIE(int depth) { switch (depth) { case CV_8U: return IE::Precision::U8; + case CV_32S: return IE::Precision::I32; case CV_32F: return IE::Precision::FP32; default: GAPI_Assert(false && "Unsupported data type"); } @@ -113,6 +114,7 @@ inline int toCV(IE::Precision prec) { switch (prec) { case IE::Precision::U8: return CV_8U; case IE::Precision::FP32: return CV_32F; + case IE::Precision::I32: return CV_32S; default: GAPI_Assert(false && "Unsupported data type"); } return -1; @@ -154,6 +156,7 @@ inline IE::Blob::Ptr wrapIE(const cv::Mat &mat, cv::gapi::ie::TraitAs hint) { case CV_##E: return IE::make_shared_blob(tDesc, const_cast(mat.ptr())) HANDLE(8U, uint8_t); HANDLE(32F, float); + HANDLE(32S, int); #undef HANDLE default: GAPI_Assert(false && "Unsupported data type"); } @@ -189,6 +192,7 @@ inline void copyFromIE(const IE::Blob::Ptr &blob, MatType &mat) { break; HANDLE(U8, uint8_t); HANDLE(FP32, float); + HANDLE(I32, int); #undef HANDLE default: GAPI_Assert(false && "Unsupported data type"); } -- 2.7.4