From: Alexander Alekhin Date: Fri, 3 Jul 2020 19:14:05 +0000 (+0000) Subject: dnn(test): add YOLOv4-tiny tests X-Git-Tag: submit/tizen/20210224.033012~2^2~104^2~11^2~2 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=99c4b76a6d4df9159e3e75bfcb3ad989bc40222c;p=platform%2Fupstream%2Fopencv.git dnn(test): add YOLOv4-tiny tests --- diff --git a/modules/dnn/perf/perf_net.cpp b/modules/dnn/perf/perf_net.cpp index f4d008f..3bee231 100644 --- a/modules/dnn/perf/perf_net.cpp +++ b/modules/dnn/perf/perf_net.cpp @@ -216,6 +216,19 @@ PERF_TEST_P_(DNNTestNetwork, YOLOv4) processNet("dnn/yolov4.weights", "dnn/yolov4.cfg", "", inp); } +PERF_TEST_P_(DNNTestNetwork, YOLOv4_tiny) +{ + if (backend == DNN_BACKEND_HALIDE) + throw SkipTestException(""); + if (target == DNN_TARGET_MYRIAD) + throw SkipTestException(""); + Mat sample = imread(findDataFile("dnn/dog416.png")); + cvtColor(sample, sample, COLOR_BGR2RGB); + Mat inp; + sample.convertTo(inp, CV_32FC3, 1.0f / 255, 0); + processNet("dnn/yolov4-tiny.weights", "dnn/yolov4-tiny.cfg", "", inp); +} + PERF_TEST_P_(DNNTestNetwork, EAST_text_detection) { if (backend == DNN_BACKEND_HALIDE) diff --git a/modules/dnn/test/test_common.impl.hpp b/modules/dnn/test/test_common.impl.hpp index fdd1fe2..559b74f 100644 --- a/modules/dnn/test/test_common.impl.hpp +++ b/modules/dnn/test/test_common.impl.hpp @@ -96,9 +96,12 @@ void normAssertDetections( const char *comment /*= ""*/, double confThreshold /*= 0.0*/, double scores_diff /*= 1e-5*/, double boxes_iou_diff /*= 1e-4*/) { + ASSERT_FALSE(testClassIds.empty()) << "No detections"; std::vector matchedRefBoxes(refBoxes.size(), false); + std::vector refBoxesIoUDiff(refBoxes.size(), 1.0); for (int i = 0; i < testBoxes.size(); ++i) { + //cout << "Test[i=" << i << "]: score=" << testScores[i] << " id=" << testClassIds[i] << " box " << testBoxes[i] << endl; double testScore = testScores[i]; if (testScore < confThreshold) continue; @@ -115,6 +118,7 @@ void normAssertDetections( double interArea = (testBox & refBoxes[j]).area(); double iou = interArea / (testBox.area() + refBoxes[j].area() - interArea); topIoU = std::max(topIoU, iou); + refBoxesIoUDiff[j] = std::min(refBoxesIoUDiff[j], 1.0f - iou); if (1.0 - iou < boxes_iou_diff) { matched = true; @@ -137,7 +141,9 @@ void normAssertDetections( if (!matchedRefBoxes[i] && refScores[i] > confThreshold) { std::cout << cv::format("Unmatched reference: class %d score %f box ", - refClassIds[i], refScores[i]) << refBoxes[i] << std::endl; + refClassIds[i], refScores[i]) << refBoxes[i] + << " IoU diff: " << refBoxesIoUDiff[i] + << std::endl; EXPECT_LE(refScores[i], confThreshold) << comment; } } diff --git a/modules/dnn/test/test_darknet_importer.cpp b/modules/dnn/test/test_darknet_importer.cpp index fcc9088..f328b29 100644 --- a/modules/dnn/test/test_darknet_importer.cpp +++ b/modules/dnn/test/test_darknet_importer.cpp @@ -254,6 +254,13 @@ public: } + if (cvIsNaN(iouDiff)) + { + if (b == 0) + std::cout << "Skip accuracy checks" << std::endl; + continue; + } + normAssertDetections(refClassIds[b], refConfidences[b], refBoxes[b], nms_classIds, nms_confidences, nms_boxes, format("batch size %d, sample %d\n", batch_size, b).c_str(), confThreshold, scoreDiff, iouDiff); } @@ -449,7 +456,7 @@ TEST_P(Test_Darknet_nets_async, Accuracy) } INSTANTIATE_TEST_CASE_P(/**/, Test_Darknet_nets_async, Combine( - Values("yolo-voc", "tiny-yolo-voc", "yolov3", "yolov4"), + Values("yolo-voc", "tiny-yolo-voc", "yolov3", "yolov4", "yolov4-tiny"), dnnBackendsAndTargets() )); @@ -587,6 +594,63 @@ TEST_P(Test_Darknet_nets, YOLOv4) } } +TEST_P(Test_Darknet_nets, YOLOv4_tiny) +{ + applyTestTag( + target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB + ); + + const double confThreshold = 0.5; + // batchId, classId, confidence, left, top, right, bottom + const int N0 = 2; + const int N1 = 3; + static const float ref_[/* (N0 + N1) * 7 */] = { +0, 7, 0.85935f, 0.593484f, 0.141211f, 0.920356f, 0.291593f, +0, 16, 0.795188f, 0.169207f, 0.386886f, 0.423753f, 0.933004f, + +1, 2, 0.996832f, 0.653802f, 0.464573f, 0.815193f, 0.653292f, +1, 2, 0.963325f, 0.451151f, 0.458915f, 0.496255f, 0.52241f, +1, 0, 0.926244f, 0.194851f, 0.361743f, 0.260277f, 0.632364f, + }; + Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_); + + double scoreDiff = 0.01f; + double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.15 : 0.01f; + + std::string config_file = "yolov4-tiny.cfg"; + std::string weights_file = "yolov4-tiny.weights"; + +#if defined(INF_ENGINE_RELEASE) + if (target == DNN_TARGET_MYRIAD) // bad accuracy + iouDiff = std::numeric_limits::quiet_NaN(); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL) + iouDiff = std::numeric_limits::quiet_NaN(); + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && DNN_TARGET_OPENCL_FP16) + iouDiff = std::numeric_limits::quiet_NaN(); +#endif + + { + SCOPED_TRACE("batch size 1"); + testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff, confThreshold); + } + + { + SCOPED_TRACE("batch size 2"); + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, confThreshold); + } + +#if defined(INF_ENGINE_RELEASE) + if (target == DNN_TARGET_MYRIAD) // bad accuracy + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif +} + INSTANTIATE_TEST_CASE_P(/**/, Test_Darknet_nets, dnnBackendsAndTargets());