From acc089ca64273b37b1a248bc68288efaffa1e92c Mon Sep 17 00:00:00 2001 From: Polina Smolnikova <43805563+rayonnant14@users.noreply.github.com> Date: Sun, 26 Jan 2020 22:18:42 +0300 Subject: [PATCH] Merge pull request #15338 from rayonnant14:my_detect_and_decode_3.4 QR-Code detector : multiple detection * change in qr-codes detection * change in qr-codes detection * change in test * change in test * add multiple detection * multiple detection * multiple detect * add parallel implementation * add functional for performance tests * change in test * add perftest * returned implementation for 1 qr-code, added support for vector and vector> in MultipleDetectAndDecode * deleted all lambda expressions * changing in triangle sort * fixed warnings * fixed errors * add java and python tests * change in java tests * change in java and python tests * change in perf test * change in qrcode.cpp * add spaces * change in qrcode.cpp * change in qrcode.cpp * change in qrcode.cpp * change in java tests * change in java tests * solved problems * solved problems * change in java and python tests * change in python tests * change in python tests * change in python tests * change in methods name * deleted sample qrcode_multi, change in qrcode.cpp * change in perf tests * change in objdetect.hpp * deleted code duplication in sample qrcode.cpp * returned spaces * added spaces * deleted draw function * change in qrcode.cpp * change in qrcode.cpp * deleted all draw functions * objdetect(QR): extractVerticalLines * objdetect(QR): whitespaces * objdetect(QR): simplify operations, avoid duplicated code * change in interface, additional checks in java and python tests, added new key in sample for saving original image from camera * fix warnings and errors in python test * fix * write in file with space key * solved error with empty mat check in python test * correct path to test image * deleted spaces * solved error with check empty mat in python tests * added check of empty vector of points * samples: rework qrcode.cpp * objdetect(QR): fix API, input parameters must be first * objdetect(QR): test/fix points layout --- modules/java/generator/gen_java.py | 5 +- modules/objdetect/include/opencv2/objdetect.hpp | 60 +- .../misc/java/test/QRCodeDetectorTest.java | 20 + .../misc/python/test/test_qrcode_detect.py | 36 +- modules/objdetect/perf/perf_qrcode_pipeline.cpp | 57 + modules/objdetect/src/qrcode.cpp | 1441 ++++++++++++++++++-- modules/objdetect/test/test_qrcode.cpp | 153 ++- samples/cpp/qrcode.cpp | 328 +++-- 8 files changed, 1866 insertions(+), 234 deletions(-) diff --git a/modules/java/generator/gen_java.py b/modules/java/generator/gen_java.py index d98a98f..ea1b89e 100755 --- a/modules/java/generator/gen_java.py +++ b/modules/java/generator/gen_java.py @@ -914,7 +914,10 @@ class JavaWrapperGenerator(object): c_epilogue.append("Mat* _retval_ = new Mat();") c_epilogue.append(fi.ctype+"_to_Mat(_ret_val_vector_, *_retval_);") else: - c_epilogue.append("return " + fi.ctype + "_to_List(env, _ret_val_vector_);") + if ret: + c_epilogue.append("jobject _retval_ = " + fi.ctype + "_to_List(env, _ret_val_vector_);") + else: + c_epilogue.append("return " + fi.ctype + "_to_List(env, _ret_val_vector_);") if fi.classname: if not fi.ctype: # c-tor retval = fi.fullClass(isCPP=True) + "* _retval_ = " diff --git a/modules/objdetect/include/opencv2/objdetect.hpp b/modules/objdetect/include/opencv2/objdetect.hpp index cc9c4e1..5bd6a11 100644 --- a/modules/objdetect/include/opencv2/objdetect.hpp +++ b/modules/objdetect/include/opencv2/objdetect.hpp @@ -694,8 +694,8 @@ public: CV_WRAP bool detect(InputArray img, OutputArray points) const; /** @brief Decodes QR code in image once it's found by the detect() method. - Returns UTF8-encoded output string or empty string if the code cannot be decoded. + Returns UTF8-encoded output string or empty string if the code cannot be decoded. @param img grayscale or color (BGR) image containing QR code. @param points Quadrangle vertices found by detect() method (or some other algorithm). @param straight_qrcode The optional output image containing rectified and binarized QR code @@ -705,11 +705,44 @@ public: /** @brief Both detects and decodes QR code @param img grayscale or color (BGR) image containing QR code. - @param points opiotnal output array of vertices of the found QR code quadrangle. Will be empty if not found. + @param points optional output array of vertices of the found QR code quadrangle. Will be empty if not found. @param straight_qrcode The optional output image containing rectified and binarized QR code */ CV_WRAP cv::String detectAndDecode(InputArray img, OutputArray points=noArray(), OutputArray straight_qrcode = noArray()); + /** @brief Detects QR codes in image and returns the vector of the quadrangles containing the codes. + @param img grayscale or color (BGR) image containing (or not) QR codes. + @param points Output vector of vector of vertices of the minimum-area quadrangle containing the codes. + */ + CV_WRAP + bool detectMulti(InputArray img, OutputArray points) const; + + /** @brief Decodes QR codes in image once it's found by the detect() method. + @param img grayscale or color (BGR) image containing QR codes. + @param decoded_info UTF8-encoded output vector of string or empty vector of string if the codes cannot be decoded. + @param points vector of Quadrangle vertices found by detect() method (or some other algorithm). + @param straight_qrcode The optional output vector of images containing rectified and binarized QR codes + */ + CV_WRAP + bool decodeMulti( + InputArray img, InputArray points, + CV_OUT std::vector& decoded_info, + OutputArrayOfArrays straight_qrcode = noArray() + ) const; + + /** @brief Both detects and decodes QR codes + @param img grayscale or color (BGR) image containing QR codes. + @param decoded_info UTF8-encoded output vector of string or empty vector of string if the codes cannot be decoded. + @param points optional output vector of vertices of the found QR code quadrangles. Will be empty if not found. + @param straight_qrcode The optional output vector of images containing rectified and binarized QR codes + */ + CV_WRAP + bool detectAndDecodeMulti( + InputArray img, CV_OUT std::vector& decoded_info, + OutputArray points = noArray(), + OutputArrayOfArrays straight_qrcode = noArray() + ) const; + protected: struct Impl; Ptr p; @@ -731,6 +764,29 @@ CV_EXPORTS bool detectQRCode(InputArray in, std::vector &points, double e */ CV_EXPORTS bool decodeQRCode(InputArray in, InputArray points, std::string &decoded_info, OutputArray straight_qrcode = noArray()); +/** @brief Detect QR codes in image and return vector of minimum area of quadrangle that describes QR codes. + @param in Matrix of the type CV_8UC1 containing an image where QR codes are detected. + @param points Output vector of vertices of quadrangles of minimal area that describes QR codes. + @param eps_x Epsilon neighborhood, which allows you to determine the horizontal pattern of the scheme 1:1:3:1:1 according to QR code standard. + @param eps_y Epsilon neighborhood, which allows you to determine the vertical pattern of the scheme 1:1:3:1:1 according to QR code standard. + */ +CV_EXPORTS +bool detectQRCodeMulti( + InputArray in, std::vector &points, + double eps_x = 0.2, double eps_y = 0.1); + +/** @brief Decode QR codes in image and return text that is encrypted in QR code. + @param in Matrix of the type CV_8UC1 containing an image where QR code are detected. + @param points Input vector of vertices of quadrangles of minimal area that describes QR codes. + @param decoded_info vector of String information that is encrypted in QR codes. + @param straight_qrcode vector of Matrixes of the type CV_8UC1 containing an binary straight QR codes. + */ +CV_EXPORTS +bool decodeQRCodeMulti( + InputArray in, InputArray points, + CV_OUT std::vector &decoded_info, + OutputArrayOfArrays straight_qrcode = noArray()); + //! @} objdetect } diff --git a/modules/objdetect/misc/java/test/QRCodeDetectorTest.java b/modules/objdetect/misc/java/test/QRCodeDetectorTest.java index 9879772..cd8be40 100644 --- a/modules/objdetect/misc/java/test/QRCodeDetectorTest.java +++ b/modules/objdetect/misc/java/test/QRCodeDetectorTest.java @@ -1,9 +1,11 @@ package org.opencv.test.objdetect; +import java.util.List; import org.opencv.core.Mat; import org.opencv.objdetect.QRCodeDetector; import org.opencv.imgcodecs.Imgcodecs; import org.opencv.test.OpenCVTestCase; +import java.util.ArrayList; public class QRCodeDetectorTest extends OpenCVTestCase { @@ -21,9 +23,27 @@ public class QRCodeDetectorTest extends OpenCVTestCase { public void testDetectAndDecode() { Mat img = Imgcodecs.imread(testDataPath + "/cv/qrcode/link_ocv.jpg"); + assertFalse(img.empty()); QRCodeDetector detector = new QRCodeDetector(); + assertNotNull(detector); String output = detector.detectAndDecode(img); assertEquals(output, "https://opencv.org/"); } + public void testDetectAndDecodeMulti() { + Mat img = Imgcodecs.imread(testDataPath + "/cv/qrcode/multiple/6_qrcodes.png"); + assertFalse(img.empty()); + QRCodeDetector detector = new QRCodeDetector(); + assertNotNull(detector); + List < String > output = new ArrayList< String >(); + boolean result = detector.detectAndDecodeMulti(img, output); + assertTrue(result); + assertEquals(output.size(), 6); + assertEquals(output.get(0), "SKIP"); + assertEquals(output.get(1), "EXTRA"); + assertEquals(output.get(2), "TWO STEPS FORWARD"); + assertEquals(output.get(3), "STEP BACK"); + assertEquals(output.get(4), "QUESTION"); + assertEquals(output.get(5), "STEP FORWARD"); + } } diff --git a/modules/objdetect/misc/python/test/test_qrcode_detect.py b/modules/objdetect/misc/python/test/test_qrcode_detect.py index 18f7ed7..8a95c8b 100644 --- a/modules/objdetect/misc/python/test/test_qrcode_detect.py +++ b/modules/objdetect/misc/python/test/test_qrcode_detect.py @@ -11,8 +11,42 @@ import cv2 as cv from tests_common import NewOpenCVTests class qrcode_detector_test(NewOpenCVTests): + + def test_detect(self): + img = cv.imread(os.path.join(self.extraTestDataPath, 'cv/qrcode/link_ocv.jpg')) + self.assertFalse(img is None) + detector = cv.QRCodeDetector() + retval, points = detector.detect(img) + self.assertTrue(retval) + self.assertEqual(points.shape, (1, 4, 2)) + def test_detect_and_decode(self): img = cv.imread(os.path.join(self.extraTestDataPath, 'cv/qrcode/link_ocv.jpg')) + self.assertFalse(img is None) detector = cv.QRCodeDetector() retval, points, straight_qrcode = detector.detectAndDecode(img) - self.assertEqual(retval, "https://opencv.org/"); + self.assertEqual(retval, "https://opencv.org/") + self.assertEqual(points.shape, (1, 4, 2)) + + def test_detect_multi(self): + img = cv.imread(os.path.join(self.extraTestDataPath, 'cv/qrcode/multiple/6_qrcodes.png')) + self.assertFalse(img is None) + detector = cv.QRCodeDetector() + retval, points = detector.detectMulti(img) + self.assertTrue(retval) + self.assertEqual(points.shape, (6, 4, 2)) + + def test_detect_and_decode_multi(self): + img = cv.imread(os.path.join(self.extraTestDataPath, 'cv/qrcode/multiple/6_qrcodes.png')) + self.assertFalse(img is None) + detector = cv.QRCodeDetector() + retval, decoded_data, points, straight_qrcode = detector.detectAndDecodeMulti(img) + self.assertTrue(retval) + self.assertEqual(len(decoded_data), 6) + self.assertEqual(decoded_data[0], "TWO STEPS FORWARD") + self.assertEqual(decoded_data[1], "EXTRA") + self.assertEqual(decoded_data[2], "SKIP") + self.assertEqual(decoded_data[3], "STEP FORWARD") + self.assertEqual(decoded_data[4], "STEP BACK") + self.assertEqual(decoded_data[5], "QUESTION") + self.assertEqual(points.shape, (6, 4, 2)) diff --git a/modules/objdetect/perf/perf_qrcode_pipeline.cpp b/modules/objdetect/perf/perf_qrcode_pipeline.cpp index da5f278..44ab54f 100644 --- a/modules/objdetect/perf/perf_qrcode_pipeline.cpp +++ b/modules/objdetect/perf/perf_qrcode_pipeline.cpp @@ -53,6 +53,56 @@ PERF_TEST_P_(Perf_Objdetect_QRCode, decode) } #endif +typedef ::perf::TestBaseWithParam< std::string > Perf_Objdetect_QRCode_Multi; + +PERF_TEST_P_(Perf_Objdetect_QRCode_Multi, detectMulti) +{ + const std::string name_current_image = GetParam(); + const std::string root = "cv/qrcode/multiple/"; + + std::string image_path = findDataFile(root + name_current_image); + Mat src = imread(image_path); + ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path; + std::vector corners; + QRCodeDetector qrcode; + TEST_CYCLE() ASSERT_TRUE(qrcode.detectMulti(src, corners)); + SANITY_CHECK(corners); +} + +#ifdef HAVE_QUIRC +PERF_TEST_P_(Perf_Objdetect_QRCode_Multi, decodeMulti) +{ + const std::string name_current_image = GetParam(); + const std::string root = "cv/qrcode/multiple/"; + + std::string image_path = findDataFile(root + name_current_image); + Mat src = imread(image_path); + ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path; + QRCodeDetector qrcode; + std::vector corners; + ASSERT_TRUE(qrcode.detectMulti(src, corners)); + std::vector straight_barcode; + std::vector< cv::String > decoded_info; + TEST_CYCLE() + { + ASSERT_TRUE(qrcode.decodeMulti(src, corners, decoded_info, straight_barcode)); + for(size_t i = 0; i < decoded_info.size(); i++) + { + ASSERT_FALSE(decoded_info[i].empty()); + } + } + std::vector < std::vector< uint8_t > > decoded_info_uint8_t; + for(size_t i = 0; i < decoded_info.size(); i++) + { + std::vector< uint8_t > tmp(decoded_info[i].begin(), decoded_info[i].end()); + decoded_info_uint8_t.push_back(tmp); + } + SANITY_CHECK(decoded_info_uint8_t); + SANITY_CHECK(straight_barcode); + +} +#endif + INSTANTIATE_TEST_CASE_P(/*nothing*/, Perf_Objdetect_QRCode, ::testing::Values( "version_1_down.jpg", "version_1_left.jpg", "version_1_right.jpg", "version_1_up.jpg", "version_1_top.jpg", @@ -61,6 +111,13 @@ INSTANTIATE_TEST_CASE_P(/*nothing*/, Perf_Objdetect_QRCode, ) ); +INSTANTIATE_TEST_CASE_P(/*nothing*/, Perf_Objdetect_QRCode_Multi, + ::testing::Values( + "2_qrcodes.png", "3_close_qrcodes.png", "3_qrcodes.png", "4_qrcodes.png", + "5_qrcodes.png", "6_qrcodes.png", "7_qrcodes.png", "8_close_qrcodes.png" + ) +); + typedef ::perf::TestBaseWithParam< tuple< std::string, Size > > Perf_Objdetect_Not_QRCode; PERF_TEST_P_(Perf_Objdetect_Not_QRCode, detect) diff --git a/modules/objdetect/src/qrcode.cpp b/modules/objdetect/src/qrcode.cpp index 88929b0..3467803 100644 --- a/modules/objdetect/src/qrcode.cpp +++ b/modules/objdetect/src/qrcode.cpp @@ -22,6 +22,48 @@ namespace cv { using std::vector; +static bool checkQRInputImage(InputArray img, Mat& gray) +{ + CV_Assert(!img.empty()); + CV_CheckDepthEQ(img.depth(), CV_8U, ""); + + if (img.cols() <= 20 || img.rows() <= 20) + { + return false; // image data is not enough for providing reliable results + } + int incn = img.channels(); + CV_Check(incn, incn == 1 || incn == 3 || incn == 3, ""); + if (incn == 3 || incn == 4) + { + cvtColor(img, gray, COLOR_BGR2GRAY); + } + else + { + gray = img.getMat(); + } + return true; +} + +static void updatePointsResult(OutputArray points_, const vector& points) +{ + if (points_.needed()) + { + int N = int(points.size() / 4); + if (N > 0) + { + Mat m_p(N, 4, CV_32FC2, (void*)&points[0]); + int points_type = points_.fixedType() ? points_.type() : CV_32FC2; + m_p.reshape(2, points_.rows()).convertTo(points_, points_type); // Mat layout: N x 4 x 2cn + } + else + { + points_.release(); + } + } +} + + + class QRDetect { public: @@ -35,6 +77,7 @@ public: protected: vector searchHorizontalLines(); vector separateVerticalLines(const vector &list_lines); + vector extractVerticalLines(const vector &list_lines, double eps); void fixationPoints(vector &local_point); vector getQuadrilateral(vector angle_list); bool testBypassRoute(vector hull, int start, int finish); @@ -112,7 +155,7 @@ vector QRDetect::searchHorizontalLines() { if (bin_barcode_row[x] == future_pixel) { - future_pixel = 255 - future_pixel; + future_pixel = static_cast(~future_pixel); pixels_position.push_back(x); } } @@ -125,7 +168,7 @@ vector QRDetect::searchHorizontalLines() test_lines[3] = static_cast(pixels_position[i + 2] - pixels_position[i + 1]); test_lines[4] = static_cast(pixels_position[i + 3] - pixels_position[i + 2]); - double length = 0.0, weight = 0.0; + double length = 0.0, weight = 0.0; // TODO avoid 'double' calculations for (size_t j = 0; j < test_lines_size; j++) { length += test_lines[j]; } @@ -152,96 +195,115 @@ vector QRDetect::searchHorizontalLines() vector QRDetect::separateVerticalLines(const vector &list_lines) { CV_TRACE_FUNCTION(); - vector result; - int temp_length; - vector point2f_result; - uint8_t next_pixel; - vector test_lines; - for (int coeff_epsilon = 1; coeff_epsilon < 10; coeff_epsilon++) { - result.clear(); - temp_length = 0; - point2f_result.clear(); - - for (size_t pnt = 0; pnt < list_lines.size(); pnt++) + vector point2f_result = extractVerticalLines(list_lines, eps_horizontal * coeff_epsilon); + if (!point2f_result.empty()) { - const int x = cvRound(list_lines[pnt][0] + list_lines[pnt][2] * 0.5); - const int y = cvRound(list_lines[pnt][1]); + vector centers; + Mat labels; + double compactness = kmeans( + point2f_result, 3, labels, + TermCriteria(TermCriteria::EPS + TermCriteria::COUNT, 10, 0.1), + 3, KMEANS_PP_CENTERS, centers); + if (compactness == 0) + continue; + if (compactness > 0) + { + return point2f_result; + } + } + } + return vector(); // nothing +} + +vector QRDetect::extractVerticalLines(const vector &list_lines, double eps) +{ + CV_TRACE_FUNCTION(); + vector result; + vector test_lines; test_lines.reserve(6); + + for (size_t pnt = 0; pnt < list_lines.size(); pnt++) + { + const int x = cvRound(list_lines[pnt][0] + list_lines[pnt][2] * 0.5); + const int y = cvRound(list_lines[pnt][1]); - // --------------- Search vertical up-lines --------------- // + // --------------- Search vertical up-lines --------------- // - test_lines.clear(); - uint8_t future_pixel_up = 255; + test_lines.clear(); + uint8_t future_pixel_up = 255; - for (int j = y; j < bin_barcode.rows - 1; j++) + int temp_length_up = 0; + for (int j = y; j < bin_barcode.rows - 1; j++) + { + uint8_t next_pixel = bin_barcode.ptr(j + 1)[x]; + temp_length_up++; + if (next_pixel == future_pixel_up) { - next_pixel = bin_barcode.ptr(j + 1)[x]; - temp_length++; - if (next_pixel == future_pixel_up) - { - future_pixel_up = 255 - future_pixel_up; - test_lines.push_back(temp_length); - temp_length = 0; - if (test_lines.size() == 3) { break; } - } + future_pixel_up = static_cast(~future_pixel_up); + test_lines.push_back(temp_length_up); + temp_length_up = 0; + if (test_lines.size() == 3) + break; } + } - // --------------- Search vertical down-lines --------------- // + // --------------- Search vertical down-lines --------------- // - uint8_t future_pixel_down = 255; - for (int j = y; j >= 1; j--) + int temp_length_down = 0; + uint8_t future_pixel_down = 255; + for (int j = y; j >= 1; j--) + { + uint8_t next_pixel = bin_barcode.ptr(j - 1)[x]; + temp_length_down++; + if (next_pixel == future_pixel_down) { - next_pixel = bin_barcode.ptr(j - 1)[x]; - temp_length++; - if (next_pixel == future_pixel_down) - { - future_pixel_down = 255 - future_pixel_down; - test_lines.push_back(temp_length); - temp_length = 0; - if (test_lines.size() == 6) { break; } - } + future_pixel_down = static_cast(~future_pixel_down); + test_lines.push_back(temp_length_down); + temp_length_down = 0; + if (test_lines.size() == 6) + break; } + } - // --------------- Compute vertical lines --------------- // + // --------------- Compute vertical lines --------------- // - if (test_lines.size() == 6) - { - double length = 0.0, weight = 0.0; + if (test_lines.size() == 6) + { + double length = 0.0, weight = 0.0; // TODO avoid 'double' calculations - for (size_t i = 0; i < test_lines.size(); i++) { length += test_lines[i]; } + for (size_t i = 0; i < test_lines.size(); i++) + length += test_lines[i]; - CV_Assert(length > 0); - for (size_t i = 0; i < test_lines.size(); i++) + CV_Assert(length > 0); + for (size_t i = 0; i < test_lines.size(); i++) + { + if (i % 3 != 0) { - if (i % 3 != 0) { weight += fabs((test_lines[i] / length) - 1.0/ 7.0); } - else { weight += fabs((test_lines[i] / length) - 3.0/14.0); } + weight += fabs((test_lines[i] / length) - 1.0/ 7.0); } - - if(weight < eps_horizontal * coeff_epsilon) + else { - result.push_back(list_lines[pnt]); + weight += fabs((test_lines[i] / length) - 3.0/14.0); } } - } - if (result.size() > 2) - { - for (size_t i = 0; i < result.size(); i++) + + if (weight < eps) { - point2f_result.push_back( - Point2f(static_cast(result[i][0] + result[i][2] * 0.5), - static_cast(result[i][1]))); + result.push_back(list_lines[pnt]); } + } + } - vector centers; - Mat labels; - double compactness; - compactness = kmeans(point2f_result, 3, labels, - TermCriteria( TermCriteria::EPS + TermCriteria::COUNT, 10, 0.1), - 3, KMEANS_PP_CENTERS, centers); - if (compactness == 0) { continue; } - if (compactness > 0) { break; } + vector point2f_result; + if (result.size() > 2) + { + for (size_t i = 0; i < result.size(); i++) + { + point2f_result.push_back( + Point2f(static_cast(result[i][0] + result[i][2] * 0.5), + static_cast(result[i][1]))); } } return point2f_result; @@ -302,19 +364,25 @@ void QRDetect::fixationPoints(vector &local_point) for (size_t k = 0; k < list_line_iter.size(); k++) { + LineIterator& li = list_line_iter[k]; uint8_t future_pixel = 255, count_index = 0; - for(int j = 0; j < list_line_iter[k].count; j++, ++list_line_iter[k]) + for(int j = 0; j < li.count; j++, ++li) { - if (list_line_iter[k].pos().x >= bin_barcode.cols || - list_line_iter[k].pos().y >= bin_barcode.rows) { break; } - const uint8_t value = bin_barcode.at(list_line_iter[k].pos()); + const Point p = li.pos(); + if (p.x >= bin_barcode.cols || + p.y >= bin_barcode.rows) + { + break; + } + + const uint8_t value = bin_barcode.at(p); if (value == future_pixel) { - future_pixel = 255 - future_pixel; + future_pixel = static_cast(~future_pixel); count_index++; if (count_index == 3) { - list_area_pnt.push_back(list_line_iter[k].pos()); + list_area_pnt.push_back(p); break; } } @@ -454,7 +522,7 @@ bool QRDetect::computeTransformationPoints() next_pixel = bin_barcode.ptr(cvRound(localization_points[i].y))[index + 1]; if (next_pixel == future_pixel) { - future_pixel = 255 - future_pixel; + future_pixel = static_cast(~future_pixel); count_test_lines++; if (count_test_lines == 2) { @@ -623,11 +691,12 @@ vector QRDetect::getQuadrilateral(vector angle_list) angle_list[(i + 1) % angle_size]); for(int j = 0; j < line_iter.count; j++, ++line_iter) { - value = bin_barcode.at(line_iter.pos()); - mask_value = mask.at(line_iter.pos() + Point(1, 1)); + Point p = line_iter.pos(); + value = bin_barcode.at(p); + mask_value = mask.at(p + Point(1, 1)); if (value == 0 && mask_value == 0) { - floodFill(fill_bin_barcode, mask, line_iter.pos(), 255, + floodFill(fill_bin_barcode, mask, p, 255, 0, Scalar(), Scalar(), FLOODFILL_MASK_ONLY); } } @@ -868,26 +937,16 @@ void QRCodeDetector::setEpsY(double epsY) { p->epsY = epsY; } bool QRCodeDetector::detect(InputArray in, OutputArray points) const { - Mat inarr = in.getMat(); - CV_Assert(!inarr.empty()); - CV_Assert(inarr.depth() == CV_8U); - if (inarr.cols <= 20 || inarr.rows <= 20) - return false; // image data is not enough for providing reliable results - - int incn = inarr.channels(); - if( incn == 3 || incn == 4 ) - { - Mat gray; - cvtColor(inarr, gray, COLOR_BGR2GRAY); - inarr = gray; - } + Mat inarr; + if (!checkQRInputImage(in, inarr)) + return false; QRDetect qrdet; qrdet.init(inarr, p->epsX, p->epsY); if (!qrdet.localization()) { return false; } if (!qrdet.computeTransformationPoints()) { return false; } vector pnts2f = qrdet.getTransformationPoints(); - Mat(pnts2f).convertTo(points, points.fixedType() ? points.type() : CV_32FC2); + updatePointsResult(points, pnts2f); return true; } @@ -925,24 +984,7 @@ void QRDecode::init(const Mat &src, const vector &points) { CV_TRACE_FUNCTION(); vector bbox = points; - double coeff_expansion; - const int min_side = std::min(src.size().width, src.size().height); - if (min_side > 512) - { - coeff_expansion = min_side / 512; - const int width = cvRound(src.size().width / coeff_expansion); - const int height = cvRound(src.size().height / coeff_expansion); - Size new_size(width, height); - resize(src, original, new_size, 0, 0, INTER_AREA); - for (size_t i = 0; i < bbox.size(); i++) - { - bbox[i] /= static_cast(coeff_expansion); - } - } - else - { - original = src.clone(); - } + original = src.clone(); intermediate = Mat::zeros(original.size(), CV_8UC1); original_points = bbox; version = 0; @@ -1008,7 +1050,11 @@ bool QRDecode::versionDefinition() for(int j = 0; j < line_iter.count; j++, ++line_iter) { const uint8_t value = intermediate.at(line_iter.pos()); - if (value == 0) { black_point = line_iter.pos(); break; } + if (value == 0) + { + black_point = line_iter.pos(); + break; + } } Mat mask = Mat::zeros(intermediate.rows + 2, intermediate.cols + 2, CV_8UC1); @@ -1041,7 +1087,7 @@ bool QRDecode::versionDefinition() { if (intermediate_row[i] == future_pixel) { - future_pixel = 255 - future_pixel; + future_pixel = static_cast(~future_pixel); transition_x++; } } @@ -1052,11 +1098,10 @@ bool QRDecode::versionDefinition() const uint8_t value = intermediate.at(Point(j, remote_point.x)); if (value == future_pixel) { - future_pixel = 255 - future_pixel; + future_pixel = static_cast(~future_pixel); transition_y++; } } - version = saturate_cast((std::min(transition_x, transition_y) - 1) * 0.25 - 1); if ( !( 0 < version && version <= 40 ) ) { return false; } version_size = 21 + (version - 1) * 4; @@ -1177,19 +1222,9 @@ bool decodeQRCode(InputArray in, InputArray points, std::string &decoded_info, O cv::String QRCodeDetector::decode(InputArray in, InputArray points, OutputArray straight_qrcode) { - Mat inarr = in.getMat(); - CV_Assert(!inarr.empty()); - CV_Assert(inarr.depth() == CV_8U); - if (inarr.cols <= 20 || inarr.rows <= 20) - return cv::String(); // image data is not enough for providing reliable results - - int incn = inarr.channels(); - if( incn == 3 || incn == 4 ) - { - Mat gray; - cvtColor(inarr, gray, COLOR_BGR2GRAY); - inarr = gray; - } + Mat inarr; + if (!checkQRInputImage(in, inarr)) + return std::string(); vector src_points; points.copyTo(src_points); @@ -1216,34 +1251,1150 @@ cv::String QRCodeDetector::detectAndDecode(InputArray in, OutputArray points_, OutputArray straight_qrcode) { - Mat inarr = in.getMat(); - CV_Assert(!inarr.empty()); - CV_Assert(inarr.depth() == CV_8U); - if (inarr.cols <= 20 || inarr.rows <= 20) - return cv::String(); // image data is not enough for providing reliable results - - int incn = inarr.channels(); - if( incn == 3 || incn == 4 ) + Mat inarr; + if (!checkQRInputImage(in, inarr)) { - Mat gray; - cvtColor(inarr, gray, COLOR_BGR2GRAY); - inarr = gray; + points_.release(); + return std::string(); } vector points; bool ok = detect(inarr, points); - if( points_.needed() ) + if (!ok) { - if( ok ) - Mat(points).copyTo(points_); - else - points_.release(); + points_.release(); + return std::string(); } - std::string decoded_info; - if( ok ) - decoded_info = decode(inarr, points, straight_qrcode); + updatePointsResult(points_, points); + std::string decoded_info = decode(inarr, points, straight_qrcode); return decoded_info; } +class QRDetectMulti : public QRDetect +{ +public: + void init(const Mat& src, double eps_vertical_ = 0.2, double eps_horizontal_ = 0.1); + bool localization(); + bool computeTransformationPoints(const size_t cur_ind); + vector< vector < Point2f > > getTransformationPoints() { return transformation_points;} + +protected: + int findNumberLocalizationPoints(vector& tmp_localization_points); + void findQRCodeContours(vector& tmp_localization_points, vector< vector< Point2f > >& true_points_group, const int& num_qrcodes); + bool checkSets(vector >& true_points_group, vector >& true_points_group_copy, + vector& tmp_localization_points); + void deleteUsedPoints(vector >& true_points_group, vector >& loc, + vector& tmp_localization_points); + void fixationPoints(vector &local_point); + bool checkPoints(const vector& quadrangle_points); + bool checkPointsInsideQuadrangle(const vector& quadrangle_points); + bool checkPointsInsideTriangle(const vector& triangle_points); + + Mat bin_barcode_fullsize, bin_barcode_temp; + vector not_resized_loc_points; + vector resized_loc_points; + vector< vector< Point2f > > localization_points, transformation_points; + struct compareDistanse_y + { + bool operator()(const Point2f& a, const Point2f& b) const + { + return a.y < b.y; + } + }; + struct compareSquare + { + const vector& points; + compareSquare(const vector& points_) : points(points_) {} + bool operator()(const Vec3i& a, const Vec3i& b) const; + }; + Mat original; + class ParallelSearch : public ParallelLoopBody + { + public: + ParallelSearch(vector< vector< Point2f > >& true_points_group_, + vector< vector< Point2f > >& loc_, int iter_, int* end_, + vector< vector< Vec3i > >& all_points_, + QRDetectMulti& cl_) + : + true_points_group(true_points_group_), + loc(loc_), + iter(iter_), + end(end_), + all_points(all_points_), + cl(cl_) + { + } + void operator()(const Range& range) const CV_OVERRIDE; + vector< vector< Point2f > >& true_points_group; + vector< vector< Point2f > >& loc; + int iter; + int* end; + vector< vector< Vec3i > >& all_points; + QRDetectMulti& cl; + }; +}; + +void QRDetectMulti::ParallelSearch::operator()(const Range& range) const +{ + for (int s = range.start; s < range.end; s++) + { + bool flag = false; + for (int r = iter; r < end[s]; r++) + { + if (flag) + break; + + size_t x = iter + s; + size_t k = r - iter; + vector triangle; + + for (int l = 0; l < 3; l++) + { + triangle.push_back(true_points_group[s][all_points[s][k][l]]); + } + + if (cl.checkPointsInsideTriangle(triangle)) + { + bool flag_for_break = false; + cl.fixationPoints(triangle); + if (triangle.size() == 3) + { + cl.localization_points[x] = triangle; + if (cl.purpose == cl.SHRINKING) + { + + for (size_t j = 0; j < 3; j++) + { + cl.localization_points[x][j] *= cl.coeff_expansion; + } + } + else if (cl.purpose == cl.ZOOMING) + { + for (size_t j = 0; j < 3; j++) + { + cl.localization_points[x][j] /= cl.coeff_expansion; + } + } + for (size_t i = 0; i < 3; i++) + { + for (size_t j = i + 1; j < 3; j++) + { + if (norm(cl.localization_points[x][i] - cl.localization_points[x][j]) < 10) + { + cl.localization_points[x].clear(); + flag_for_break = true; + break; + } + } + if (flag_for_break) + break; + } + if ((!flag_for_break) + && (cl.localization_points[x].size() == 3) + && (cl.computeTransformationPoints(x)) + && (cl.checkPointsInsideQuadrangle(cl.transformation_points[x])) + && (cl.checkPoints(cl.transformation_points[x]))) + { + for (int l = 0; l < 3; l++) + { + loc[s][all_points[s][k][l]].x = -1; + } + + flag = true; + break; + } + } + if (flag) + { + break; + } + else + { + cl.transformation_points[x].clear(); + cl.localization_points[x].clear(); + } + } + } + } +} + +void QRDetectMulti::init(const Mat& src, double eps_vertical_, double eps_horizontal_) +{ + CV_TRACE_FUNCTION(); + + CV_Assert(!src.empty()); + const double min_side = std::min(src.size().width, src.size().height); + if (min_side < 512.0) + { + purpose = ZOOMING; + coeff_expansion = 512.0 / min_side; + const int width = cvRound(src.size().width * coeff_expansion); + const int height = cvRound(src.size().height * coeff_expansion); + Size new_size(width, height); + resize(src, barcode, new_size, 0, 0, INTER_LINEAR); + } + else if (min_side > 512.0) + { + purpose = SHRINKING; + coeff_expansion = min_side / 512.0; + const int width = cvRound(src.size().width / coeff_expansion); + const int height = cvRound(src.size().height / coeff_expansion); + Size new_size(width, height); + resize(src, barcode, new_size, 0, 0, INTER_AREA); + } + else + { + purpose = UNCHANGED; + coeff_expansion = 1.0; + barcode = src.clone(); + } + + eps_vertical = eps_vertical_; + eps_horizontal = eps_horizontal_; + adaptiveThreshold(barcode, bin_barcode, 255, ADAPTIVE_THRESH_GAUSSIAN_C, THRESH_BINARY, 83, 2); + adaptiveThreshold(src, bin_barcode_fullsize, 255, ADAPTIVE_THRESH_GAUSSIAN_C, THRESH_BINARY, 83, 2); +} + +void QRDetectMulti::fixationPoints(vector &local_point) +{ + CV_TRACE_FUNCTION(); + + Point2f v0(local_point[1] - local_point[2]); + Point2f v1(local_point[0] - local_point[2]); + Point2f v2(local_point[1] - local_point[0]); + + double cos_angles[3], norm_triangl[3]; + norm_triangl[0] = norm(v0); + norm_triangl[1] = norm(v1); + norm_triangl[2] = norm(v2); + + cos_angles[0] = v2.dot(-v1) / (norm_triangl[1] * norm_triangl[2]); + cos_angles[1] = v2.dot(v0) / (norm_triangl[0] * norm_triangl[2]); + cos_angles[2] = v1.dot(v0) / (norm_triangl[0] * norm_triangl[1]); + + const double angle_barrier = 0.85; + if (fabs(cos_angles[0]) > angle_barrier || fabs(cos_angles[1]) > angle_barrier || fabs(cos_angles[2]) > angle_barrier) + { + local_point.clear(); + return; + } + + size_t i_min_cos = + (cos_angles[0] < cos_angles[1] && cos_angles[0] < cos_angles[2]) ? 0 : + (cos_angles[1] < cos_angles[0] && cos_angles[1] < cos_angles[2]) ? 1 : 2; + + size_t index_max = 0; + double max_area = std::numeric_limits::min(); + for (size_t i = 0; i < local_point.size(); i++) + { + const size_t current_index = i % 3; + const size_t left_index = (i + 1) % 3; + const size_t right_index = (i + 2) % 3; + + const Point2f current_point(local_point[current_index]); + const Point2f left_point(local_point[left_index]); + const Point2f right_point(local_point[right_index]); + const Point2f central_point(intersectionLines( + current_point, + Point2f(static_cast((local_point[left_index].x + local_point[right_index].x) * 0.5), + static_cast((local_point[left_index].y + local_point[right_index].y) * 0.5)), + Point2f(0, static_cast(bin_barcode_temp.rows - 1)), + Point2f(static_cast(bin_barcode_temp.cols - 1), + static_cast(bin_barcode_temp.rows - 1)))); + + + vector list_area_pnt; + list_area_pnt.push_back(current_point); + + vector list_line_iter; + list_line_iter.push_back(LineIterator(bin_barcode_temp, current_point, left_point)); + list_line_iter.push_back(LineIterator(bin_barcode_temp, current_point, central_point)); + list_line_iter.push_back(LineIterator(bin_barcode_temp, current_point, right_point)); + + for (size_t k = 0; k < list_line_iter.size(); k++) + { + LineIterator& li = list_line_iter[k]; + uint8_t future_pixel = 255, count_index = 0; + for (int j = 0; j < li.count; j++, ++li) + { + Point p = li.pos(); + if (p.x >= bin_barcode_temp.cols || + p.y >= bin_barcode_temp.rows) + { + break; + } + + const uint8_t value = bin_barcode_temp.at(p); + if (value == future_pixel) + { + future_pixel = static_cast(~future_pixel); + count_index++; + if (count_index == 3) + { + list_area_pnt.push_back(p); + break; + } + } + } + } + + const double temp_check_area = contourArea(list_area_pnt); + if (temp_check_area > max_area) + { + index_max = current_index; + max_area = temp_check_area; + } + + } + if (index_max == i_min_cos) + { + std::swap(local_point[0], local_point[index_max]); + } + else + { + local_point.clear(); + return; + } + const Point2f rpt = local_point[0], bpt = local_point[1], gpt = local_point[2]; + Matx22f m(rpt.x - bpt.x, rpt.y - bpt.y, gpt.x - rpt.x, gpt.y - rpt.y); + if (determinant(m) > 0) + { + std::swap(local_point[1], local_point[2]); + } } + +bool QRDetectMulti::checkPoints(const vector& quadrangle_points) +{ + if (quadrangle_points.size() != 4) + return false; + vector quadrangle = quadrangle_points; + std::sort(quadrangle.begin(), quadrangle.end(), compareDistanse_y()); + LineIterator it1(bin_barcode_fullsize, quadrangle[1], quadrangle[0]); + LineIterator it2(bin_barcode_fullsize, quadrangle[2], quadrangle[0]); + LineIterator it3(bin_barcode_fullsize, quadrangle[1], quadrangle[3]); + LineIterator it4(bin_barcode_fullsize, quadrangle[2], quadrangle[3]); + vector list_line_iter; + list_line_iter.push_back(it1); + list_line_iter.push_back(it2); + list_line_iter.push_back(it3); + list_line_iter.push_back(it4); + int count_w = 0; + int count_b = 0; + for (int j = 0; j < 3; j +=2) + { + LineIterator& li = list_line_iter[j]; + LineIterator& li2 = list_line_iter[j + 1]; + for (int i = 0; i < li.count; i++) + { + + Point pt1 = li.pos(); + Point pt2 = li2.pos(); + LineIterator it0(bin_barcode_fullsize, pt1, pt2); + for (int r = 0; r < it0.count; r++) + { + int pixel = bin_barcode.at(it0.pos().y , it0.pos().x); + if (pixel == 255) + { + count_w++; + } + if (pixel == 0) + { + count_b++; + } + it0++; + } + li++; + li2++; + } + } + + double frac = double(count_b) / double(count_w); + double bottom_bound = 0.76; + double upper_bound = 1.24; + if ((frac <= bottom_bound) || (frac >= upper_bound)) + return false; + return true; +} + +bool QRDetectMulti::checkPointsInsideQuadrangle(const vector& quadrangle_points) +{ + if (quadrangle_points.size() != 4) + return false; + + int count = 0; + for (size_t i = 0; i < not_resized_loc_points.size(); i++) + { + if (pointPolygonTest(quadrangle_points, not_resized_loc_points[i], true) > 0) + { + count++; + } + } + if (count == 3) + return true; + else + return false; +} + +bool QRDetectMulti::checkPointsInsideTriangle(const vector& triangle_points) +{ + if (triangle_points.size() != 3) + return false; + double eps = 3; + for (size_t i = 0; i < resized_loc_points.size(); i++) + { + if (pointPolygonTest( triangle_points, resized_loc_points[i], true ) > 0) + { + if ((abs(resized_loc_points[i].x - triangle_points[0].x) > eps) + && (abs(resized_loc_points[i].x - triangle_points[1].x) > eps) + && (abs(resized_loc_points[i].x - triangle_points[2].x) > eps)) + { + return false; + } + } + } + return true; +} + +bool QRDetectMulti::compareSquare::operator()(const Vec3i& a, const Vec3i& b) const +{ + Point2f a0 = points[a[0]]; + Point2f a1 = points[a[1]]; + Point2f a2 = points[a[2]]; + Point2f b0 = points[b[0]]; + Point2f b1 = points[b[1]]; + Point2f b2 = points[b[2]]; + return fabs((a1.x - a0.x) * (a2.y - a0.y) - (a2.x - a0.x) * (a1.y - a0.y)) < + fabs((b1.x - b0.x) * (b2.y - b0.y) - (b2.x - b0.x) * (b1.y - b0.y)); +} + +int QRDetectMulti::findNumberLocalizationPoints(vector& tmp_localization_points) +{ + size_t number_possible_purpose = 1; + if (purpose == SHRINKING) + number_possible_purpose = 2; + Mat tmp_shrinking = bin_barcode; + int tmp_num_points = 0; + int num_points = -1; + for (eps_horizontal = 0.1; eps_horizontal < 0.4; eps_horizontal += 0.1) + { + tmp_num_points = 0; + num_points = -1; + if (purpose == SHRINKING) + number_possible_purpose = 2; + else + number_possible_purpose = 1; + for (size_t k = 0; k < number_possible_purpose; k++) + { + if (k == 1) + bin_barcode = bin_barcode_fullsize; + vector list_lines_x = searchHorizontalLines(); + if (list_lines_x.empty()) + { + if (k == 0) + { + k = 1; + bin_barcode = bin_barcode_fullsize; + list_lines_x = searchHorizontalLines(); + if (list_lines_x.empty()) + break; + } + else + break; + } + vector list_lines_y = extractVerticalLines(list_lines_x, eps_horizontal); + if (list_lines_y.size() < 3) + { + if (k == 0) + { + k = 1; + bin_barcode = bin_barcode_fullsize; + list_lines_x = searchHorizontalLines(); + if (list_lines_x.empty()) + break; + list_lines_y = extractVerticalLines(list_lines_x, eps_horizontal); + if (list_lines_y.size() < 3) + break; + } + else + break; + } + vector index_list_lines_y; + for (size_t i = 0; i < list_lines_y.size(); i++) + index_list_lines_y.push_back(-1); + num_points = 0; + for (size_t i = 0; i < list_lines_y.size() - 1; i++) + { + for (size_t j = i; j < list_lines_y.size(); j++ ) + { + + double points_distance = norm(list_lines_y[i] - list_lines_y[j]); + if (points_distance <= 10) + { + if ((index_list_lines_y[i] == -1) && (index_list_lines_y[j] == -1)) + { + index_list_lines_y[i] = num_points; + index_list_lines_y[j] = num_points; + num_points++; + } + else if (index_list_lines_y[i] != -1) + index_list_lines_y[j] = index_list_lines_y[i]; + else if (index_list_lines_y[j] != -1) + index_list_lines_y[i] = index_list_lines_y[j]; + } + } + } + for (size_t i = 0; i < index_list_lines_y.size(); i++) + { + if (index_list_lines_y[i] == -1) + { + index_list_lines_y[i] = num_points; + num_points++; + } + } + if ((tmp_num_points < num_points) && (k == 1)) + { + purpose = UNCHANGED; + tmp_num_points = num_points; + bin_barcode = bin_barcode_fullsize; + coeff_expansion = 1.0; + } + if ((tmp_num_points < num_points) && (k == 0)) + { + tmp_num_points = num_points; + } + } + + if ((tmp_num_points < 3) && (tmp_num_points >= 1)) + { + const double min_side = std::min(bin_barcode_fullsize.size().width, bin_barcode_fullsize.size().height); + if (min_side > 512) + { + bin_barcode = tmp_shrinking; + purpose = SHRINKING; + coeff_expansion = min_side / 512.0; + } + if (min_side < 512) + { + bin_barcode = tmp_shrinking; + purpose = ZOOMING; + coeff_expansion = 512 / min_side; + } + } + else + break; + } + if (purpose == SHRINKING) + bin_barcode = tmp_shrinking; + num_points = tmp_num_points; + vector list_lines_x = searchHorizontalLines(); + if (list_lines_x.empty()) + return num_points; + vector list_lines_y = extractVerticalLines(list_lines_x, eps_horizontal); + if (list_lines_y.size() < 3) + return num_points; + if (num_points < 3) + return num_points; + + Mat labels; + kmeans(list_lines_y, num_points, labels, + TermCriteria( TermCriteria::EPS + TermCriteria::COUNT, 10, 0.1), + num_points, KMEANS_PP_CENTERS, tmp_localization_points); + bin_barcode_temp = bin_barcode.clone(); + if (purpose == SHRINKING) + { + const int width = cvRound(bin_barcode.size().width * coeff_expansion); + const int height = cvRound(bin_barcode.size().height * coeff_expansion); + Size new_size(width, height); + Mat intermediate; + resize(bin_barcode, intermediate, new_size, 0, 0, INTER_LINEAR); + bin_barcode = intermediate.clone(); + } + else if (purpose == ZOOMING) + { + const int width = cvRound(bin_barcode.size().width / coeff_expansion); + const int height = cvRound(bin_barcode.size().height / coeff_expansion); + Size new_size(width, height); + Mat intermediate; + resize(bin_barcode, intermediate, new_size, 0, 0, INTER_LINEAR); + bin_barcode = intermediate.clone(); + } + else + { + bin_barcode = bin_barcode_fullsize.clone(); + } + return num_points; +} + +void QRDetectMulti::findQRCodeContours(vector& tmp_localization_points, + vector< vector< Point2f > >& true_points_group, const int& num_qrcodes) +{ + Mat gray, blur_image, threshold_output; + Mat bar = barcode; + const int width = cvRound(bin_barcode.size().width); + const int height = cvRound(bin_barcode.size().height); + Size new_size(width, height); + resize(bar, bar, new_size, 0, 0, INTER_LINEAR); + blur(bar, blur_image, Size(3, 3)); + threshold(blur_image, threshold_output, 50, 255, THRESH_BINARY); + + vector< vector< Point > > contours; + vector hierarchy; + findContours(threshold_output, contours, hierarchy, RETR_TREE, CHAIN_APPROX_SIMPLE, Point(0, 0)); + vector all_contours_points; + for (size_t i = 0; i < contours.size(); i++) + { + for (size_t j = 0; j < contours[i].size(); j++) + { + all_contours_points.push_back(contours[i][j]); + } + } + Mat qrcode_labels; + vector clustered_localization_points; + int count_contours = num_qrcodes; + if (all_contours_points.size() < size_t(num_qrcodes)) + count_contours = (int)all_contours_points.size(); + kmeans(all_contours_points, count_contours, qrcode_labels, + TermCriteria( TermCriteria::EPS + TermCriteria::COUNT, 10, 0.1), + count_contours, KMEANS_PP_CENTERS, clustered_localization_points); + + vector< vector< Point2f > > qrcode_clusters(count_contours); + for (int i = 0; i < count_contours; i++) + for (int j = 0; j < int(all_contours_points.size()); j++) + { + if (qrcode_labels.at(j, 0) == i) + { + qrcode_clusters[i].push_back(all_contours_points[j]); + } + } + vector< vector< Point2f > > hull(count_contours); + for (size_t i = 0; i < qrcode_clusters.size(); i++) + convexHull(Mat(qrcode_clusters[i]), hull[i]); + not_resized_loc_points = tmp_localization_points; + resized_loc_points = tmp_localization_points; + if (purpose == SHRINKING) + { + for (size_t j = 0; j < not_resized_loc_points.size(); j++) + { + not_resized_loc_points[j] *= coeff_expansion; + } + } + else if (purpose == ZOOMING) + { + for (size_t j = 0; j < not_resized_loc_points.size(); j++) + { + not_resized_loc_points[j] /= coeff_expansion; + } + } + + true_points_group.resize(hull.size()); + + for (size_t j = 0; j < hull.size(); j++) + { + for (size_t i = 0; i < not_resized_loc_points.size(); i++) + { + if (pointPolygonTest(hull[j], not_resized_loc_points[i], true) > 0) + { + true_points_group[j].push_back(tmp_localization_points[i]); + tmp_localization_points[i].x = -1; + } + + } + } + vector copy; + for (size_t j = 0; j < tmp_localization_points.size(); j++) + { + if (tmp_localization_points[j].x != -1) + copy.push_back(tmp_localization_points[j]); + } + tmp_localization_points = copy; +} + +bool QRDetectMulti::checkSets(vector >& true_points_group, vector >& true_points_group_copy, + vector& tmp_localization_points) +{ + for (size_t i = 0; i < true_points_group.size(); i++) + { + if (true_points_group[i].size() < 3) + { + for (size_t j = 0; j < true_points_group[i].size(); j++) + tmp_localization_points.push_back(true_points_group[i][j]); + true_points_group[i].clear(); + } + } + vector< vector< Point2f > > temp_for_copy; + for (size_t i = 0; i < true_points_group.size(); i++) + { + if (true_points_group[i].size() != 0) + temp_for_copy.push_back(true_points_group[i]); + } + true_points_group = temp_for_copy; + if (true_points_group.size() == 0) + { + true_points_group.push_back(tmp_localization_points); + tmp_localization_points.clear(); + } + if (true_points_group.size() == 0) + return false; + if (true_points_group[0].size() < 3) + return false; + + + int* set_size = new int[true_points_group.size()]; + for (size_t i = 0; i < true_points_group.size(); i++) + { + set_size[i] = int(0.5 * (true_points_group[i].size() - 2 ) * (true_points_group[i].size() - 1)); + } + vector< vector< Vec3i > > all_points(true_points_group.size()); + for (size_t i = 0; i < true_points_group.size(); i++) + all_points[i].resize(set_size[i]); + int cur_cluster = 0; + for (size_t i = 0; i < true_points_group.size(); i++) + { + cur_cluster = 0; + for (size_t j = 1; j < true_points_group[i].size() - 1; j++) + for (size_t k = j + 1; k < true_points_group[i].size(); k++) + { + all_points[i][cur_cluster][0] = 0; + all_points[i][cur_cluster][1] = int(j); + all_points[i][cur_cluster][2] = int(k); + cur_cluster++; + } + } + + for (size_t i = 0; i < true_points_group.size(); i++) + { + std::sort(all_points[i].begin(), all_points[i].end(), compareSquare(true_points_group[i])); + } + if (true_points_group.size() == 1) + { + int check_number = 35; + if (set_size[0] > check_number) + set_size[0] = check_number; + all_points[0].resize(set_size[0]); + } + int iter = (int)localization_points.size(); + localization_points.resize(iter + true_points_group.size()); + transformation_points.resize(iter + true_points_group.size()); + + true_points_group_copy = true_points_group; + int* end = new int[true_points_group.size()]; + for (size_t i = 0; i < true_points_group.size(); i++) + end[i] = iter + set_size[i]; + ParallelSearch parallelSearch(true_points_group, + true_points_group_copy, iter, end, all_points, *this); + parallel_for_(Range(0, (int)true_points_group.size()), parallelSearch); + + return true; +} + +void QRDetectMulti::deleteUsedPoints(vector >& true_points_group, vector >& loc, + vector& tmp_localization_points) +{ + size_t iter = localization_points.size() - true_points_group.size() ; + for (size_t s = 0; s < true_points_group.size(); s++) + { + if (localization_points[iter + s].empty()) + loc[s][0].x = -2; + + if (loc[s].size() == 3) + { + + if ((true_points_group.size() > 1) || ((true_points_group.size() == 1) && (tmp_localization_points.size() != 0)) ) + { + for (size_t j = 0; j < true_points_group[s].size(); j++) + { + if (loc[s][j].x != -1) + { + loc[s][j].x = -1; + tmp_localization_points.push_back(true_points_group[s][j]); + } + } + } + } + vector for_copy; + for (size_t j = 0; j < loc[s].size(); j++) + { + if ((loc[s][j].x != -1) && (loc[s][j].x != -2) ) + { + for_copy.push_back(true_points_group[s][j]); + } + if ((loc[s][j].x == -2) && (true_points_group.size() > 1)) + { + tmp_localization_points.push_back(true_points_group[s][j]); + } + } + true_points_group[s] = for_copy; + } + + vector< vector< Point2f > > for_copy_loc; + vector< vector< Point2f > > for_copy_trans; + + + for (size_t i = 0; i < localization_points.size(); i++) + { + if ((localization_points[i].size() == 3) && (transformation_points[i].size() == 4)) + { + for_copy_loc.push_back(localization_points[i]); + for_copy_trans.push_back(transformation_points[i]); + } + } + localization_points = for_copy_loc; + transformation_points = for_copy_trans; +} + +bool QRDetectMulti::localization() +{ + CV_TRACE_FUNCTION(); + vector tmp_localization_points; + int num_points = findNumberLocalizationPoints(tmp_localization_points); + if (num_points < 3) + return false; + int num_qrcodes = divUp(num_points, 3); + vector > true_points_group; + findQRCodeContours(tmp_localization_points, true_points_group, num_qrcodes); + for (int q = 0; q < num_qrcodes; q++) + { + vector > loc; + size_t iter = localization_points.size(); + + if (!checkSets(true_points_group, loc, tmp_localization_points)) + break; + deleteUsedPoints(true_points_group, loc, tmp_localization_points); + if ((localization_points.size() - iter) == 1) + q--; + if (((localization_points.size() - iter) == 0) && (tmp_localization_points.size() == 0) && (true_points_group.size() == 1) ) + break; + } + if ((transformation_points.size() == 0) || (localization_points.size() == 0)) + return false; + return true; +} + +bool QRDetectMulti::computeTransformationPoints(const size_t cur_ind) +{ + CV_TRACE_FUNCTION(); + + if (localization_points[cur_ind].size() != 3) + { + return false; + } + + vector locations, non_zero_elem[3], newHull; + vector new_non_zero_elem[3]; + for (size_t i = 0; i < 3 ; i++) + { + Mat mask = Mat::zeros(bin_barcode.rows + 2, bin_barcode.cols + 2, CV_8UC1); + uint8_t next_pixel, future_pixel = 255; + int localization_point_x = cvRound(localization_points[cur_ind][i].x); + int localization_point_y = cvRound(localization_points[cur_ind][i].y); + int count_test_lines = 0, index = localization_point_x; + for (; index < bin_barcode.cols - 1; index++) + { + next_pixel = bin_barcode.at(localization_point_y, index + 1); + if (next_pixel == future_pixel) + { + future_pixel = static_cast(~future_pixel); + count_test_lines++; + + if (count_test_lines == 2) + { + // TODO avoid drawing functions + floodFill(bin_barcode, mask, + Point(index + 1, localization_point_y), 255, + 0, Scalar(), Scalar(), FLOODFILL_MASK_ONLY); + break; + } + } + + } + Mat mask_roi = mask(Range(1, bin_barcode.rows - 1), Range(1, bin_barcode.cols - 1)); + findNonZero(mask_roi, non_zero_elem[i]); + newHull.insert(newHull.end(), non_zero_elem[i].begin(), non_zero_elem[i].end()); + } + convexHull(newHull, locations); + for (size_t i = 0; i < locations.size(); i++) + { + for (size_t j = 0; j < 3; j++) + { + for (size_t k = 0; k < non_zero_elem[j].size(); k++) + { + if (locations[i] == non_zero_elem[j][k]) + { + new_non_zero_elem[j].push_back(locations[i]); + } + } + } + } + + if (new_non_zero_elem[0].size() == 0) + return false; + + double pentagon_diag_norm = -1; + Point2f down_left_edge_point, up_right_edge_point, up_left_edge_point; + for (size_t i = 0; i < new_non_zero_elem[1].size(); i++) + { + for (size_t j = 0; j < new_non_zero_elem[2].size(); j++) + { + double temp_norm = norm(new_non_zero_elem[1][i] - new_non_zero_elem[2][j]); + if (temp_norm > pentagon_diag_norm) + { + down_left_edge_point = new_non_zero_elem[1][i]; + up_right_edge_point = new_non_zero_elem[2][j]; + pentagon_diag_norm = temp_norm; + } + } + } + + if (down_left_edge_point == Point2f(0, 0) || + up_right_edge_point == Point2f(0, 0)) + { + return false; + } + + double max_area = -1; + up_left_edge_point = new_non_zero_elem[0][0]; + + for (size_t i = 0; i < new_non_zero_elem[0].size(); i++) + { + vector list_edge_points; + list_edge_points.push_back(new_non_zero_elem[0][i]); + list_edge_points.push_back(down_left_edge_point); + list_edge_points.push_back(up_right_edge_point); + + double temp_area = fabs(contourArea(list_edge_points)); + if (max_area < temp_area) + { + up_left_edge_point = new_non_zero_elem[0][i]; + max_area = temp_area; + } + } + + Point2f down_max_delta_point, up_max_delta_point; + double norm_down_max_delta = -1, norm_up_max_delta = -1; + for (size_t i = 0; i < new_non_zero_elem[1].size(); i++) + { + double temp_norm_delta = norm(up_left_edge_point - new_non_zero_elem[1][i]) + norm(down_left_edge_point - new_non_zero_elem[1][i]); + if (norm_down_max_delta < temp_norm_delta) + { + down_max_delta_point = new_non_zero_elem[1][i]; + norm_down_max_delta = temp_norm_delta; + } + } + + + for (size_t i = 0; i < new_non_zero_elem[2].size(); i++) + { + double temp_norm_delta = norm(up_left_edge_point - new_non_zero_elem[2][i]) + norm(up_right_edge_point - new_non_zero_elem[2][i]); + if (norm_up_max_delta < temp_norm_delta) + { + up_max_delta_point = new_non_zero_elem[2][i]; + norm_up_max_delta = temp_norm_delta; + } + } + vector tmp_transformation_points; + tmp_transformation_points.push_back(down_left_edge_point); + tmp_transformation_points.push_back(up_left_edge_point); + tmp_transformation_points.push_back(up_right_edge_point); + tmp_transformation_points.push_back(intersectionLines( + down_left_edge_point, down_max_delta_point, + up_right_edge_point, up_max_delta_point)); + transformation_points[cur_ind] = tmp_transformation_points; + + vector quadrilateral = getQuadrilateral(transformation_points[cur_ind]); + transformation_points[cur_ind] = quadrilateral; + + return true; +} + +bool QRCodeDetector::detectMulti(InputArray in, OutputArray points) const +{ + Mat inarr; + if (!checkQRInputImage(in, inarr)) + { + points.release(); + return false; + } + + QRDetectMulti qrdet; + qrdet.init(inarr, p->epsX, p->epsY); + if (!qrdet.localization()) + { + points.release(); + return false; + } + vector< vector< Point2f > > pnts2f = qrdet.getTransformationPoints(); + vector trans_points; + for(size_t i = 0; i < pnts2f.size(); i++) + for(size_t j = 0; j < pnts2f[i].size(); j++) + trans_points.push_back(pnts2f[i][j]); + + updatePointsResult(points, trans_points); + + return true; +} + +bool detectQRCodeMulti(InputArray in, vector< Point > &points, double eps_x, double eps_y) +{ + QRCodeDetector qrdetector; + qrdetector.setEpsX(eps_x); + qrdetector.setEpsY(eps_y); + return qrdetector.detectMulti(in, points); +} + +class ParallelDecodeProcess : public ParallelLoopBody +{ +public: + ParallelDecodeProcess(Mat& inarr_, vector& qrdec_, vector& decoded_info_, + vector& straight_barcode_, vector< vector< Point2f > >& src_points_) + : inarr(inarr_), qrdec(qrdec_), decoded_info(decoded_info_) + , straight_barcode(straight_barcode_), src_points(src_points_) + { + // nothing + } + void operator()(const Range& range) const CV_OVERRIDE + { + for (int i = range.start; i < range.end; i++) + { + qrdec[i].init(inarr, src_points[i]); + bool ok = qrdec[i].fullDecodingProcess(); + if (ok) + { + decoded_info[i] = qrdec[i].getDecodeInformation(); + straight_barcode[i] = qrdec[i].getStraightBarcode(); + } + else if (std::min(inarr.size().width, inarr.size().height) > 512) + { + const int min_side = std::min(inarr.size().width, inarr.size().height); + double coeff_expansion = min_side / 512; + const int width = cvRound(inarr.size().width / coeff_expansion); + const int height = cvRound(inarr.size().height / coeff_expansion); + Size new_size(width, height); + Mat inarr2; + resize(inarr, inarr2, new_size, 0, 0, INTER_AREA); + for (size_t j = 0; j < 4; j++) + { + src_points[i][j] /= static_cast(coeff_expansion); + } + qrdec[i].init(inarr2, src_points[i]); + ok = qrdec[i].fullDecodingProcess(); + if (ok) + { + decoded_info[i] = qrdec[i].getDecodeInformation(); + straight_barcode[i] = qrdec[i].getStraightBarcode(); + } + } + if (decoded_info[i].empty()) + decoded_info[i] = ""; + } + } + +private: + Mat& inarr; + vector& qrdec; + vector& decoded_info; + vector& straight_barcode; + vector< vector< Point2f > >& src_points; + +}; + +bool QRCodeDetector::decodeMulti( + InputArray img, + InputArray points, + CV_OUT std::vector& decoded_info, + OutputArrayOfArrays straight_qrcode + ) const +{ + Mat inarr; + if (!checkQRInputImage(img, inarr)) + return false; + CV_Assert(points.size().width > 0); + CV_Assert((points.size().width % 4) == 0); + vector< vector< Point2f > > src_points ; + Mat qr_points = points.getMat(); + for (int i = 0; i < points.size().width ; i += 4) + { + vector tempMat = qr_points.colRange(i, i + 4); + if (contourArea(tempMat) > 0.0) + { + src_points.push_back(tempMat); + } + } + CV_Assert(src_points.size() > 0); + vector qrdec(src_points.size()); + vector straight_barcode(src_points.size()); + vector info(src_points.size()); + ParallelDecodeProcess parallelDecodeProcess(inarr, qrdec, info, straight_barcode, src_points); + parallel_for_(Range(0, int(src_points.size())), parallelDecodeProcess); + vector for_copy; + for (size_t i = 0; i < straight_barcode.size(); i++) + { + if (!(straight_barcode[i].empty())) + for_copy.push_back(straight_barcode[i]); + } + straight_barcode = for_copy; + vector tmp_straight_qrcodes; + if (straight_qrcode.needed()) + { + for (size_t i = 0; i < straight_barcode.size(); i++) + { + Mat tmp_straight_qrcode; + tmp_straight_qrcodes.push_back(tmp_straight_qrcode); + straight_barcode[i].convertTo(((OutputArray)tmp_straight_qrcodes[i]), + ((OutputArray)tmp_straight_qrcodes[i]).fixedType() ? + ((OutputArray)tmp_straight_qrcodes[i]).type() : CV_32FC2); + } + straight_qrcode.createSameSize(tmp_straight_qrcodes, CV_32FC2); + straight_qrcode.assign(tmp_straight_qrcodes); + } + decoded_info.clear(); + for (size_t i = 0; i < info.size(); i++) + { + decoded_info.push_back(info[i]); + } + if (!decoded_info.empty()) + return true; + else + return false; +} + +bool QRCodeDetector::detectAndDecodeMulti( + InputArray img, + CV_OUT std::vector& decoded_info, + OutputArray points_, + OutputArrayOfArrays straight_qrcode + ) const +{ + Mat inarr; + if (!checkQRInputImage(img, inarr)) + { + points_.release(); + return false; + } + + vector points; + bool ok = detectMulti(inarr, points); + if (!ok) + { + points_.release(); + return false; + } + updatePointsResult(points_, points); + decoded_info.clear(); + ok = decodeMulti(inarr, points, decoded_info, straight_qrcode); + return ok; +} + +bool decodeQRCodeMulti( + InputArray in, InputArray points, + vector &decoded_info, OutputArrayOfArrays straight_qrcode) +{ + QRCodeDetector qrcode; + vector info; + bool ok = qrcode.decodeMulti(in, points, info, straight_qrcode); + for (size_t i = 0; i < info.size(); i++) + decoded_info.push_back(info[i]); + return ok; +} + +} // namespace diff --git a/modules/objdetect/test/test_qrcode.cpp b/modules/objdetect/test/test_qrcode.cpp index 8c02f3d..d26323e 100644 --- a/modules/objdetect/test/test_qrcode.cpp +++ b/modules/objdetect/test/test_qrcode.cpp @@ -21,7 +21,11 @@ std::string qrcode_images_close[] = { std::string qrcode_images_monitor[] = { "monitor_1.png", "monitor_2.png", "monitor_3.png", "monitor_4.png", "monitor_5.png" }; -// #define UPDATE_QRCODE_TEST_DATA +std::string qrcode_images_multiple[] = { + "2_qrcodes.png", "3_close_qrcodes.png", "3_qrcodes.png", "4_qrcodes.png", + "5_qrcodes.png", "6_qrcodes.png", "7_qrcodes.png", "8_close_qrcodes.png" +}; +//#define UPDATE_QRCODE_TEST_DATA #ifdef UPDATE_QRCODE_TEST_DATA TEST(Objdetect_QRCode, generate_test_data) @@ -134,6 +138,66 @@ TEST(Objdetect_QRCode_Monitor, generate_test_data) file_config.release(); } + +TEST(Objdetect_QRCode_Multi, generate_test_data) +{ + const std::string root = "qrcode/multiple/"; + const std::string dataset_config = findDataFile(root + "dataset_config.json"); + FileStorage file_config(dataset_config, FileStorage::WRITE); + + file_config << "multiple_images" << "[:"; + size_t multiple_count = sizeof(qrcode_images_multiple) / sizeof(qrcode_images_multiple[0]); + for (size_t i = 0; i < multiple_count; i++) + { + file_config << "{:" << "image_name" << qrcode_images_multiple[i]; + std::string image_path = findDataFile(root + qrcode_images_multiple[i]); + Mat src = imread(image_path); + + ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path; + std::vector corners; + EXPECT_TRUE(detectQRCodeMulti(src, corners)); +#ifdef HAVE_QUIRC + std::vector decoded_info; + std::vector straight_barcode; + EXPECT_TRUE(decodeQRCodeMulti(src, corners, decoded_info, straight_barcode)); +#endif + file_config << "x" << "[:"; + for(size_t j = 0; j < corners.size(); j += 4) + { + file_config << "[:"; + for (size_t k = 0; k < 4; k++) + { + file_config << corners[j + k].x; + } + file_config << "]"; + } + file_config << "]"; + file_config << "y" << "[:"; + for(size_t j = 0; j < corners.size(); j += 4) + { + file_config << "[:"; + for (size_t k = 0; k < 4; k++) + { + file_config << corners[j + k].y; + } + file_config << "]"; + } + file_config << "]"; + file_config << "info"; + file_config << "[:"; + + for(size_t j = 0; j < decoded_info.size(); j++) + { + file_config << decoded_info[j]; + } + file_config << "]"; + file_config << "}"; + } + + file_config << "]"; + file_config.release(); +} + #else typedef testing::TestWithParam< std::string > Objdetect_QRCode; @@ -326,9 +390,96 @@ TEST_P(Objdetect_QRCode_Monitor, regression) } } +typedef testing::TestWithParam < std::string > Objdetect_QRCode_Multi; +TEST_P(Objdetect_QRCode_Multi, regression) +{ + const std::string name_current_image = GetParam(); + const std::string root = "qrcode/multiple/"; + const int pixels_error = 3; + + std::string image_path = findDataFile(root + name_current_image); + Mat src = imread(image_path); + ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path; + QRCodeDetector qrcode; + std::vector corners; +#ifdef HAVE_QUIRC + std::vector decoded_info; + std::vector straight_barcode; + EXPECT_TRUE(qrcode.detectAndDecodeMulti(src, decoded_info, corners, straight_barcode)); + ASSERT_FALSE(corners.empty()); + ASSERT_FALSE(decoded_info.empty()); +#else + ASSERT_TRUE(qrcode.detectMulti(src, corners)); +#endif + + const std::string dataset_config = findDataFile(root + "dataset_config.json"); + FileStorage file_config(dataset_config, FileStorage::READ); + ASSERT_TRUE(file_config.isOpened()) << "Can't read validation data: " << dataset_config; + { + FileNode images_list = file_config["multiple_images"]; + size_t images_count = static_cast(images_list.size()); + ASSERT_GT(images_count, 0u) << "Can't find validation data entries in 'test_images': " << dataset_config; + for (size_t index = 0; index < images_count; index++) + { + FileNode config = images_list[(int)index]; + std::string name_test_image = config["image_name"]; + if (name_test_image == name_current_image) + { + for(int j = 0; j < int(corners.size()); j += 4) + { + bool ok = false; + for (int k = 0; k < int(corners.size() / 4); k++) + { + int count_eq_points = 0; + for (int i = 0; i < 4; i++) + { + int x = config["x"][k][i]; + int y = config["y"][k][i]; + if(((abs(corners[j + i].x - x)) <= pixels_error) && ((abs(corners[j + i].y - y)) <= pixels_error)) + count_eq_points++; + } + if (count_eq_points == 4) + { + ok = true; + break; + } + } + EXPECT_TRUE(ok); + } + +#ifdef HAVE_QUIRC + size_t count_eq_info = 0; + for(int i = 0; i < int(decoded_info.size()); i++) + { + for(int j = 0; j < int(decoded_info.size()); j++) + { + std::string original_info = config["info"][j]; + if(original_info == decoded_info[i]) + { + count_eq_info++; + break; + } + } + } + EXPECT_EQ(decoded_info.size(), count_eq_info); +#endif + + return; // done + } + } + std::cerr + << "Not found results for '" << name_current_image + << "' image in config file:" << dataset_config << std::endl + << "Re-run tests with enabled UPDATE_QRCODE_TEST_DATA macro to update test data." + << std::endl; + } +} + + INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode, testing::ValuesIn(qrcode_images_name)); INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode_Close, testing::ValuesIn(qrcode_images_close)); INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode_Monitor, testing::ValuesIn(qrcode_images_monitor)); +INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode_Multi, testing::ValuesIn(qrcode_images_multiple)); TEST(Objdetect_QRCode_basic, not_found_qrcode) { diff --git a/samples/cpp/qrcode.cpp b/samples/cpp/qrcode.cpp index 1e938f6..af332d3 100644 --- a/samples/cpp/qrcode.cpp +++ b/samples/cpp/qrcode.cpp @@ -2,23 +2,45 @@ #include "opencv2/imgproc.hpp" #include "opencv2/highgui.hpp" #include "opencv2/videoio.hpp" +#include "opencv2/imgcodecs.hpp" #include #include using namespace std; using namespace cv; -static void drawQRCodeContour(Mat &color_image, vector transform); -static void drawFPS(Mat &color_image, double fps); -static int liveQRCodeDetect(const string& out_file); -static int imageQRCodeDetect(const string& in_file, const string& out_file); +static int liveQRCodeDetect(); +static int imageQRCodeDetect(const string& in_file); + +static bool g_modeMultiQR = false; +static bool g_detectOnly = false; + +static string g_out_file_name, g_out_file_ext; +static int g_save_idx = 0; + +static bool g_saveDetections = false; +static bool g_saveAll = false; + +static string getQRModeString() +{ + std::ostringstream out; + out << "QR" + << (g_modeMultiQR ? " multi" : "") + << (g_detectOnly ? " detector" : " decoder"); + return out.str(); +} int main(int argc, char *argv[]) { const string keys = "{h help ? | | print help messages }" - "{i in | | input path to file for detect (with parameter - show image, otherwise - camera)}" - "{o out | | output path to file (save image, work with -i parameter) }"; + "{i in | | input image path (also switches to image detection mode) }" + "{detect | false | detect QR code only (skip decoding) }" + "{m multi | | use detect for multiple qr-codes }" + "{o out | qr_code.png | path to result file }" + "{save_detections | false | save all QR detections (video mode only) }" + "{save_all | false | save all processed frames (video mode only) }" + ; CommandLineParser cmd_parser(argc, argv, keys); cmd_parser.about("This program detects the QR-codes from camera or images using the OpenCV library."); @@ -28,32 +50,51 @@ int main(int argc, char *argv[]) return 0; } - string in_file_name = cmd_parser.get("in"); // input path to image - string out_file_name; - if (cmd_parser.has("out")) - out_file_name = cmd_parser.get("out"); // output path to image + string in_file_name = cmd_parser.get("in"); // path to input image + if (cmd_parser.has("out")) + { + std::string fpath = cmd_parser.get("out"); // path to output image + std::string::size_type idx = fpath.rfind('.'); + if (idx != std::string::npos) + { + g_out_file_name = fpath.substr(0, idx); + g_out_file_ext = fpath.substr(idx); + } + else + { + g_out_file_name = fpath; + g_out_file_ext = ".png"; + } + } if (!cmd_parser.check()) { cmd_parser.printErrors(); return -1; } + g_modeMultiQR = cmd_parser.has("multi") && cmd_parser.get("multi"); + g_detectOnly = cmd_parser.has("detect") && cmd_parser.get("detect"); + + g_saveDetections = cmd_parser.has("save_detections") && cmd_parser.get("save_detections"); + g_saveAll = cmd_parser.has("save_all") && cmd_parser.get("save_all"); + int return_code = 0; if (in_file_name.empty()) { - return_code = liveQRCodeDetect(out_file_name); + return_code = liveQRCodeDetect(); } else { - return_code = imageQRCodeDetect(samples::findFile(in_file_name), out_file_name); + return_code = imageQRCodeDetect(samples::findFile(in_file_name)); } return return_code; } -void drawQRCodeContour(Mat &color_image, vector transform) +static +void drawQRCodeContour(Mat &color_image, const vector& corners) { - if (!transform.empty()) + if (!corners.empty()) { double show_radius = (color_image.rows > color_image.cols) ? (2.813 * color_image.rows) / color_image.cols @@ -61,127 +102,246 @@ void drawQRCodeContour(Mat &color_image, vector transform) double contour_radius = show_radius * 0.4; vector< vector > contours; - contours.push_back(transform); + contours.push_back(corners); drawContours(color_image, contours, 0, Scalar(211, 0, 148), cvRound(contour_radius)); RNG rng(1000); for (size_t i = 0; i < 4; i++) { Scalar color = Scalar(rng.uniform(0,255), rng.uniform(0, 255), rng.uniform(0, 255)); - circle(color_image, transform[i], cvRound(show_radius), color, -1); + circle(color_image, corners[i], cvRound(show_radius), color, -1); } } } +static void drawFPS(Mat &color_image, double fps) { ostringstream convert; - convert << cvRound(fps) << " FPS (QR detection)"; + convert << cv::format("%.2f", fps) << " FPS (" << getQRModeString() << ")"; putText(color_image, convert.str(), Point(25, 25), FONT_HERSHEY_DUPLEX, 1, Scalar(0, 0, 255), 2); } -int liveQRCodeDetect(const string& out_file) +static +void drawQRCodeResults(Mat& frame, const vector& corners, const vector& decode_info, double fps) +{ + if (!corners.empty()) + { + for (size_t i = 0; i < corners.size(); i += 4) + { + size_t qr_idx = i / 4; + vector qrcode_contour(corners.begin() + i, corners.begin() + i + 4); + drawQRCodeContour(frame, qrcode_contour); + + cout << "QR[" << qr_idx << "] @ " << Mat(qrcode_contour).reshape(2, 1) << ": "; + if (decode_info.size() > qr_idx) + { + if (!decode_info[qr_idx].empty()) + cout << "'" << decode_info[qr_idx] << "'" << endl; + else + cout << "can't decode QR code" << endl; + } + else + { + cout << "decode information is not available (disabled)" << endl; + } + } + } + else + { + cout << "QR code is not detected" << endl; + } + + drawFPS(frame, fps); +} + +static +void runQR( + QRCodeDetector& qrcode, const Mat& input, + vector& corners, vector& decode_info + // +global: bool g_modeMultiQR, bool g_detectOnly +) +{ + if (!g_modeMultiQR) + { + if (!g_detectOnly) + { + String decode_info1 = qrcode.detectAndDecode(input, corners); + decode_info.push_back(decode_info1); + } + else + { + bool detection_result = qrcode.detect(input, corners); + CV_UNUSED(detection_result); + } + } + else + { + if (!g_detectOnly) + { + bool result_detection = qrcode.detectAndDecodeMulti(input, decode_info, corners); + CV_UNUSED(result_detection); + } + else + { + bool result_detection = qrcode.detectMulti(input, corners); + CV_UNUSED(result_detection); + } + } +} + +static +double processQRCodeDetection(QRCodeDetector& qrcode, const Mat& input, Mat& result, vector& corners) +{ + if (input.channels() == 1) + cvtColor(input, result, COLOR_GRAY2BGR); + else + input.copyTo(result); + + cout << "Run " << getQRModeString() + << " on image: " << input.size() << " (" << typeToString(input.type()) << ")" + << endl; + + TickMeter timer; + + vector decode_info; + timer.start(); + runQR(qrcode, input, corners, decode_info); + timer.stop(); + + double fps = 1 / timer.getTimeSec(); + drawQRCodeResults(result, corners, decode_info, fps); + + return fps; +} + +int liveQRCodeDetect() { VideoCapture cap(0); - if(!cap.isOpened()) + + if (!cap.isOpened()) { cout << "Cannot open a camera" << endl; - return -4; + return 2; } + cout << "Press 'm' to switch between detectAndDecode and detectAndDecodeMulti" << endl; + cout << "Press 'd' to switch between decoder and detector" << endl; + cout << "Press ' ' (space) to save result into images" << endl; + cout << "Press 'ESC' to exit" << endl; QRCodeDetector qrcode; - TickMeter total; - for(;;) + + for (;;) { - Mat frame, src, straight_barcode; - string decode_info; - vector transform; + Mat frame; cap >> frame; if (frame.empty()) { cout << "End of video stream" << endl; break; } - cvtColor(frame, src, COLOR_BGR2GRAY); - total.start(); - bool result_detection = qrcode.detect(src, transform); - if (result_detection) + bool forceSave = g_saveAll; + + Mat result; + + try + { + vector corners; + double fps = processQRCodeDetection(qrcode, frame, result, corners); + cout << "FPS: " << fps << endl; + forceSave |= (g_saveDetections && !corners.empty()); + //forceSave |= fps < 1.0; + } + catch (const cv::Exception& e) { - decode_info = qrcode.decode(src, transform, straight_barcode); - if (!decode_info.empty()) { cout << decode_info << endl; } + cerr << "ERROR exception: " << e.what() << endl; + forceSave = true; } - total.stop(); - double fps = 1 / total.getTimeSec(); - total.reset(); - if (result_detection) { drawQRCodeContour(frame, transform); } - drawFPS(frame, fps); + if (!result.empty()) + imshow("QR code", result); + + int code = waitKey(1); + if (code < 0 && !forceSave) + continue; // timeout + char c = (char)code; + if (c == ' ' || forceSave) + { + string fsuffix = cv::format("-%05d", g_save_idx++); + + string fname_input = g_out_file_name + fsuffix + "_input.png"; + cout << "Saving QR code detection input: '" << fname_input << "' ..." << endl; + imwrite(fname_input, frame); + + string fname = g_out_file_name + fsuffix + g_out_file_ext; + cout << "Saving QR code detection result: '" << fname << "' ..." << endl; + imwrite(fname, result); - imshow("Live QR code detector", frame); - char c = (char)waitKey(30); + cout << "Saved" << endl; + } + if (c == 'm') + { + g_modeMultiQR = !g_modeMultiQR; + cout << "Switching QR code mode ==> " << (g_modeMultiQR ? "detectAndDecodeMulti" : "detectAndDecode") << endl; + } + if (c == 'd') + { + g_detectOnly = !g_detectOnly; + cout << "Switching QR decoder mode ==> " << (g_detectOnly ? "detect" : "decode") << endl; + } if (c == 27) + { + cout << "'ESC' is pressed. Exiting..." << endl; break; - if (c == ' ' && !out_file.empty()) - imwrite(out_file, frame); // TODO write original frame too + } } + cout << "Exit." << endl; + return 0; } -int imageQRCodeDetect(const string& in_file, const string& out_file) +int imageQRCodeDetect(const string& in_file) { - Mat color_src = imread(in_file, IMREAD_COLOR), src; - cvtColor(color_src, src, COLOR_BGR2GRAY); - Mat straight_barcode; - string decoded_info; - vector transform; const int count_experiments = 10; - double transform_time = 0.0; - bool result_detection = false; - TickMeter total; + + Mat input = imread(in_file, IMREAD_COLOR); + cout << "Run " << getQRModeString() + << " on image: " << input.size() << " (" << typeToString(input.type()) << ")" + << endl; + QRCodeDetector qrcode; + vector corners; + vector decode_info; + + TickMeter timer; for (size_t i = 0; i < count_experiments; i++) { - total.start(); - transform.clear(); - result_detection = qrcode.detect(src, transform); - total.stop(); - transform_time += total.getTimeSec(); - total.reset(); - if (!result_detection) - continue; - - total.start(); - decoded_info = qrcode.decode(src, transform, straight_barcode); - total.stop(); - transform_time += total.getTimeSec(); - total.reset(); + corners.clear(); + decode_info.clear(); + + timer.start(); + runQR(qrcode, input, corners, decode_info); + timer.stop(); } - double fps = count_experiments / transform_time; - if (!result_detection) - cout << "QR code not found" << endl; - if (decoded_info.empty()) - cout << "QR code cannot be decoded" << endl; - - drawQRCodeContour(color_src, transform); - drawFPS(color_src, fps); - - cout << "Input image file path: " << in_file << endl; - cout << "Output image file path: " << out_file << endl; - cout << "Size: " << color_src.size() << endl; + double fps = count_experiments / timer.getTimeSec(); cout << "FPS: " << fps << endl; - cout << "Decoded info: " << decoded_info << endl; - if (!out_file.empty()) - { - imwrite(out_file, color_src); - } + Mat result; input.copyTo(result); + drawQRCodeResults(result, corners, decode_info, fps); + + imshow("QR", result); waitKey(1); - for(;;) + if (!g_out_file_name.empty()) { - imshow("Detect QR code on image", color_src); - if (waitKey(0) == 27) - break; + string out_file = g_out_file_name + g_out_file_ext; + cout << "Saving result: " << out_file << endl; + imwrite(out_file, result); } + + cout << "Press any key to exit ..." << endl; + waitKey(0); + cout << "Exit." << endl; + return 0; } -- 2.7.4