From: Vladislav Sovrasov <sovrasov.vlad@gmail.com>
Date: Wed, 28 Jun 2017 09:03:07 +0000 (+0300)
Subject: core: fix infinite recursion in compare
X-Git-Tag: accepted/tizen/6.0/unified/20201030.111113~880^2
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=35a1ecef2a7d28d98412a213a49a6afae96e1ee8;p=platform%2Fupstream%2Fopencv.git

core: fix infinite recursion in compare
---

diff --git a/modules/core/src/arithm.cpp b/modules/core/src/arithm.cpp
index 37a2303..b4406c4 100644
--- a/modules/core/src/arithm.cpp
+++ b/modules/core/src/arithm.cpp
@@ -1239,7 +1239,10 @@ void cv::compare(InputArray _src1, InputArray _src2, OutputArray _dst, int op)
             || !_src1.sameSize(_src2)
             || _src1.type() != _src2.type())
     {
-        if (checkScalar(_src1, _src2.type(), _src1.kind(), _src2.kind()))
+        bool is_src1_scalar = checkScalar(_src1, _src2.type(), _src1.kind(), _src2.kind());
+        bool is_src2_scalar = checkScalar(_src2, _src1.type(), _src2.kind(), _src1.kind());
+
+        if (is_src1_scalar && !is_src2_scalar)
         {
             op = op == CMP_LT ? CMP_GT : op == CMP_LE ? CMP_GE :
                 op == CMP_GE ? CMP_LE : op == CMP_GT ? CMP_LT : op;
@@ -1247,7 +1250,7 @@ void cv::compare(InputArray _src1, InputArray _src2, OutputArray _dst, int op)
             compare(_src2, _src1, _dst, op);
             return;
         }
-        else if( !checkScalar(_src2, _src1.type(), _src2.kind(), _src1.kind()) )
+        else if( (is_src1_scalar && is_src2_scalar) || (!is_src1_scalar && !is_src2_scalar) )
             CV_Error( CV_StsUnmatchedSizes,
                      "The operation is neither 'array op array' (where arrays have the same size and the same type), "
                      "nor 'array op scalar', nor 'scalar op array'" );
diff --git a/modules/core/test/test_arithm.cpp b/modules/core/test/test_arithm.cpp
index 58bec68..bac5c7f 100644
--- a/modules/core/test/test_arithm.cpp
+++ b/modules/core/test/test_arithm.cpp
@@ -1922,3 +1922,13 @@ TEST(Compare, empty)
     EXPECT_TRUE(dst1.empty());
     EXPECT_TRUE(dst2.empty());
 }
+
+TEST(Compare, regression_8999)
+{
+    Mat_<double> A(4,1); A << 1, 3, 2, 4;
+    Mat_<double> B(1,1); B << 2;
+    Mat C;
+    ASSERT_ANY_THROW({
+                        compare(A, B, C, CMP_LT);
+                     });
+}