From: Nathan Godwin <nathanrgodwin@gmail.com>
Date: Tue, 22 Sep 2020 02:56:28 +0000 (-0500)
Subject: Fixed assertions on ippe solver
X-Git-Tag: submit/tizen/20210224.033012~2^2~15^2~15^2
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=2f9072efdcf0d72023be31f174bd1bf9abd03757;p=platform%2Fupstream%2Fopencv.git

Fixed assertions on ippe solver
---

diff --git a/modules/calib3d/src/ippe.cpp b/modules/calib3d/src/ippe.cpp
index 3f1e6a7add..ec6089596c 100644
--- a/modules/calib3d/src/ippe.cpp
+++ b/modules/calib3d/src/ippe.cpp
@@ -77,18 +77,18 @@ void PoseSolver::solveGeneric(InputArray _objectPoints, InputArray _normalizedIn
                               OutputArray _Ma, OutputArray _Mb)
 {
     //argument checking:
-    size_t n = static_cast<size_t>(_objectPoints.rows() * _objectPoints.cols()); //number of points
+    size_t n = static_cast<size_t>(_normalizedInputPoints.rows()) * static_cast<size_t>(_normalizedInputPoints.cols()); //number of points
     int objType = _objectPoints.type();
     int type_input = _normalizedInputPoints.type();
 
     CV_CheckType(objType, objType == CV_32FC3 || objType == CV_64FC3,
                  "Type of _objectPoints must be CV_32FC3 or CV_64FC3" );
     CV_CheckType(type_input, type_input == CV_32FC2 || type_input == CV_64FC2,
-                 "Type of _normalizedInputPoints must be CV_32FC3 or CV_64FC3" );
+                 "Type of _normalizedInputPoints must be CV_32FC2 or CV_64FC2" );
     CV_Assert(_objectPoints.rows() == 1 || _objectPoints.cols() == 1);
     CV_Assert(_objectPoints.rows() >= 4 || _objectPoints.cols() >= 4);
     CV_Assert(_normalizedInputPoints.rows() == 1 || _normalizedInputPoints.cols() == 1);
-    CV_Assert(static_cast<size_t>(_objectPoints.rows() * _objectPoints.cols()) == n);
+    CV_Assert(static_cast<size_t>(_objectPoints.rows()) * static_cast<size_t>(_objectPoints.cols()) == n);
 
     Mat normalizedInputPoints;
     if (type_input == CV_32FC2)
@@ -101,7 +101,7 @@ void PoseSolver::solveGeneric(InputArray _objectPoints, InputArray _normalizedIn
     }
 
     Mat objectInputPoints;
-    if (type_input == CV_32FC3)
+    if (objType == CV_32FC3)
     {
         _objectPoints.getMat().convertTo(objectInputPoints, CV_64F);
     }