From 9d871abd32e7515f8cfe81c35f714a5ee3d4eaeb Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Wed, 11 Apr 2012 14:02:10 +0000 Subject: [PATCH] Refactored videostab module. Added normalization into motion estimators. --- .../include/opencv2/videostab/global_motion.hpp | 23 ++-- modules/videostab/src/global_motion.cpp | 123 ++++++++++++++------- modules/videostab/src/stabilizer.cpp | 17 ++- modules/videostab/src/wobble_suppression.cpp | 2 +- samples/cpp/videostab.cpp | 18 ++- 5 files changed, 126 insertions(+), 57 deletions(-) diff --git a/modules/videostab/include/opencv2/videostab/global_motion.hpp b/modules/videostab/include/opencv2/videostab/global_motion.hpp index 0dd5e8b..c523c5a 100644 --- a/modules/videostab/include/opencv2/videostab/global_motion.hpp +++ b/modules/videostab/include/opencv2/videostab/global_motion.hpp @@ -66,8 +66,7 @@ enum MotionModel }; CV_EXPORTS Mat estimateGlobalMotionLeastSquares( - const std::vector &points0, const std::vector &points1, - int model = AFFINE, float *rmse = 0); + int npoints, Point2f *points0, Point2f *points1, int model = AFFINE, float *rmse = 0); struct CV_EXPORTS RansacParams { @@ -80,16 +79,24 @@ struct CV_EXPORTS RansacParams RansacParams(int size, float thresh, float eps, float prob) : size(size), thresh(thresh), eps(eps), prob(prob) {} - static RansacParams translation2dMotionStd() { return RansacParams(1, 0.5f, 0.5f, 0.99f); } - static RansacParams translationAndScale2dMotionStd() { return RansacParams(2, 0.5f, 0.5f, 0.99f); } - static RansacParams linearSimilarity2dMotionStd() { return RansacParams(2, 0.5f, 0.5f, 0.99f); } - static RansacParams affine2dMotionStd() { return RansacParams(3, 0.5f, 0.5f, 0.99f); } - static RansacParams homography2dMotionStd() { return RansacParams(4, 0.5f, 0.5f, 0.99f); } + static RansacParams default2dMotion(MotionModel model) + { + CV_Assert(model < UNKNOWN); + if (model == TRANSLATION) + return RansacParams(1, 0.5f, 0.5f, 0.99f); + if (model == TRANSLATION_AND_SCALE) + return RansacParams(2, 0.5f, 0.5f, 0.99f); + if (model == LINEAR_SIMILARITY) + return RansacParams(2, 0.5f, 0.5f, 0.99f); + if (model == AFFINE) + return RansacParams(3, 0.5f, 0.5f, 0.99f); + return RansacParams(4, 0.5f, 0.5f, 0.99f); + } }; CV_EXPORTS Mat estimateGlobalMotionRobust( const std::vector &points0, const std::vector &points1, - int model = AFFINE, const RansacParams ¶ms = RansacParams::affine2dMotionStd(), + int model = AFFINE, const RansacParams ¶ms = RansacParams::default2dMotion(AFFINE), float *rmse = 0, int *ninliers = 0); class CV_EXPORTS GlobalMotionEstimatorBase diff --git a/modules/videostab/src/global_motion.cpp b/modules/videostab/src/global_motion.cpp index 81a5d4f..c419324 100644 --- a/modules/videostab/src/global_motion.cpp +++ b/modules/videostab/src/global_motion.cpp @@ -51,8 +51,44 @@ namespace cv namespace videostab { +// does isotropic normalization +static Mat normalizePoints(int npoints, Point2f *points) +{ + float cx = 0.f, cy = 0.f; + for (int i = 0; i < npoints; ++i) + { + cx += points[i].x; + cy += points[i].y; + } + cx /= npoints; + cy /= npoints; + + float d = 0.f; + for (int i = 0; i < npoints; ++i) + { + points[i].x -= cx; + points[i].y -= cy; + d += sqrt(sqr(points[i].x) + sqr(points[i].y)); + } + d /= npoints; + + float s = sqrt(2.f) / d; + for (int i = 0; i < npoints; ++i) + { + points[i].x *= s; + points[i].y *= s; + } + + Mat_ T = Mat::eye(3, 3, CV_32F); + T(0,0) = T(1,1) = s; + T(0,2) = -cx*s; + T(1,2) = -cy*s; + return T; +} + + static Mat estimateGlobMotionLeastSquaresTranslation( - int npoints, const Point2f *points0, const Point2f *points1, float *rmse) + int npoints, Point2f *points0, Point2f *points1, float *rmse) { Mat_ M = Mat::eye(3, 3, CV_32F); for (int i = 0; i < npoints; ++i) @@ -62,6 +98,7 @@ static Mat estimateGlobMotionLeastSquaresTranslation( } M(0,2) /= npoints; M(1,2) /= npoints; + if (rmse) { *rmse = 0; @@ -70,13 +107,17 @@ static Mat estimateGlobMotionLeastSquaresTranslation( sqr(points1[i].y - points0[i].y - M(1,2)); *rmse = sqrt(*rmse / npoints); } + return M; } static Mat estimateGlobMotionLeastSquaresTranslationAndScale( - int npoints, const Point2f *points0, const Point2f *points1, float *rmse) + int npoints, Point2f *points0, Point2f *points1, float *rmse) { + Mat_ T0 = normalizePoints(npoints, points0); + Mat_ T1 = normalizePoints(npoints, points1); + Mat_ A(2*npoints, 3), b(2*npoints, 1); float *a0, *a1; Point2f p0, p1; @@ -103,13 +144,17 @@ static Mat estimateGlobMotionLeastSquaresTranslationAndScale( M(0,0) = M(1,1) = sol(0,0); M(0,2) = sol(1,0); M(1,2) = sol(2,0); - return M; + + return T1.inv() * M * T0; } static Mat estimateGlobMotionLeastSquaresLinearSimilarity( - int npoints, const Point2f *points0, const Point2f *points1, float *rmse) + int npoints, Point2f *points0, Point2f *points1, float *rmse) { + Mat_ T0 = normalizePoints(npoints, points0); + Mat_ T1 = normalizePoints(npoints, points1); + Mat_ A(2*npoints, 4), b(2*npoints, 1); float *a0, *a1; Point2f p0, p1; @@ -138,13 +183,17 @@ static Mat estimateGlobMotionLeastSquaresLinearSimilarity( M(1,0) = -sol(1,0); M(0,2) = sol(2,0); M(1,2) = sol(3,0); - return M; + + return T1.inv() * M * T0; } static Mat estimateGlobMotionLeastSquaresAffine( - int npoints, const Point2f *points0, const Point2f *points1, float *rmse) + int npoints, Point2f *points0, Point2f *points1, float *rmse) { + Mat_ T0 = normalizePoints(npoints, points0); + Mat_ T1 = normalizePoints(npoints, points1); + Mat_ A(2*npoints, 6), b(2*npoints, 1); float *a0, *a1; Point2f p0, p1; @@ -172,24 +221,22 @@ static Mat estimateGlobMotionLeastSquaresAffine( for (int j = 0; j < 3; ++j, ++k) M(i,j) = sol(k,0); - return M; + return T1.inv() * M * T0; } Mat estimateGlobalMotionLeastSquares( - const vector &points0, const vector &points1, int model, float *rmse) + int npoints, Point2f *points0, Point2f *points1, int model, float *rmse) { CV_Assert(model <= AFFINE); - CV_Assert(points0.size() == points1.size()); - typedef Mat (*Impl)(int, const Point2f*, const Point2f*, float*); + typedef Mat (*Impl)(int, Point2f*, Point2f*, float*); static Impl impls[] = { estimateGlobMotionLeastSquaresTranslation, estimateGlobMotionLeastSquaresTranslationAndScale, estimateGlobMotionLeastSquaresLinearSimilarity, estimateGlobMotionLeastSquaresAffine }; - int npoints = static_cast(points0.size()); - return impls[model](npoints, &points0[0], &points1[0], rmse); + return impls[model](npoints, points0, points1, rmse); } @@ -200,22 +247,22 @@ Mat estimateGlobalMotionRobust( CV_Assert(model <= AFFINE); CV_Assert(points0.size() == points1.size()); - typedef Mat (*Impl)(int, const Point2f*, const Point2f*, float*); - static Impl impls[] = { estimateGlobMotionLeastSquaresTranslation, - estimateGlobMotionLeastSquaresTranslationAndScale, - estimateGlobMotionLeastSquaresLinearSimilarity, - estimateGlobMotionLeastSquaresAffine }; - const int npoints = static_cast(points0.size()); const int niters = static_cast(ceil(log(1 - params.prob) / log(1 - pow(1 - params.eps, params.size)))); - RNG rng(0); + // current hypothesis vector indices(params.size); - vector subset0(params.size), subset1(params.size); - vector subset0best(params.size), subset1best(params.size); + vector subset0(params.size); + vector subset1(params.size); + + // best hypothesis + vector subset0best(params.size); + vector subset1best(params.size); Mat_ bestM; int ninliersMax = -1; + + RNG rng(0); Point2f p0, p1; float x, y; @@ -239,7 +286,8 @@ Mat estimateGlobalMotionRobust( subset1[i] = points1[indices[i]]; } - Mat_ M = impls[model](params.size, &subset0[0], &subset1[0], 0); + Mat_ M = estimateGlobalMotionLeastSquares( + params.size, &subset0[0], &subset1[0], model, 0); int ninliers = 0; for (int i = 0; i < npoints; ++i) @@ -260,8 +308,9 @@ Mat estimateGlobalMotionRobust( } if (ninliersMax < params.size) - // compute rmse - bestM = impls[model](params.size, &subset0best[0], &subset1best[0], rmse); + // compute RMSE + bestM = estimateGlobalMotionLeastSquares( + params.size, &subset0best[0], &subset1best[0], model, rmse); else { subset0.resize(ninliersMax); @@ -278,7 +327,8 @@ Mat estimateGlobalMotionRobust( j++; } } - bestM = impls[model](ninliersMax, &subset0[0], &subset1[0], rmse); + bestM = estimateGlobalMotionLeastSquares( + ninliersMax, &subset0[0], &subset1[0], model, rmse); } if (ninliers) @@ -332,16 +382,11 @@ PyrLkRobustMotionEstimator::PyrLkRobustMotionEstimator(MotionModel model) setDetector(new GoodFeaturesToTrackDetector()); setOptFlowEstimator(new SparsePyrLkOptFlowEstimator()); setMotionModel(model); - if (model == TRANSLATION) - setRansacParams(RansacParams::translation2dMotionStd()); - else if (model == TRANSLATION_AND_SCALE) - setRansacParams(RansacParams::translationAndScale2dMotionStd()); - else if (model == LINEAR_SIMILARITY) - setRansacParams(RansacParams::linearSimilarity2dMotionStd()); - else if (model == AFFINE) - setRansacParams(RansacParams::affine2dMotionStd()); - else if (model == HOMOGRAPHY) - setRansacParams(RansacParams::homography2dMotionStd()); + + RansacParams ransac = RansacParams::default2dMotion(model); + ransac.size *= 2; // we use more points than needed, but result looks better + setRansacParams(ransac); + setMaxRmse(0.5f); setMinInlierRatio(0.1f); setGridSize(Size(0,0)); @@ -362,6 +407,7 @@ Mat PyrLkRobustMotionEstimator::estimate(const Mat &frame0, const Mat &frame1, b keypointsPrev_.push_back(KeyPoint((x+1)*dx, (y+1)*dy, 0.f)); } + // draw keypoints /*Mat img; drawKeypoints(frame0, keypointsPrev_, img); imshow("frame0_keypoints", img); @@ -374,10 +420,9 @@ Mat PyrLkRobustMotionEstimator::estimate(const Mat &frame0, const Mat &frame1, b optFlowEstimator_->run(frame0, frame1, pointsPrev_, points_, status_, noArray()); size_t npoints = points_.size(); - pointsPrevGood_.clear(); - pointsPrevGood_.reserve(npoints); - pointsGood_.clear(); - pointsGood_.reserve(npoints); + pointsPrevGood_.clear(); pointsPrevGood_.reserve(npoints); + pointsGood_.clear(); pointsGood_.reserve(npoints); + for (size_t i = 0; i < npoints; ++i) { if (status_[i]) diff --git a/modules/videostab/src/stabilizer.cpp b/modules/videostab/src/stabilizer.cpp index 06d6246..67b7491 100644 --- a/modules/videostab/src/stabilizer.cpp +++ b/modules/videostab/src/stabilizer.cpp @@ -338,16 +338,18 @@ void TwoPassStabilizer::runPrePassIfNecessary() WobbleSuppressorBase *wobbleSuppressor = static_cast(wobbleSuppressor_); doWobbleSuppression_ = dynamic_cast(wobbleSuppressor) == 0; - bool okEst; + bool ok = true, ok2 = true; + while (!(frame = frameSource_->nextFrame()).empty()) { if (frameCount_ > 0) { - motions_.push_back(motionEstimator_->estimate(prevFrame, frame)); + motions_.push_back(motionEstimator_->estimate(prevFrame, frame, &ok)); + if (doWobbleSuppression_) { - Mat M = wobbleSuppressor_->motionEstimator()->estimate(prevFrame, frame, &okEst); - if (okEst) + Mat M = wobbleSuppressor_->motionEstimator()->estimate(prevFrame, frame, &ok2); + if (ok2) motions2_.push_back(M); else motions2_.push_back(motions_.back()); @@ -363,7 +365,12 @@ void TwoPassStabilizer::runPrePassIfNecessary() prevFrame = frame; frameCount_++; - log_->print("."); + if (ok) + { + if (ok2) log_->print("."); + else log_->print("?"); + } + else log_->print("x"); } for (int i = 0; i < radius_; ++i) diff --git a/modules/videostab/src/wobble_suppression.cpp b/modules/videostab/src/wobble_suppression.cpp index 1a05211..450ec0c 100644 --- a/modules/videostab/src/wobble_suppression.cpp +++ b/modules/videostab/src/wobble_suppression.cpp @@ -55,7 +55,7 @@ WobbleSuppressorBase::WobbleSuppressorBase() : motions_(0), stabilizationMotions { PyrLkRobustMotionEstimator *est = new PyrLkRobustMotionEstimator(); est->setMotionModel(HOMOGRAPHY); - est->setRansacParams(RansacParams::homography2dMotionStd()); + est->setRansacParams(RansacParams::default2dMotion(HOMOGRAPHY)); setMotionEstimator(est); } diff --git a/samples/cpp/videostab.cpp b/samples/cpp/videostab.cpp index 10e0b64..f401dcc 100644 --- a/samples/cpp/videostab.cpp +++ b/samples/cpp/videostab.cpp @@ -67,6 +67,8 @@ void printHelp() "Arguments:\n" " -m, --model=(transl|transl_and_scale|linear_sim|affine|homography)\n" " Set motion model. The default is affine.\n" + " --subset=(|auto)\n" + " Number of random samples per one motion hypothesis. The default is auto.\n" " --outlier-ratio=\n" " Motion estimation outlier ratio hypothesis. The default is 0.5.\n" " --min-inlier-ratio=\n" @@ -116,10 +118,12 @@ void printHelp() " --ws-model=(transl|transl_and_scale|linear_sim|affine|homography)\n" " Set wobble suppression motion model (must have more DOF than motion \n" " estimation model). The default is homography.\n" - " --ws-min-inlier-ratio=\n" - " Minimum inlier ratio to decide if estimated motion is OK. The default is 0.1.\n" + " --ws-subset=(|auto)\n" + " Number of random samples per one motion hypothesis. The default is auto.\n" " --ws-outlier-ratio=\n" " Motion estimation outlier ratio hypothesis. The default is 0.5.\n" + " --ws-min-inlier-ratio=\n" + " Minimum inlier ratio to decide if estimated motion is OK. The default is 0.1.\n" " --ws-nkps=\n" " Number of keypoints to find in each frame. The default is 1000.\n" " --ws-extra-kps=\n" @@ -147,8 +151,9 @@ int main(int argc, const char **argv) const char *keys = "{ 1 | | | | }" "{ m | model | affine| }" - "{ | min-inlier-ratio | 0.1 | }" + "{ | subset | auto | }" "{ | outlier-ratio | 0.5 | }" + "{ | min-inlier-ratio | 0.1 | }" "{ | nkps | 1000 | }" "{ | extra-kps | 0 | }" "{ sm | save-motions | no | }" @@ -170,8 +175,9 @@ int main(int argc, const char **argv) "{ ws | wobble-suppress | no | }" "{ | ws-period | 30 | }" "{ | ws-model | homography | }" - "{ | ws-min-inlier-ratio | 0.1 | }" + "{ | ws-subset | auto | }" "{ | ws-outlier-ratio | 0.5 | }" + "{ | ws-min-inlier-ratio | 0.1 | }" "{ | ws-nkps | 1000 | }" "{ | ws-extra-kps | 0 | }" "{ sm2 | save-motions2 | no | }" @@ -230,6 +236,8 @@ int main(int argc, const char **argv) est->setDetector(new GoodFeaturesToTrackDetector(argi("ws-nkps"))); RansacParams ransac = est->ransacParams(); + if (arg("ws-subset") != "auto") + ransac.size = argi("ws-subset"); ransac.eps = argf("ws-outlier-ratio"); est->setRansacParams(ransac); est->setMinInlierRatio(argf("ws-min-inlier-ratio")); @@ -288,6 +296,8 @@ int main(int argc, const char **argv) est->setDetector(new GoodFeaturesToTrackDetector(argi("nkps"))); RansacParams ransac = est->ransacParams(); + if (arg("subset") != "auto") + ransac.size = argi("subset"); ransac.eps = argf("outlier-ratio"); est->setRansacParams(ransac); est->setMinInlierRatio(argf("min-inlier-ratio")); -- 2.7.4