From 71391eb76fa8af00c7394929ddafcbe7c8fdf6f6 Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Wed, 25 Apr 2012 13:21:38 +0000 Subject: [PATCH] Added LP based global motion estimation (videostab) --- .../include/opencv2/videostab/global_motion.hpp | 94 +++++-- modules/videostab/src/clp.hpp | 63 +++++ modules/videostab/src/global_motion.cpp | 210 +++++++++++++- modules/videostab/src/motion_stabilizing.cpp | 26 +- modules/videostab/src/stabilizer.cpp | 2 +- modules/videostab/src/wobble_suppression.cpp | 2 +- samples/cpp/videostab.cpp | 310 ++++++++++++++------- 7 files changed, 541 insertions(+), 166 deletions(-) create mode 100644 modules/videostab/src/clp.hpp diff --git a/modules/videostab/include/opencv2/videostab/global_motion.hpp b/modules/videostab/include/opencv2/videostab/global_motion.hpp index e785645..4dfb3dc 100644 --- a/modules/videostab/include/opencv2/videostab/global_motion.hpp +++ b/modules/videostab/include/opencv2/videostab/global_motion.hpp @@ -57,6 +57,8 @@ #include "opencv2/gpu/gpu.hpp" #endif +// TODO remove code duplications (in cpp too) + namespace cv { namespace videostab @@ -91,6 +93,7 @@ class CV_EXPORTS FromFileMotionReader : public GlobalMotionEstimatorBase { public: FromFileMotionReader(const std::string &path); + virtual Mat estimate(const Mat &frame0, const Mat &frame1, bool *ok = 0); private: @@ -101,6 +104,7 @@ class CV_EXPORTS ToFileMotionWriter : public GlobalMotionEstimatorBase { public: ToFileMotionWriter(const std::string &path, Ptr estimator); + virtual Mat estimate(const Mat &frame0, const Mat &frame1, bool *ok = 0); private: @@ -108,30 +112,10 @@ private: Ptr estimator_; }; -class CV_EXPORTS PyrLkRobustMotionEstimatorBase : public GlobalMotionEstimatorBase -{ -public: - virtual void setRansacParams(const RansacParams &val) { ransacParams_ = val; } - virtual RansacParams ransacParams() const { return ransacParams_; } - - virtual void setOutlierRejector(Ptr val) { outlierRejector_ = val; } - virtual Ptr outlierRejector() const { return outlierRejector_; } - - virtual void setMinInlierRatio(float val) { minInlierRatio_ = val; } - virtual float minInlierRatio() const { return minInlierRatio_; } - -protected: - PyrLkRobustMotionEstimatorBase(MotionModel model); - - RansacParams ransacParams_; - Ptr outlierRejector_; - float minInlierRatio_; -}; - -class CV_EXPORTS PyrLkRobustMotionEstimator : public PyrLkRobustMotionEstimatorBase +class CV_EXPORTS RansacMotionEstimator : public GlobalMotionEstimatorBase { public: - PyrLkRobustMotionEstimator(MotionModel model = MM_AFFINE); + RansacMotionEstimator(MotionModel model = MM_AFFINE); void setDetector(Ptr val) { detector_ = val; } Ptr detector() const { return detector_; } @@ -142,12 +126,24 @@ public: void setGridSize(Size val) { gridSize_ = val; } Size gridSize() const { return gridSize_; } + void setRansacParams(const RansacParams &val) { ransacParams_ = val; } + RansacParams ransacParams() const { return ransacParams_; } + + void setOutlierRejector(Ptr val) { outlierRejector_ = val; } + Ptr outlierRejector() const { return outlierRejector_; } + + void setMinInlierRatio(float val) { minInlierRatio_ = val; } + float minInlierRatio() const { return minInlierRatio_; } + virtual Mat estimate(const Mat &frame0, const Mat &frame1, bool *ok = 0); private: Ptr detector_; Ptr optFlowEstimator_; Size gridSize_; + RansacParams ransacParams_; + Ptr outlierRejector_; + float minInlierRatio_; std::vector status_; std::vector keypointsPrev_; @@ -156,10 +152,19 @@ private: }; #if HAVE_OPENCV_GPU -class CV_EXPORTS PyrLkRobustMotionEstimatorGpu : public PyrLkRobustMotionEstimatorBase +class CV_EXPORTS RansacMotionEstimatorGpu : public GlobalMotionEstimatorBase { public: - PyrLkRobustMotionEstimatorGpu(MotionModel model = MM_AFFINE); + RansacMotionEstimatorGpu(MotionModel model = MM_AFFINE); + + void setRansacParams(const RansacParams &val) { ransacParams_ = val; } + RansacParams ransacParams() const { return ransacParams_; } + + void setOutlierRejector(Ptr val) { outlierRejector_ = val; } + Ptr outlierRejector() const { return outlierRejector_; } + + void setMinInlierRatio(float val) { minInlierRatio_ = val; } + float minInlierRatio() const { return minInlierRatio_; } virtual Mat estimate(const Mat &frame0, const Mat &frame1, bool *ok = 0); Mat estimate(const gpu::GpuMat &frame0, const gpu::GpuMat &frame1, bool *ok = 0); @@ -167,6 +172,9 @@ public: private: gpu::GoodFeaturesToTrackDetector_GPU detector_; SparsePyrLkOptFlowEstimatorGpu optFlowEstimator_; + RansacParams ransacParams_; + Ptr outlierRejector_; + float minInlierRatio_; gpu::GpuMat frame0_, grayFrame0_, frame1_; gpu::GpuMat pointsPrev_, points_; @@ -178,6 +186,44 @@ private: }; #endif +class CV_EXPORTS LpBasedMotionEstimator : public GlobalMotionEstimatorBase +{ +public: + LpBasedMotionEstimator(MotionModel model = MM_AFFINE); + + void setDetector(Ptr val) { detector_ = val; } + Ptr detector() const { return detector_; } + + void setOptFlowEstimator(Ptr val) { optFlowEstimator_ = val; } + Ptr optFlowEstimator() const { return optFlowEstimator_; } + + void setOutlierRejector(Ptr val) { outlierRejector_ = val; } + Ptr outlierRejector() const { return outlierRejector_; } + + virtual Mat estimate(const Mat &frame0, const Mat &frame1, bool *ok); + +private: + Ptr detector_; + Ptr optFlowEstimator_; + Ptr outlierRejector_; + + std::vector status_; + std::vector keypointsPrev_; + std::vector pointsPrev_, points_; + std::vector pointsPrevGood_, pointsGood_; + + std::vector obj_, collb_, colub_; + std::vector rows_, cols_; + std::vector elems_, rowlb_, rowub_; + + void set(int row, int col, double coef) + { + rows_.push_back(row); + cols_.push_back(col); + elems_.push_back(coef); + } +}; + CV_EXPORTS Mat getMotion(int from, int to, const std::vector &motions); } // namespace videostab diff --git a/modules/videostab/src/clp.hpp b/modules/videostab/src/clp.hpp new file mode 100644 index 0000000..d819e5f --- /dev/null +++ b/modules/videostab/src/clp.hpp @@ -0,0 +1,63 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. +// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#ifndef __OPENCV_VIDEOSTAB_CLP_HPP__ +#define __OPENCV_VIDEOSTAB_CLP_HPP__ + +#ifdef HAVE_CLP + #include "ClpSimplex.hpp" + #include "ClpPresolve.hpp" + #include "ClpPrimalColumnSteepest.hpp" + #include "ClpDualRowSteepest.hpp" + #define INF 1e10 +#endif + +// Clp replaces min and max with ?: globally, we can't use std::min and std::max in case +// when HAVE_CLP is true. We create the defines by ourselves when HAVE_CLP == 0. +#ifndef min + #define min(a,b) std::min(a,b) +#endif +#ifndef max + #define max(a,b) std::max(a,b) +#endif + +#endif diff --git a/modules/videostab/src/global_motion.cpp b/modules/videostab/src/global_motion.cpp index b67798c..a28acb4 100644 --- a/modules/videostab/src/global_motion.cpp +++ b/modules/videostab/src/global_motion.cpp @@ -45,6 +45,7 @@ #include "opencv2/videostab/ring_buffer.hpp" #include "opencv2/videostab/outlier_rejection.hpp" #include "opencv2/opencv_modules.hpp" +#include "clp.hpp" using namespace std; @@ -434,25 +435,19 @@ Mat ToFileMotionWriter::estimate(const Mat &frame0, const Mat &frame1, bool *ok) } -PyrLkRobustMotionEstimatorBase::PyrLkRobustMotionEstimatorBase(MotionModel model) +RansacMotionEstimator::RansacMotionEstimator(MotionModel model) : GlobalMotionEstimatorBase(model) { - setRansacParams(RansacParams::default2dMotion(model)); - setOutlierRejector(new NullOutlierRejector()); - setMinInlierRatio(0.1f); -} - - -PyrLkRobustMotionEstimator::PyrLkRobustMotionEstimator(MotionModel model) - : PyrLkRobustMotionEstimatorBase(model) -{ setDetector(new GoodFeaturesToTrackDetector()); setOptFlowEstimator(new SparsePyrLkOptFlowEstimator()); setGridSize(Size(0,0)); + setRansacParams(RansacParams::default2dMotion(model)); + setOutlierRejector(new NullOutlierRejector()); + setMinInlierRatio(0.1f); } -Mat PyrLkRobustMotionEstimator::estimate(const Mat &frame0, const Mat &frame1, bool *ok) +Mat RansacMotionEstimator::estimate(const Mat &frame0, const Mat &frame1, bool *ok) { // find keypoints @@ -549,14 +544,17 @@ Mat PyrLkRobustMotionEstimator::estimate(const Mat &frame0, const Mat &frame1, b #if HAVE_OPENCV_GPU -PyrLkRobustMotionEstimatorGpu::PyrLkRobustMotionEstimatorGpu(MotionModel model) - : PyrLkRobustMotionEstimatorBase(model) +RansacMotionEstimatorGpu::RansacMotionEstimatorGpu(MotionModel model) + : GlobalMotionEstimatorBase(model) { CV_Assert(gpu::getCudaEnabledDeviceCount() > 0); + setRansacParams(RansacParams::default2dMotion(model)); + setOutlierRejector(new NullOutlierRejector()); + setMinInlierRatio(0.1f); } -Mat PyrLkRobustMotionEstimatorGpu::estimate(const Mat &frame0, const Mat &frame1, bool *ok) +Mat RansacMotionEstimatorGpu::estimate(const Mat &frame0, const Mat &frame1, bool *ok) { frame0_.upload(frame0); frame1_.upload(frame1); @@ -564,7 +562,7 @@ Mat PyrLkRobustMotionEstimatorGpu::estimate(const Mat &frame0, const Mat &frame1 } -Mat PyrLkRobustMotionEstimatorGpu::estimate(const gpu::GpuMat &frame0, const gpu::GpuMat &frame1, bool *ok) +Mat RansacMotionEstimatorGpu::estimate(const gpu::GpuMat &frame0, const gpu::GpuMat &frame1, bool *ok) { // convert frame to gray if it's color @@ -652,6 +650,188 @@ Mat PyrLkRobustMotionEstimatorGpu::estimate(const gpu::GpuMat &frame0, const gpu #endif // #if HAVE_OPENCV_GPU +LpBasedMotionEstimator::LpBasedMotionEstimator(MotionModel model) + : GlobalMotionEstimatorBase(model) +{ + setDetector(new GoodFeaturesToTrackDetector()); + setOptFlowEstimator(new SparsePyrLkOptFlowEstimator()); + setOutlierRejector(new NullOutlierRejector()); +} + +// TODO will estimation of all motions as one LP problem be faster? + +Mat LpBasedMotionEstimator::estimate(const Mat &frame0, const Mat &frame1, bool *ok) +{ + // find keypoints + + detector_->detect(frame0, keypointsPrev_); + + // extract points from keypoints + + pointsPrev_.resize(keypointsPrev_.size()); + for (size_t i = 0; i < keypointsPrev_.size(); ++i) + pointsPrev_[i] = keypointsPrev_[i].pt; + + // find correspondences + + optFlowEstimator_->run(frame0, frame1, pointsPrev_, points_, status_, noArray()); + + // leave good correspondences only + + pointsPrevGood_.clear(); pointsPrevGood_.reserve(points_.size()); + pointsGood_.clear(); pointsGood_.reserve(points_.size()); + + for (size_t i = 0; i < points_.size(); ++i) + { + if (status_[i]) + { + pointsPrevGood_.push_back(pointsPrev_[i]); + pointsGood_.push_back(points_[i]); + } + } + + // perfrom outlier rejection + + IOutlierRejector *outlierRejector = static_cast(outlierRejector_); + if (!dynamic_cast(outlierRejector)) + { + pointsPrev_.swap(pointsPrevGood_); + points_.swap(pointsGood_); + + outlierRejector_->process(frame0.size(), pointsPrev_, points_, status_); + + pointsPrevGood_.clear(); pointsPrevGood_.reserve(points_.size()); + pointsGood_.clear(); pointsGood_.reserve(points_.size()); + + for (size_t i = 0; i < points_.size(); ++i) + { + if (status_[i]) + { + pointsPrevGood_.push_back(pointsPrev_[i]); + pointsGood_.push_back(points_[i]); + } + } + } + + int npoints = static_cast(pointsGood_.size()); + + // prepare LP problem + +#ifndef HAVE_CLP + + CV_Error(CV_StsError, "The library is built without Clp support"); + if (ok) *ok = false; + return Mat::eye(3, 3, CV_32F); + +#else + + CV_Assert(motionModel_ <= MM_AFFINE && motionModel_ != MM_RIGID); + + int ncols = 6 + 2*npoints; + int nrows = 4*npoints; + + if (motionModel_ == MM_SIMILARITY) + nrows += 2; + else if (motionModel_ == MM_TRANSLATION_AND_SCALE) + nrows += 3; + else if (motionModel_ == MM_TRANSLATION) + nrows += 4; + + rows_.clear(); + cols_.clear(); + elems_.clear(); + + obj_.assign(ncols, 0); + collb_.assign(ncols, -INF); + colub_.assign(ncols, INF); + + int c = 6; + + for (int i = 0; i < npoints; ++i, c += 2) + { + obj_[c] = 1; + collb_[c] = 0; + + obj_[c+1] = 1; + collb_[c+1] = 0; + } + + elems_.clear(); + rowlb_.assign(nrows, -INF); + rowub_.assign(nrows, INF); + + int r = 0; + Point2f p0, p1; + + for (int i = 0; i < npoints; ++i, r += 4) + { + p0 = pointsPrevGood_[i]; + p1 = pointsGood_[i]; + + set(r, 0, p0.x); set(r, 1, p0.y); set(r, 2, 1); set(r, 6+2*i, -1); + rowub_[r] = p1.x; + + set(r+1, 3, p0.x); set(r+1, 4, p0.y); set(r+1, 5, 1); set(r+1, 6+2*i+1, -1); + rowub_[r+1] = p1.y; + + set(r+2, 0, p0.x); set(r+2, 1, p0.y); set(r+2, 2, 1); set(r+2, 6+2*i, 1); + rowlb_[r+2] = p1.x; + + set(r+3, 3, p0.x); set(r+3, 4, p0.y); set(r+3, 5, 1); set(r+3, 6+2*i+1, 1); + rowlb_[r+3] = p1.y; + } + + if (motionModel_ == MM_SIMILARITY) + { + set(r, 0, 1); set(r, 4, -1); rowlb_[r] = rowub_[r] = 0; + set(r+1, 1, 1); set(r+1, 3, 1); rowlb_[r+1] = rowub_[r+1] = 0; + } + else if (motionModel_ == MM_TRANSLATION_AND_SCALE) + { + set(r, 0, 1); set(r, 4, -1); rowlb_[r] = rowub_[r] = 0; + set(r+1, 1, 1); rowlb_[r+1] = rowub_[r+1] = 0; + set(r+2, 3, 1); rowlb_[r+2] = rowub_[r+2] = 0; + } + else if (motionModel_ == MM_TRANSLATION) + { + set(r, 0, 1); rowlb_[r] = rowub_[r] = 1; + set(r+1, 1, 1); rowlb_[r+1] = rowub_[r+1] = 0; + set(r+2, 3, 1); rowlb_[r+2] = rowub_[r+2] = 0; + set(r+3, 4, 1); rowlb_[r+3] = rowub_[r+3] = 1; + } + + // solve + + CoinPackedMatrix A(true, &rows_[0], &cols_[0], &elems_[0], elems_.size()); + A.setDimensions(nrows, ncols); + + ClpSimplex model(false); + model.loadProblem(A, &collb_[0], &colub_[0], &obj_[0], &rowlb_[0], &rowub_[0]); + + ClpDualRowSteepest dualSteep(1); + model.setDualRowPivotAlgorithm(dualSteep); + model.scaling(1); + + model.dual(); + + // extract motion + + const double *sol = model.getColSolution(); + + Mat_ M = Mat::eye(3, 3, CV_32F); + M(0,0) = sol[0]; + M(0,1) = sol[1]; + M(0,2) = sol[2]; + M(1,0) = sol[3]; + M(1,1) = sol[4]; + M(1,2) = sol[5]; + + if (ok) *ok = true; + return M; +#endif +} + + Mat getMotion(int from, int to, const vector &motions) { Mat M = Mat::eye(3, 3, CV_32F); diff --git a/modules/videostab/src/motion_stabilizing.cpp b/modules/videostab/src/motion_stabilizing.cpp index 31f0422..417ccf7 100644 --- a/modules/videostab/src/motion_stabilizing.cpp +++ b/modules/videostab/src/motion_stabilizing.cpp @@ -44,23 +44,7 @@ #include "opencv2/videostab/motion_stabilizing.hpp" #include "opencv2/videostab/global_motion.hpp" #include "opencv2/videostab/ring_buffer.hpp" - -#ifdef HAVE_CLP - #include "ClpSimplex.hpp" - #include "ClpPresolve.hpp" - #include "ClpPrimalColumnSteepest.hpp" - #include "ClpDualRowSteepest.hpp" - #define INF 1e10 -#endif - -// Clp replaces min and max with ?: globally, we can't use std::min and std::max in case -// when HAVE_CLP is true, otherwise we create the defines by ourselves -#ifndef min - #define min(a,b) std::min(a,b) -#endif -#ifndef max - #define max(a,b) std::max(a,b) -#endif +#include "clp.hpp" using namespace std; @@ -172,6 +156,10 @@ void LpMotionStabilizer::stabilize( int ncols = 4*N + 6*(N-1) + 6*(N-2) + 6*(N-3); int nrows = 8*N + 2*6*(N-1) + 2*6*(N-2) + 2*6*(N-3); + rows_.clear(); + cols_.clear(); + elems_.clear(); + obj_.assign(ncols, 0); collb_.assign(ncols, -INF); colub_.assign(ncols, INF); @@ -574,10 +562,8 @@ void LpMotionStabilizer::stabilize( S0(0,2) = sol[c+2]; S0(1,2) = sol[c+3]; S[t] = S0; - } + } } - - #endif // #ifndef HAVE_CLP diff --git a/modules/videostab/src/stabilizer.cpp b/modules/videostab/src/stabilizer.cpp index d619d85..44cb112 100644 --- a/modules/videostab/src/stabilizer.cpp +++ b/modules/videostab/src/stabilizer.cpp @@ -58,7 +58,7 @@ StabilizerBase::StabilizerBase() { setLog(new LogToStdout()); setFrameSource(new NullFrameSource()); - setMotionEstimator(new PyrLkRobustMotionEstimator()); + setMotionEstimator(new RansacMotionEstimator()); setDeblurer(new NullDeblurer()); setInpainter(new NullInpainter()); setRadius(15); diff --git a/modules/videostab/src/wobble_suppression.cpp b/modules/videostab/src/wobble_suppression.cpp index e3cd7fa..a398b6a 100644 --- a/modules/videostab/src/wobble_suppression.cpp +++ b/modules/videostab/src/wobble_suppression.cpp @@ -53,7 +53,7 @@ namespace videostab WobbleSuppressorBase::WobbleSuppressorBase() : motions_(0), stabilizationMotions_(0) { - PyrLkRobustMotionEstimator *est = new PyrLkRobustMotionEstimator(); + RansacMotionEstimator *est = new RansacMotionEstimator(); est->setMotionModel(MM_HOMOGRAPHY); est->setRansacParams(RansacParams::default2dMotion(MM_HOMOGRAPHY)); } diff --git a/samples/cpp/videostab.cpp b/samples/cpp/videostab.cpp index 4c1ab2a..12ebcfc 100644 --- a/samples/cpp/videostab.cpp +++ b/samples/cpp/videostab.cpp @@ -73,6 +73,8 @@ void printHelp() "Arguments:\n" " -m, --model=(transl|transl_and_scale|rigid|similarity|affine|homography)\n" " Set motion model. The default is affine.\n" + " -lp, --lin-prog-motion-est=(yes|no)\n" + " Turn on/off LP based motion estimation. The default is no.\n" " --subset=(|auto)\n" " Number of random samples per one motion hypothesis. The default is auto.\n" " --thresh=(|auto)\n" @@ -135,6 +137,8 @@ void printHelp() " The default is 2.0\n\n" " -ws, --wobble-suppress=(yes|no)\n" " Perform wobble suppression. The default is no.\n" + " --ws-lp=(yes|no)\n" + " Turn on/off LP based motion estimation. The default is no.\n" " --ws-period=\n" " Set wobble suppression period. The default is 30.\n" " --ws-model=(transl|transl_and_scale|rigid|similarity|affine|homography)\n" @@ -171,6 +175,133 @@ void printHelp() "Note: some argument configurations lead to two passes, some to single pass.\n\n"; } +// motion estimator builders are for concise creation of motion estimators + +class IMotionEstimatorBuilder +{ +public: + virtual ~IMotionEstimatorBuilder() {} + virtual Ptr build() = 0; +protected: + IMotionEstimatorBuilder(CommandLineParser &cmd) : cmd(cmd) {} + CommandLineParser cmd; +}; + + +class RansacMotionEstimatorBuilder : public IMotionEstimatorBuilder +{ +public: + RansacMotionEstimatorBuilder(CommandLineParser &cmd, const string &prefix = "") + : IMotionEstimatorBuilder(cmd), prefix(prefix) {} + + virtual Ptr build() + { + RansacMotionEstimator *est = new RansacMotionEstimator(motionModel(arg(prefix + "model"))); + + est->setDetector(new GoodFeaturesToTrackDetector(argi(prefix + "nkps"))); + + RansacParams ransac = est->ransacParams(); + if (arg(prefix + "subset") != "auto") + ransac.size = argi(prefix + "subset"); + if (arg(prefix + "thresh") != "auto") + ransac.thresh = argi(prefix + "thresh"); + ransac.eps = argf(prefix + "outlier-ratio"); + est->setRansacParams(ransac); + + est->setGridSize(Size(argi(prefix + "extra-kps"), argi(prefix + "extra-kps"))); + + Ptr outlierRejector = new NullOutlierRejector(); + if (arg(prefix + "local-outlier-rejection") == "yes") + { + TranslationBasedLocalOutlierRejector *tor = new TranslationBasedLocalOutlierRejector(); + RansacParams ransacParams = tor->ransacParams(); + if (arg(prefix + "thresh") != "auto") + ransacParams.thresh = argf(prefix + "thresh"); + tor->setRansacParams(ransacParams); + outlierRejector = tor; + } + est->setOutlierRejector(outlierRejector); + + est->setMinInlierRatio(argf(prefix + "min-inlier-ratio")); + + return est; + } +private: + string prefix; +}; + + +#if HAVE_OPENCV_GPU +class RansacMotionEstimatorBuilderGpu : public IMotionEstimatorBuilder +{ +public: + RansacMotionEstimatorBuilderGpu(CommandLineParser &cmd, const string &prefix = "") + : IMotionEstimatorBuilder(cmd), prefix(prefix) {} + + virtual Ptr build() + { + RansacMotionEstimatorGpu *est = new RansacMotionEstimatorGpu(motionModel(arg(prefix + "model"))); + + RansacParams ransac = est->ransacParams(); + if (arg(prefix + "subset") != "auto") + ransac.size = argi(prefix + "subset"); + if (arg(prefix + "thresh") != "auto") + ransac.thresh = argi(prefix + "thresh"); + ransac.eps = argf(prefix + "outlier-ratio"); + est->setRansacParams(ransac); + + Ptr outlierRejector = new NullOutlierRejector(); + if (arg(prefix + "local-outlier-rejection") == "yes") + { + TranslationBasedLocalOutlierRejector *tor = new TranslationBasedLocalOutlierRejector(); + RansacParams ransacParams = tor->ransacParams(); + if (arg(prefix + "thresh") != "auto") + ransacParams.thresh = argf(prefix + "thresh"); + tor->setRansacParams(ransacParams); + outlierRejector = tor; + } + est->setOutlierRejector(outlierRejector); + + est->setMinInlierRatio(argf(prefix + "min-inlier-ratio")); + + return est; + } +private: + string prefix; +}; +#endif + + +class LpBasedMotionEstimatorBuilder : public IMotionEstimatorBuilder +{ +public: + LpBasedMotionEstimatorBuilder(CommandLineParser &cmd, const string &prefix = "") + : IMotionEstimatorBuilder(cmd), prefix(prefix) {} + + virtual Ptr build() + { + LpBasedMotionEstimator *est = new LpBasedMotionEstimator(motionModel(arg(prefix + "model"))); + + est->setDetector(new GoodFeaturesToTrackDetector(argi(prefix + "nkps"))); + + Ptr outlierRejector = new NullOutlierRejector(); + if (arg(prefix + "local-outlier-rejection") == "yes") + { + TranslationBasedLocalOutlierRejector *tor = new TranslationBasedLocalOutlierRejector(); + RansacParams ransacParams = tor->ransacParams(); + if (arg(prefix + "thresh") != "auto") + ransacParams.thresh = argf(prefix + "thresh"); + tor->setRansacParams(ransacParams); + outlierRejector = tor; + } + est->setOutlierRejector(outlierRejector); + + return est; + } +private: + string prefix; +}; + int main(int argc, const char **argv) { @@ -178,7 +309,8 @@ int main(int argc, const char **argv) { const char *keys = "{ 1 | | | | }" - "{ m | model | affine| }" + "{ m | model | affine | }" + "{ lp | lin-prog-motion-est | no | }" "{ | subset | auto | }" "{ | thresh | auto | }" "{ | outlier-ratio | 0.5 | }" @@ -218,6 +350,7 @@ int main(int argc, const char **argv) "{ | ws-nkps | 1000 | }" "{ | ws-extra-kps | 0 | }" "{ | ws-local-outlier-rejection | no | }" + "{ | ws-lp | no | }" "{ sm2 | save-motions2 | no | }" "{ lm2 | load-motions2 | no | }" "{ gpu | | no }" @@ -246,23 +379,75 @@ int main(int argc, const char **argv) } #endif + StabilizerBase *stabilizer = 0; + + // check if source video is specified + string inputPath = arg("1"); - if (inputPath.empty()) throw runtime_error("specify video file path"); + if (inputPath.empty()) + throw runtime_error("specify video file path"); + + // get source video parameters VideoFileSource *source = new VideoFileSource(inputPath); cout << "frame count (rough): " << source->count() << endl; - if (arg("fps") == "auto") outputFps = source->fps(); else outputFps = argd("fps"); + if (arg("fps") == "auto") + outputFps = source->fps(); + else + outputFps = argd("fps"); - StabilizerBase *stabilizer; + // prepare motion estimation builders + Ptr motionEstBuilder; +#if HAVE_OPENCV_GPU + if (arg("gpu") == "yes") + { + if (arg("lin-prog-motion-est") == "yes") + motionEstBuilder = new LpBasedMotionEstimatorBuilder(cmd); + else + motionEstBuilder = new RansacMotionEstimatorBuilderGpu(cmd); + } + else +#endif + { + if (arg("lin-prog-motion-est") == "yes") + motionEstBuilder = new LpBasedMotionEstimatorBuilder(cmd); + else + motionEstBuilder = new RansacMotionEstimatorBuilder(cmd); + } + + Ptr wsMotionEstBuilder; +#if HAVE_OPENCV_GPU + if (arg("gpu") == "yes") + { + if (arg("ws-lp") == "yes") + wsMotionEstBuilder = new LpBasedMotionEstimatorBuilder(cmd, "ws-"); + else + wsMotionEstBuilder = new RansacMotionEstimatorBuilderGpu(cmd, "ws-"); + } + else +#endif + { + if (arg("ws-lp") == "yes") + wsMotionEstBuilder = new LpBasedMotionEstimatorBuilder(cmd, "ws-"); + else + wsMotionEstBuilder = new RansacMotionEstimatorBuilder(cmd, "ws-"); + } + + // determine whether we must use one pass or two pass stabilizer bool isTwoPass = arg("est-trim") == "yes" || arg("wobble-suppress") == "yes" || arg("lin-prog-stab") == "yes"; if (isTwoPass) { + // we must use two pass stabilizer + TwoPassStabilizer *twoPassStabilizer = new TwoPassStabilizer(); stabilizer = twoPassStabilizer; twoPassStabilizer->setEstimateTrimRatio(arg("est-trim") == "yes"); + + // determine stabilization technique + if (arg("lin-prog-stab") == "yes") { LpMotionStabilizer *stab = new LpMotionStabilizer(); @@ -278,65 +463,22 @@ int main(int argc, const char **argv) twoPassStabilizer->setMotionStabilizer(new GaussianMotionFilter(argi("radius"))); else twoPassStabilizer->setMotionStabilizer(new GaussianMotionFilter(argi("radius"), argf("stdev"))); - if (arg("wobble-suppress") == "yes") - { - MoreAccurateMotionWobbleSuppressorBase *ws = 0; - - Ptr outlierRejector = new NullOutlierRejector(); - if (arg("local-outlier-rejection") == "yes") - { - TranslationBasedLocalOutlierRejector *tor = new TranslationBasedLocalOutlierRejector(); - RansacParams ransacParams = tor->ransacParams(); - if (arg("ws-thresh") != "auto") ransacParams.thresh = argf("ws-thresh"); - tor->setRansacParams(ransacParams); - outlierRejector = tor; - } - - if (arg("gpu") == "no") - { - PyrLkRobustMotionEstimator *est = new PyrLkRobustMotionEstimator(motionModel(arg("ws-model"))); - est->setDetector(new GoodFeaturesToTrackDetector(argi("ws-nkps"))); - - RansacParams ransac = est->ransacParams(); - if (arg("ws-subset") != "auto") ransac.size = argi("ws-subset"); - if (arg("ws-thresh") != "auto") ransac.thresh = argi("ws-thresh"); - ransac.eps = argf("ws-outlier-ratio"); - est->setRansacParams(ransac); - est->setMinInlierRatio(argf("ws-min-inlier-ratio")); - est->setGridSize(Size(argi("ws-extra-kps"), argi("ws-extra-kps"))); - est->setOutlierRejector(outlierRejector); + // init wobble suppressor if necessary - ws = new MoreAccurateMotionWobbleSuppressor(); - ws->setMotionEstimator(est); - } - else if (arg("gpu") == "yes") - { + if (arg("wobble-suppress") == "yes") + { + MoreAccurateMotionWobbleSuppressorBase *ws = new MoreAccurateMotionWobbleSuppressor(); + if (arg("gpu") == "yes") #if HAVE_OPENCV_GPU - PyrLkRobustMotionEstimatorGpu *est = new PyrLkRobustMotionEstimatorGpu(motionModel(arg("ws-model"))); - - RansacParams ransac = est->ransacParams(); - if (arg("ws-subset") != "auto") ransac.size = argi("ws-subset"); - if (arg("ws-thresh") != "auto") ransac.thresh = argi("ws-thresh"); - ransac.eps = argf("ws-outlier-ratio"); - est->setRansacParams(ransac); - - est->setMinInlierRatio(argf("ws-min-inlier-ratio")); - est->setOutlierRejector(outlierRejector); - ws = new MoreAccurateMotionWobbleSuppressorGpu(); - ws->setMotionEstimator(est); #else throw runtime_error("OpenCV is built without GPU support"); #endif - } - else - { - throw runtime_error("bad gpu optimization argument value: " + arg("gpu")); - } - twoPassStabilizer->setWobbleSuppressor(ws); + ws->setMotionEstimator(wsMotionEstBuilder->build()); ws->setPeriod(argi("ws-period")); + twoPassStabilizer->setWobbleSuppressor(ws); MotionModel model = ws->motionEstimator()->motionModel(); if (arg("load-motions2") != "no") @@ -353,6 +495,8 @@ int main(int argc, const char **argv) } else { + // we must use one pass stabilizer + OnePassStabilizer *onePassStabilizer = new OnePassStabilizer(); stabilizer = onePassStabilizer; if (arg("stdev") == "auto") @@ -362,58 +506,10 @@ int main(int argc, const char **argv) } stabilizer->setFrameSource(source); - stabilizedFrames = dynamic_cast(stabilizer); - - Ptr outlierRejector = new NullOutlierRejector(); - if (arg("local-outlier-rejection") == "yes") - { - TranslationBasedLocalOutlierRejector *tor = new TranslationBasedLocalOutlierRejector(); - RansacParams ransacParams = tor->ransacParams(); - if (arg("thresh") != "auto") ransacParams.thresh = argf("thresh"); - tor->setRansacParams(ransacParams); - outlierRejector = tor; - } - - if (arg("gpu") == "no") - { - PyrLkRobustMotionEstimator *est = new PyrLkRobustMotionEstimator(motionModel(arg("model")));; - est->setDetector(new GoodFeaturesToTrackDetector(argi("nkps"))); + stabilizer->setMotionEstimator(motionEstBuilder->build()); - RansacParams ransac = est->ransacParams(); - if (arg("subset") != "auto") ransac.size = argi("subset"); - if (arg("thresh") != "auto") ransac.thresh = argi("thresh"); - ransac.eps = argf("outlier-ratio"); - est->setRansacParams(ransac); - - est->setMinInlierRatio(argf("min-inlier-ratio")); - est->setGridSize(Size(argi("extra-kps"), argi("extra-kps"))); - est->setOutlierRejector(outlierRejector); - - stabilizer->setMotionEstimator(est); - } - else if (arg("gpu") == "yes") - { -#if HAVE_OPENCV_GPU - PyrLkRobustMotionEstimatorGpu *est = new PyrLkRobustMotionEstimatorGpu(motionModel(arg("model")));; - - RansacParams ransac = est->ransacParams(); - if (arg("subset") != "auto") ransac.size = argi("subset"); - if (arg("thresh") != "auto") ransac.thresh = argi("thresh"); - ransac.eps = argf("outlier-ratio"); - est->setRansacParams(ransac); - - est->setMinInlierRatio(argf("min-inlier-ratio")); - est->setOutlierRejector(outlierRejector); - - stabilizer->setMotionEstimator(est); -#else - throw runtime_error("OpenCV is built without GPU support"); -#endif - } - else - { - throw runtime_error("bad gpu optimization argument value: " + arg("gpu")); - } + // cast stabilizer to simple frame source interface to read stabilized frames + stabilizedFrames = dynamic_cast(stabilizer); MotionModel model = stabilizer->motionEstimator()->motionModel(); if (arg("load-motions") != "no") @@ -428,6 +524,8 @@ int main(int argc, const char **argv) } stabilizer->setRadius(argi("radius")); + + // init deblurer if (arg("deblur") == "yes") { WeightingDeblurer *deblurer = new WeightingDeblurer(); @@ -436,6 +534,7 @@ int main(int argc, const char **argv) stabilizer->setDeblurer(deblurer); } + // set up trimming paramters stabilizer->setTrimRatio(argf("trim-ratio")); stabilizer->setCorrectionForInclusion(arg("incl-constr") == "yes"); @@ -449,6 +548,7 @@ int main(int argc, const char **argv) throw runtime_error("unknown border extrapolation mode: " + cmd.get("border-mode")); + // init inpainter InpaintingPipeline *inpainters = new InpaintingPipeline(); Ptr inpainters_(inpainters); if (arg("mosaic") == "yes") -- 2.7.4