From 95efec75392ed9e9256f58f053f7541981e75471 Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Tue, 24 Apr 2012 12:23:23 +0000 Subject: [PATCH] Added local outlier rejector. Added rigid motion estimator. Refactored videostab module. --- .../include/opencv2/videostab/global_motion.hpp | 89 ++++----- .../include/opencv2/videostab/motion_core.hpp | 103 +++++++++++ .../include/opencv2/videostab/optical_flow.hpp | 8 +- .../opencv2/videostab/outlier_rejection.hpp | 96 ++++++++++ .../include/opencv2/videostab/videostab.hpp | 2 +- .../opencv2/videostab/wobble_suppression.hpp | 8 +- modules/videostab/src/global_motion.cpp | 136 ++++++++++++-- modules/videostab/src/outlier_rejection.cpp | 201 +++++++++++++++++++++ samples/cpp/videostab.cpp | 142 +++++++-------- 9 files changed, 631 insertions(+), 154 deletions(-) create mode 100644 modules/videostab/include/opencv2/videostab/motion_core.hpp create mode 100644 modules/videostab/include/opencv2/videostab/outlier_rejection.hpp create mode 100644 modules/videostab/src/outlier_rejection.cpp diff --git a/modules/videostab/include/opencv2/videostab/global_motion.hpp b/modules/videostab/include/opencv2/videostab/global_motion.hpp index bd41386..e785645 100644 --- a/modules/videostab/include/opencv2/videostab/global_motion.hpp +++ b/modules/videostab/include/opencv2/videostab/global_motion.hpp @@ -48,8 +48,10 @@ #include #include "opencv2/core/core.hpp" #include "opencv2/features2d/features2d.hpp" -#include "opencv2/videostab/optical_flow.hpp" #include "opencv2/opencv_modules.hpp" +#include "opencv2/videostab/optical_flow.hpp" +#include "opencv2/videostab/motion_core.hpp" +#include "opencv2/videostab/outlier_rejection.hpp" #if HAVE_OPENCV_GPU #include "opencv2/gpu/gpu.hpp" @@ -60,44 +62,9 @@ namespace cv namespace videostab { -enum MotionModel -{ - MM_TRANSLATION = 0, - MM_TRANSLATION_AND_SCALE = 1, - MM_SIMILARITY = 2, - MM_AFFINE = 3, - MM_HOMOGRAPHY = 4, - MM_UNKNOWN = 5 -}; - CV_EXPORTS Mat estimateGlobalMotionLeastSquares( int npoints, Point2f *points0, Point2f *points1, int model = MM_AFFINE, float *rmse = 0); -struct CV_EXPORTS RansacParams -{ - int size; // subset size - float thresh; // max error to classify as inlier - float eps; // max outliers ratio - float prob; // probability of success - - RansacParams() : size(0), thresh(0), eps(0), prob(0) {} - RansacParams(int size, float thresh, float eps, float prob) - : size(size), thresh(thresh), eps(eps), prob(prob) {} - - static RansacParams default2dMotion(MotionModel model) - { - CV_Assert(model < MM_UNKNOWN); - if (model == MM_TRANSLATION) - return RansacParams(1, 0.5f, 0.5f, 0.99f); - if (model == MM_TRANSLATION_AND_SCALE) - return RansacParams(2, 0.5f, 0.5f, 0.99f); - if (model == MM_SIMILARITY) - return RansacParams(2, 0.5f, 0.5f, 0.99f); - if (model == MM_AFFINE) - return RansacParams(3, 0.5f, 0.5f, 0.99f); - return RansacParams(4, 0.5f, 0.5f, 0.99f); - } -}; CV_EXPORTS Mat estimateGlobalMotionRobust( const std::vector &points0, const std::vector &points1, @@ -106,8 +73,7 @@ CV_EXPORTS Mat estimateGlobalMotionRobust( class CV_EXPORTS GlobalMotionEstimatorBase { -public: - GlobalMotionEstimatorBase() : motionModel_(MM_UNKNOWN) {} +public: virtual ~GlobalMotionEstimatorBase() {} virtual void setMotionModel(MotionModel val) { motionModel_ = val; } @@ -116,6 +82,8 @@ public: virtual Mat estimate(const Mat &frame0, const Mat &frame1, bool *ok = 0) = 0; protected: + GlobalMotionEstimatorBase(MotionModel model) { setMotionModel(model); } + MotionModel motionModel_; }; @@ -140,7 +108,27 @@ private: Ptr estimator_; }; -class CV_EXPORTS PyrLkRobustMotionEstimator : public GlobalMotionEstimatorBase +class CV_EXPORTS PyrLkRobustMotionEstimatorBase : public GlobalMotionEstimatorBase +{ +public: + virtual void setRansacParams(const RansacParams &val) { ransacParams_ = val; } + virtual RansacParams ransacParams() const { return ransacParams_; } + + virtual void setOutlierRejector(Ptr val) { outlierRejector_ = val; } + virtual Ptr outlierRejector() const { return outlierRejector_; } + + virtual void setMinInlierRatio(float val) { minInlierRatio_ = val; } + virtual float minInlierRatio() const { return minInlierRatio_; } + +protected: + PyrLkRobustMotionEstimatorBase(MotionModel model); + + RansacParams ransacParams_; + Ptr outlierRejector_; + float minInlierRatio_; +}; + +class CV_EXPORTS PyrLkRobustMotionEstimator : public PyrLkRobustMotionEstimatorBase { public: PyrLkRobustMotionEstimator(MotionModel model = MM_AFFINE); @@ -151,12 +139,6 @@ public: void setOptFlowEstimator(Ptr val) { optFlowEstimator_ = val; } Ptr optFlowEstimator() const { return optFlowEstimator_; } - void setRansacParams(const RansacParams &val) { ransacParams_ = val; } - RansacParams ransacParams() const { return ransacParams_; } - - void setMinInlierRatio(float val) { minInlierRatio_ = val; } - float minInlierRatio() const { return minInlierRatio_; } - void setGridSize(Size val) { gridSize_ = val; } Size gridSize() const { return gridSize_; } @@ -165,8 +147,6 @@ public: private: Ptr detector_; Ptr optFlowEstimator_; - RansacParams ransacParams_; - float minInlierRatio_; Size gridSize_; std::vector status_; @@ -176,30 +156,25 @@ private: }; #if HAVE_OPENCV_GPU -class CV_EXPORTS PyrLkRobustMotionEstimatorGpu : public GlobalMotionEstimatorBase +class CV_EXPORTS PyrLkRobustMotionEstimatorGpu : public PyrLkRobustMotionEstimatorBase { public: PyrLkRobustMotionEstimatorGpu(MotionModel model = MM_AFFINE); - void setRansacParams(const RansacParams &val) { ransacParams_ = val; } - RansacParams ransacParams() const { return ransacParams_; } - - void setMinInlierRatio(float val) { minInlierRatio_ = val; } - float minInlierRatio() const { return minInlierRatio_; } - virtual Mat estimate(const Mat &frame0, const Mat &frame1, bool *ok = 0); Mat estimate(const gpu::GpuMat &frame0, const gpu::GpuMat &frame1, bool *ok = 0); private: gpu::GoodFeaturesToTrackDetector_GPU detector_; SparsePyrLkOptFlowEstimatorGpu optFlowEstimator_; - RansacParams ransacParams_; - float minInlierRatio_; gpu::GpuMat frame0_, grayFrame0_, frame1_; gpu::GpuMat pointsPrev_, points_; - Mat hostPointsPrev_, hostPoints_; gpu::GpuMat status_; + + Mat hostPointsPrev_, hostPoints_; + std::vector hostPointsPrevGood_, hostPointsGood_; + std::vector rejectionStatus_; }; #endif diff --git a/modules/videostab/include/opencv2/videostab/motion_core.hpp b/modules/videostab/include/opencv2/videostab/motion_core.hpp new file mode 100644 index 0000000..4102412 --- /dev/null +++ b/modules/videostab/include/opencv2/videostab/motion_core.hpp @@ -0,0 +1,103 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. +// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#ifndef __OPENCV_VIDEOSTAB_MOTION_CORE_HPP__ +#define __OPENCV_VIDEOSTAB_MOTION_CORE_HPP__ + +#include +#include "opencv2/core/core.hpp" + +namespace cv +{ +namespace videostab +{ + +enum MotionModel +{ + MM_TRANSLATION = 0, + MM_TRANSLATION_AND_SCALE = 1, + MM_RIGID = 2, + MM_SIMILARITY = 3, + MM_AFFINE = 4, + MM_HOMOGRAPHY = 5, + MM_UNKNOWN = 6 +}; + +struct CV_EXPORTS RansacParams +{ + int size; // subset size + float thresh; // max error to classify as inlier + float eps; // max outliers ratio + float prob; // probability of success + + RansacParams() : size(0), thresh(0), eps(0), prob(0) {} + RansacParams(int size, float thresh, float eps, float prob) + : size(size), thresh(thresh), eps(eps), prob(prob) {} + + int niters() const + { + return static_cast( + std::ceil(std::log(1 - prob) / std::log(1 - std::pow(1 - eps, size)))); + } + + static RansacParams default2dMotion(MotionModel model) + { + CV_Assert(model < MM_UNKNOWN); + if (model == MM_TRANSLATION) + return RansacParams(1, 0.5f, 0.5f, 0.99f); + if (model == MM_TRANSLATION_AND_SCALE) + return RansacParams(2, 0.5f, 0.5f, 0.99f); + if (model == MM_RIGID) + return RansacParams(2, 0.5f, 0.5f, 0.99f); + if (model == MM_SIMILARITY) + return RansacParams(2, 0.5f, 0.5f, 0.99f); + if (model == MM_AFFINE) + return RansacParams(3, 0.5f, 0.5f, 0.99f); + return RansacParams(4, 0.5f, 0.5f, 0.99f); + } +}; + + +} // namespace videostab +} // namespace cv + +#endif diff --git a/modules/videostab/include/opencv2/videostab/optical_flow.hpp b/modules/videostab/include/opencv2/videostab/optical_flow.hpp index 23b3ef1..4cd143c 100644 --- a/modules/videostab/include/opencv2/videostab/optical_flow.hpp +++ b/modules/videostab/include/opencv2/videostab/optical_flow.hpp @@ -78,11 +78,11 @@ class CV_EXPORTS PyrLkOptFlowEstimatorBase public: PyrLkOptFlowEstimatorBase() { setWinSize(Size(21, 21)); setMaxLevel(3); } - void setWinSize(Size val) { winSize_ = val; } - Size winSize() const { return winSize_; } + virtual void setWinSize(Size val) { winSize_ = val; } + virtual Size winSize() const { return winSize_; } - void setMaxLevel(int val) { maxLevel_ = val; } - int maxLevel() const { return maxLevel_; } + virtual void setMaxLevel(int val) { maxLevel_ = val; } + virtual int maxLevel() const { return maxLevel_; } protected: Size winSize_; diff --git a/modules/videostab/include/opencv2/videostab/outlier_rejection.hpp b/modules/videostab/include/opencv2/videostab/outlier_rejection.hpp new file mode 100644 index 0000000..a21c05d --- /dev/null +++ b/modules/videostab/include/opencv2/videostab/outlier_rejection.hpp @@ -0,0 +1,96 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. +// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#ifndef __OPENCV_VIDEOSTAB_OUTLIER_REJECTION_HPP__ +#define __OPENCV_VIDEOSTAB_OUTLIER_REJECTION_HPP__ + +#include +#include "opencv2/core/core.hpp" +#include "opencv2/videostab/motion_core.hpp" + +namespace cv +{ +namespace videostab +{ + +class CV_EXPORTS IOutlierRejector +{ +public: + virtual ~IOutlierRejector() {} + + virtual void process( + Size frameSize, InputArray points0, InputArray points1, OutputArray mask) = 0; +}; + +class CV_EXPORTS NullOutlierRejector : public IOutlierRejector +{ +public: + virtual void process( + Size frameSize, InputArray points0, InputArray points1, OutputArray mask); +}; + +class CV_EXPORTS TranslationBasedLocalOutlierRejector : public IOutlierRejector +{ +public: + TranslationBasedLocalOutlierRejector(); + + void setCellSize(Size val) { cellSize_ = val; } + Size cellSize() const { return cellSize_; } + + void setRansacParams(RansacParams val) { ransacParams_ = val; } + RansacParams ransacParams() const { return ransacParams_; } + + virtual void process( + Size frameSize, InputArray points0, InputArray points1, OutputArray mask); + +private: + Size cellSize_; + RansacParams ransacParams_; + + typedef std::vector Cell; + std::vector grid_; +}; + +} // namespace videostab +} // namespace cv + +#endif diff --git a/modules/videostab/include/opencv2/videostab/videostab.hpp b/modules/videostab/include/opencv2/videostab/videostab.hpp index d4a27e5..3f86089 100644 --- a/modules/videostab/include/opencv2/videostab/videostab.hpp +++ b/modules/videostab/include/opencv2/videostab/videostab.hpp @@ -40,7 +40,7 @@ // //M*/ -// References: +// REFERENCES // 1. "Full-Frame Video Stabilization with Motion Inpainting" // Yasuyuki Matsushita, Eyal Ofek, Weina Ge, Xiaoou Tang, Senior Member, and Heung-Yeung Shum // 2. "Auto-Directed Video Stabilization with Robust L1 Optimal Camera Paths" diff --git a/modules/videostab/include/opencv2/videostab/wobble_suppression.hpp b/modules/videostab/include/opencv2/videostab/wobble_suppression.hpp index aa0f973..1a2076f 100644 --- a/modules/videostab/include/opencv2/videostab/wobble_suppression.hpp +++ b/modules/videostab/include/opencv2/videostab/wobble_suppression.hpp @@ -101,12 +101,12 @@ public: class CV_EXPORTS MoreAccurateMotionWobbleSuppressorBase : public WobbleSuppressorBase { public: - MoreAccurateMotionWobbleSuppressorBase() { setPeriod(30); } - - void setPeriod(int val) { period_ = val; } - int period() const { return period_; } + virtual void setPeriod(int val) { period_ = val; } + virtual int period() const { return period_; } protected: + MoreAccurateMotionWobbleSuppressorBase() { setPeriod(30); } + int period_; }; diff --git a/modules/videostab/src/global_motion.cpp b/modules/videostab/src/global_motion.cpp index f3f4fc2..b67798c 100644 --- a/modules/videostab/src/global_motion.cpp +++ b/modules/videostab/src/global_motion.cpp @@ -43,6 +43,7 @@ #include "precomp.hpp" #include "opencv2/videostab/global_motion.hpp" #include "opencv2/videostab/ring_buffer.hpp" +#include "opencv2/videostab/outlier_rejection.hpp" #include "opencv2/opencv_modules.hpp" using namespace std; @@ -150,6 +151,61 @@ static Mat estimateGlobMotionLeastSquaresTranslationAndScale( } +static Mat estimateGlobMotionLeastSquaresRigid( + int npoints, Point2f *points0, Point2f *points1, float *rmse) +{ + Point2f mean0(0.f, 0.f); + Point2f mean1(0.f, 0.f); + + for (int i = 0; i < npoints; ++i) + { + mean0 += points0[i]; + mean1 += points1[i]; + } + + mean0 *= 1.f / npoints; + mean1 *= 1.f / npoints; + + Mat_ A = Mat::zeros(2, 2, CV_32F); + Point2f pt0, pt1; + + for (int i = 0; i < npoints; ++i) + { + pt0 = points0[i] - mean0; + pt1 = points1[i] - mean1; + A(0,0) += pt1.x * pt0.x; + A(0,1) += pt1.x * pt0.y; + A(1,0) += pt1.y * pt0.x; + A(1,1) += pt1.y * pt0.y; + } + + Mat_ M = Mat::eye(3, 3, CV_32F); + + SVD svd(A); + Mat_ R = svd.u * svd.vt; + Mat tmp(M(Rect(0,0,2,2))); + R.copyTo(tmp); + + M(0,2) = mean1.x - R(0,0)*mean0.x - R(0,1)*mean0.y; + M(1,2) = mean1.y - R(1,0)*mean0.x - R(1,1)*mean0.y; + + if (rmse) + { + *rmse = 0; + for (int i = 0; i < npoints; ++i) + { + pt0 = points0[i]; + pt1 = points1[i]; + *rmse += sqr(pt1.x - M(0,0)*pt0.x - M(0,1)*pt0.y - M(0,2)) + + sqr(pt1.y - M(1,0)*pt0.x - M(1,1)*pt0.y - M(1,2)); + } + *rmse = sqrt(*rmse / npoints); + } + + return M; +} + + static Mat estimateGlobMotionLeastSquaresSimilarity( int npoints, Point2f *points0, Point2f *points1, float *rmse) { @@ -234,6 +290,7 @@ Mat estimateGlobalMotionLeastSquares( typedef Mat (*Impl)(int, Point2f*, Point2f*, float*); static Impl impls[] = { estimateGlobMotionLeastSquaresTranslation, estimateGlobMotionLeastSquaresTranslationAndScale, + estimateGlobMotionLeastSquaresRigid, estimateGlobMotionLeastSquaresSimilarity, estimateGlobMotionLeastSquaresAffine }; @@ -247,8 +304,7 @@ Mat estimateGlobalMotionRobust( { CV_Assert(model <= MM_AFFINE); - const int niters = static_cast(ceil(log(1 - params.prob) / - log(1 - pow(1 - params.eps, params.size)))); + const int niters = params.niters(); // current hypothesis vector indices(params.size); @@ -338,6 +394,7 @@ Mat estimateGlobalMotionRobust( FromFileMotionReader::FromFileMotionReader(const string &path) + : GlobalMotionEstimatorBase(MM_UNKNOWN) { file_.open(path.c_str()); CV_Assert(file_.is_open()); @@ -357,6 +414,7 @@ Mat FromFileMotionReader::estimate(const Mat &/*frame0*/, const Mat &/*frame1*/, ToFileMotionWriter::ToFileMotionWriter(const string &path, Ptr estimator) + : GlobalMotionEstimatorBase(estimator->motionModel()) { file_.open(path.c_str()); CV_Assert(file_.is_open()); @@ -376,13 +434,20 @@ Mat ToFileMotionWriter::estimate(const Mat &frame0, const Mat &frame1, bool *ok) } +PyrLkRobustMotionEstimatorBase::PyrLkRobustMotionEstimatorBase(MotionModel model) + : GlobalMotionEstimatorBase(model) +{ + setRansacParams(RansacParams::default2dMotion(model)); + setOutlierRejector(new NullOutlierRejector()); + setMinInlierRatio(0.1f); +} + + PyrLkRobustMotionEstimator::PyrLkRobustMotionEstimator(MotionModel model) + : PyrLkRobustMotionEstimatorBase(model) { setDetector(new GoodFeaturesToTrackDetector()); setOptFlowEstimator(new SparsePyrLkOptFlowEstimator()); - setMotionModel(model); - setRansacParams(RansacParams::default2dMotion(model)); - setMinInlierRatio(0.1f); setGridSize(Size(0,0)); } @@ -428,6 +493,29 @@ Mat PyrLkRobustMotionEstimator::estimate(const Mat &frame0, const Mat &frame1, b } } + // perfrom outlier rejection + + IOutlierRejector *outlierRejector = static_cast(outlierRejector_); + if (!dynamic_cast(outlierRejector)) + { + pointsPrev_.swap(pointsPrevGood_); + points_.swap(pointsGood_); + + outlierRejector_->process(frame0.size(), pointsPrev_, points_, status_); + + pointsPrevGood_.clear(); pointsPrevGood_.reserve(points_.size()); + pointsGood_.clear(); pointsGood_.reserve(points_.size()); + + for (size_t i = 0; i < points_.size(); ++i) + { + if (status_[i]) + { + pointsPrevGood_.push_back(pointsPrev_[i]); + pointsGood_.push_back(points_[i]); + } + } + } + size_t npoints = pointsGood_.size(); // find motion @@ -462,11 +550,9 @@ Mat PyrLkRobustMotionEstimator::estimate(const Mat &frame0, const Mat &frame1, b #if HAVE_OPENCV_GPU PyrLkRobustMotionEstimatorGpu::PyrLkRobustMotionEstimatorGpu(MotionModel model) + : PyrLkRobustMotionEstimatorBase(model) { CV_Assert(gpu::getCudaEnabledDeviceCount() > 0); - setMotionModel(model); - setRansacParams(RansacParams::default2dMotion(model)); - setMinInlierRatio(0.1f); } @@ -506,8 +592,34 @@ Mat PyrLkRobustMotionEstimatorGpu::estimate(const gpu::GpuMat &frame0, const gpu pointsPrev_.download(hostPointsPrev_); points_.download(hostPoints_); + Point2f *points0 = hostPointsPrev_.ptr(); + Point2f *points1 = hostPoints_.ptr(); int npoints = hostPointsPrev_.cols; + // perfrom outlier rejection + + IOutlierRejector *outlierRejector = static_cast(outlierRejector_); + if (!dynamic_cast(outlierRejector)) + { + outlierRejector_->process(frame0.size(), hostPointsPrev_, hostPoints_, rejectionStatus_); + + hostPointsPrevGood_.clear(); hostPointsPrevGood_.reserve(hostPoints_.cols); + hostPointsGood_.clear(); hostPointsGood_.reserve(hostPoints_.cols); + + for (int i = 0; i < hostPoints_.cols; ++i) + { + if (rejectionStatus_[i]) + { + hostPointsPrevGood_.push_back(hostPointsPrev_.at(0,i)); + hostPointsGood_.push_back(hostPoints_.at(0,i)); + } + } + + points0 = &hostPointsPrevGood_[0]; + points1 = &hostPointsGood_[0]; + npoints = static_cast(hostPointsGood_.size()); + } + // find motion int ninliers = 0; @@ -515,12 +627,13 @@ Mat PyrLkRobustMotionEstimatorGpu::estimate(const gpu::GpuMat &frame0, const gpu if (motionModel_ != MM_HOMOGRAPHY) M = estimateGlobalMotionRobust( - npoints, hostPointsPrev_.ptr(0), hostPoints_.ptr(), motionModel_, - ransacParams_, 0, &ninliers); + npoints, points0, points1, motionModel_, ransacParams_, 0, &ninliers); else { vector mask; - M = findHomography(hostPointsPrev_, hostPoints_, mask, CV_RANSAC, ransacParams_.thresh); + M = findHomography( + Mat(1, npoints, CV_32FC2, points0), Mat(1, npoints, CV_32FC2, points1), + mask, CV_RANSAC, ransacParams_.thresh); for (int i = 0; i < npoints; ++i) if (mask[i]) ninliers++; } @@ -558,3 +671,4 @@ Mat getMotion(int from, int to, const vector &motions) } // namespace videostab } // namespace cv + diff --git a/modules/videostab/src/outlier_rejection.cpp b/modules/videostab/src/outlier_rejection.cpp new file mode 100644 index 0000000..d3a9db0 --- /dev/null +++ b/modules/videostab/src/outlier_rejection.cpp @@ -0,0 +1,201 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. +// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include "precomp.hpp" +#include "opencv2/core/core.hpp" +#include "opencv2/videostab/outlier_rejection.hpp" + +using namespace std; + +namespace cv +{ +namespace videostab +{ + +void NullOutlierRejector::process( + Size frameSize, InputArray points0, InputArray points1, OutputArray mask) +{ + CV_Assert(points0.type() == points1.type()); + CV_Assert(points0.getMat().checkVector(2) == points1.getMat().checkVector(2)); + + int npoints = points0.getMat().checkVector(2); + mask.create(1, npoints, CV_8U); + Mat mask_ = mask.getMat(); + mask_.setTo(1); +} + +TranslationBasedLocalOutlierRejector::TranslationBasedLocalOutlierRejector() +{ + setCellSize(Size(50, 50)); + setRansacParams(RansacParams::default2dMotion(MM_TRANSLATION)); +} + + +void TranslationBasedLocalOutlierRejector::process( + Size frameSize, InputArray points0, InputArray points1, OutputArray mask) +{ + CV_Assert(points0.type() == points1.type()); + CV_Assert(points0.getMat().checkVector(2) == points1.getMat().checkVector(2)); + + int npoints = points0.getMat().checkVector(2); + + const Point2f* points0_ = points0.getMat().ptr(); + const Point2f* points1_ = points1.getMat().ptr(); + + mask.create(1, npoints, CV_8U); + uchar* mask_ = mask.getMat().ptr(); + + Size ncells((frameSize.width + cellSize_.width - 1) / cellSize_.width, + (frameSize.height + cellSize_.height - 1) / cellSize_.height); + + int cx, cy; + + // fill grid cells + + grid_.assign(ncells.area(), Cell()); + + for (int i = 0; i < npoints; ++i) + { + cx = std::min(cvRound(points0_[i].x / cellSize_.width), ncells.width - 1); + cy = std::min(cvRound(points0_[i].y / cellSize_.height), ncells.height - 1); + grid_[cy * ncells.width + cx].push_back(i); + } + + // process each cell + + RNG rng(0); + int niters = ransacParams_.niters(); + int ninliers, ninliersMax; + vector inliers; + float dx, dy, dxBest, dyBest; + float x1, y1; + int idx; + + for (size_t ci = 0; ci < grid_.size(); ++ci) + { + // estimate translation model at the current cell using RANSAC + + const Cell &cell = grid_[ci]; + ninliersMax = 0; + dxBest = dyBest = 0.f; + + // find the best hypothesis + + if (!cell.empty()) + { + for (int iter = 0; iter < niters; ++iter) + { + idx = cell[static_cast(rng) % cell.size()]; + dx = points1_[idx].x - points0_[idx].x; + dy = points1_[idx].y - points0_[idx].y; + + ninliers = 0; + for (size_t i = 0; i < cell.size(); ++i) + { + x1 = points0_[cell[i]].x + dx; + y1 = points0_[cell[i]].y + dy; + if (sqr(x1 - points1_[cell[i]].x) + sqr(y1 - points1_[cell[i]].y) < + sqr(ransacParams_.thresh)) + { + ninliers++; + } + } + + if (ninliers > ninliersMax) + { + ninliersMax = ninliers; + dxBest = dx; + dyBest = dy; + } + } + } + + // get the best hypothesis inliers + + ninliers = 0; + inliers.resize(ninliersMax); + for (size_t i = 0; i < cell.size(); ++i) + { + x1 = points0_[cell[i]].x + dxBest; + y1 = points0_[cell[i]].y + dyBest; + if (sqr(x1 - points1_[cell[i]].x) + sqr(y1 - points1_[cell[i]].y) < + sqr(ransacParams_.thresh)) + { + inliers[ninliers++] = cell[i]; + } + } + + // refine the best hypothesis + + dxBest = dyBest = 0.f; + for (size_t i = 0; i < inliers.size(); ++i) + { + dxBest += points1_[inliers[i]].x - points0_[inliers[i]].x; + dyBest += points1_[inliers[i]].y - points0_[inliers[i]].y; + } + if (!inliers.empty()) + { + dxBest /= inliers.size(); + dyBest /= inliers.size(); + } + + // set mask elements for refined model inliers + + for (size_t i = 0; i < cell.size(); ++i) + { + x1 = points0_[cell[i]].x + dxBest; + y1 = points0_[cell[i]].y + dyBest; + if (sqr(x1 - points1_[cell[i]].x) + sqr(y1 - points1_[cell[i]].y) < + sqr(ransacParams_.thresh)) + { + mask_[cell[i]] = 1; + } + else + { + mask_[cell[i]] = 0; + } + } + } +} + +} // namespace videostab +} // namespace cv diff --git a/samples/cpp/videostab.cpp b/samples/cpp/videostab.cpp index ff2fb2a..4c1ab2a 100644 --- a/samples/cpp/videostab.cpp +++ b/samples/cpp/videostab.cpp @@ -29,6 +29,7 @@ bool quietMode; void run(); void saveMotionsIfNecessary(); void printHelp(); +MotionModel motionModel(const string &str); void run() @@ -70,7 +71,7 @@ void printHelp() cout << "OpenCV video stabilizer.\n" "Usage: videostab [arguments]\n\n" "Arguments:\n" - " -m, --model=(transl|transl_and_scale|similarity|affine|homography)\n" + " -m, --model=(transl|transl_and_scale|rigid|similarity|affine|homography)\n" " Set motion model. The default is affine.\n" " --subset=(|auto)\n" " Number of random samples per one motion hypothesis. The default is auto.\n" @@ -83,7 +84,9 @@ void printHelp() " --nkps=\n" " Number of keypoints to find in each frame. The default is 1000.\n" " --extra-kps=\n" - " Extra keypoint grid size for motion estimation. The default is 0.\n\n" + " Extra keypoint grid size for motion estimation. The default is 0.\n" + " --local-outlier-rejection=(yes|no)\n" + " Perform local outlier rejection. The default is no.\n\n" " -sm, --save-motions=(|no)\n" " Save estimated motions into file. The default is no.\n" " -lm, --load-motions=(|no)\n" @@ -134,7 +137,7 @@ void printHelp() " Perform wobble suppression. The default is no.\n" " --ws-period=\n" " Set wobble suppression period. The default is 30.\n" - " --ws-model=(transl|transl_and_scale|similarity|affine|homography)\n" + " --ws-model=(transl|transl_and_scale|rigid|similarity|affine|homography)\n" " Set wobble suppression motion model (must have more DOF than motion \n" " estimation model). The default is homography.\n" " --ws-subset=(|auto)\n" @@ -148,7 +151,9 @@ void printHelp() " --ws-nkps=\n" " Number of keypoints to find in each frame. The default is 1000.\n" " --ws-extra-kps=\n" - " Extra keypoint grid size for motion estimation. The default is 0.\n\n" + " Extra keypoint grid size for motion estimation. The default is 0.\n" + " --ws-local-outlier-rejection=(yes|no)\n" + " Perform local outlier rejection. The default is no.\n\n" " -sm2, --save-motions2=(|no)\n" " Save motions estimated for wobble suppression. The default is no.\n" " -lm2, --load-motions2=(|no)\n" @@ -180,6 +185,7 @@ int main(int argc, const char **argv) "{ | min-inlier-ratio | 0.1 | }" "{ | nkps | 1000 | }" "{ | extra-kps | 0 | }" + "{ | local-outlier-rejection | no | }" "{ sm | save-motions | no | }" "{ lm | load-motions | no | }" "{ r | radius | 15 | }" @@ -211,6 +217,7 @@ int main(int argc, const char **argv) "{ | ws-min-inlier-ratio | 0.1 | }" "{ | ws-nkps | 1000 | }" "{ | ws-extra-kps | 0 | }" + "{ | ws-local-outlier-rejection | no | }" "{ sm2 | save-motions2 | no | }" "{ lm2 | load-motions2 | no | }" "{ gpu | | no }" @@ -273,29 +280,21 @@ int main(int argc, const char **argv) twoPassStabilizer->setMotionStabilizer(new GaussianMotionFilter(argi("radius"), argf("stdev"))); if (arg("wobble-suppress") == "yes") { - MoreAccurateMotionWobbleSuppressorBase *ws; + MoreAccurateMotionWobbleSuppressorBase *ws = 0; - if (arg("gpu") == "no") + Ptr outlierRejector = new NullOutlierRejector(); + if (arg("local-outlier-rejection") == "yes") { - ws = new MoreAccurateMotionWobbleSuppressor(); - PyrLkRobustMotionEstimator *est = 0; - - if (arg("ws-model") == "transl") - est = new PyrLkRobustMotionEstimator(MM_TRANSLATION); - else if (arg("ws-model") == "transl_and_scale") - est = new PyrLkRobustMotionEstimator(MM_TRANSLATION_AND_SCALE); - else if (arg("ws-model") == "similarity") - est = new PyrLkRobustMotionEstimator(MM_SIMILARITY); - else if (arg("ws-model") == "affine") - est = new PyrLkRobustMotionEstimator(MM_AFFINE); - else if (arg("ws-model") == "homography") - est = new PyrLkRobustMotionEstimator(MM_HOMOGRAPHY); - else - { - delete est; - throw runtime_error("unknown wobble suppression motion model: " + arg("ws-model")); - } + TranslationBasedLocalOutlierRejector *tor = new TranslationBasedLocalOutlierRejector(); + RansacParams ransacParams = tor->ransacParams(); + if (arg("ws-thresh") != "auto") ransacParams.thresh = argf("ws-thresh"); + tor->setRansacParams(ransacParams); + outlierRejector = tor; + } + if (arg("gpu") == "no") + { + PyrLkRobustMotionEstimator *est = new PyrLkRobustMotionEstimator(motionModel(arg("ws-model"))); est->setDetector(new GoodFeaturesToTrackDetector(argi("ws-nkps"))); RansacParams ransac = est->ransacParams(); @@ -306,29 +305,15 @@ int main(int argc, const char **argv) est->setMinInlierRatio(argf("ws-min-inlier-ratio")); est->setGridSize(Size(argi("ws-extra-kps"), argi("ws-extra-kps"))); + est->setOutlierRejector(outlierRejector); + + ws = new MoreAccurateMotionWobbleSuppressor(); ws->setMotionEstimator(est); } else if (arg("gpu") == "yes") { #if HAVE_OPENCV_GPU - ws = new MoreAccurateMotionWobbleSuppressorGpu(); - PyrLkRobustMotionEstimatorGpu *est = 0; - - if (arg("ws-model") == "transl") - est = new PyrLkRobustMotionEstimatorGpu(MM_TRANSLATION); - else if (arg("ws-model") == "transl_and_scale") - est = new PyrLkRobustMotionEstimatorGpu(MM_TRANSLATION_AND_SCALE); - else if (arg("ws-model") == "similarity") - est = new PyrLkRobustMotionEstimatorGpu(MM_SIMILARITY); - else if (arg("ws-model") == "affine") - est = new PyrLkRobustMotionEstimatorGpu(MM_AFFINE); - else if (arg("ws-model") == "homography") - est = new PyrLkRobustMotionEstimatorGpu(MM_HOMOGRAPHY); - else - { - delete est; - throw runtime_error("unknown wobble suppression motion model: " + arg("ws-model")); - } + PyrLkRobustMotionEstimatorGpu *est = new PyrLkRobustMotionEstimatorGpu(motionModel(arg("ws-model"))); RansacParams ransac = est->ransacParams(); if (arg("ws-subset") != "auto") ransac.size = argi("ws-subset"); @@ -337,6 +322,9 @@ int main(int argc, const char **argv) est->setRansacParams(ransac); est->setMinInlierRatio(argf("ws-min-inlier-ratio")); + est->setOutlierRejector(outlierRejector); + + ws = new MoreAccurateMotionWobbleSuppressorGpu(); ws->setMotionEstimator(est); #else throw runtime_error("OpenCV is built without GPU support"); @@ -376,27 +364,21 @@ int main(int argc, const char **argv) stabilizer->setFrameSource(source); stabilizedFrames = dynamic_cast(stabilizer); - if (arg("gpu") == "no") + Ptr outlierRejector = new NullOutlierRejector(); + if (arg("local-outlier-rejection") == "yes") { - PyrLkRobustMotionEstimator *est = 0; - - if (arg("model") == "transl") - est = new PyrLkRobustMotionEstimator(MM_TRANSLATION); - else if (arg("model") == "transl_and_scale") - est = new PyrLkRobustMotionEstimator(MM_TRANSLATION_AND_SCALE); - else if (arg("model") == "similarity") - est = new PyrLkRobustMotionEstimator(MM_SIMILARITY); - else if (arg("model") == "affine") - est = new PyrLkRobustMotionEstimator(MM_AFFINE); - else if (arg("model") == "homography") - est = new PyrLkRobustMotionEstimator(MM_HOMOGRAPHY); - else - { - delete est; - throw runtime_error("unknown motion model: " + arg("model")); - } + TranslationBasedLocalOutlierRejector *tor = new TranslationBasedLocalOutlierRejector(); + RansacParams ransacParams = tor->ransacParams(); + if (arg("thresh") != "auto") ransacParams.thresh = argf("thresh"); + tor->setRansacParams(ransacParams); + outlierRejector = tor; + } + if (arg("gpu") == "no") + { + PyrLkRobustMotionEstimator *est = new PyrLkRobustMotionEstimator(motionModel(arg("model")));; est->setDetector(new GoodFeaturesToTrackDetector(argi("nkps"))); + RansacParams ransac = est->ransacParams(); if (arg("subset") != "auto") ransac.size = argi("subset"); if (arg("thresh") != "auto") ransac.thresh = argi("thresh"); @@ -405,28 +387,14 @@ int main(int argc, const char **argv) est->setMinInlierRatio(argf("min-inlier-ratio")); est->setGridSize(Size(argi("extra-kps"), argi("extra-kps"))); + est->setOutlierRejector(outlierRejector); + stabilizer->setMotionEstimator(est); } else if (arg("gpu") == "yes") { #if HAVE_OPENCV_GPU - PyrLkRobustMotionEstimatorGpu *est = 0; - - if (arg("model") == "transl") - est = new PyrLkRobustMotionEstimatorGpu(MM_TRANSLATION); - else if (arg("model") == "transl_and_scale") - est = new PyrLkRobustMotionEstimatorGpu(MM_TRANSLATION_AND_SCALE); - else if (arg("model") == "similarity") - est = new PyrLkRobustMotionEstimatorGpu(MM_SIMILARITY); - else if (arg("model") == "affine") - est = new PyrLkRobustMotionEstimatorGpu(MM_AFFINE); - else if (arg("model") == "homography") - est = new PyrLkRobustMotionEstimatorGpu(MM_HOMOGRAPHY); - else - { - delete est; - throw runtime_error("unknown wobble suppression motion model: " + arg("ws-model")); - } + PyrLkRobustMotionEstimatorGpu *est = new PyrLkRobustMotionEstimatorGpu(motionModel(arg("model")));; RansacParams ransac = est->ransacParams(); if (arg("subset") != "auto") ransac.size = argi("subset"); @@ -435,6 +403,8 @@ int main(int argc, const char **argv) est->setRansacParams(ransac); est->setMinInlierRatio(argf("min-inlier-ratio")); + est->setOutlierRejector(outlierRejector); + stabilizer->setMotionEstimator(est); #else throw runtime_error("OpenCV is built without GPU support"); @@ -523,3 +493,21 @@ int main(int argc, const char **argv) stabilizedFrames.release(); return 0; } + + +MotionModel motionModel(const string &str) +{ + if (str == "transl") + return MM_TRANSLATION; + if (str == "transl_and_scale") + return MM_TRANSLATION_AND_SCALE; + if (str == "rigid") + return MM_RIGID; + if (str == "similarity") + return MM_SIMILARITY; + if (str == "affine") + return MM_AFFINE; + if (str == "homography") + return MM_HOMOGRAPHY; + throw runtime_error("unknown motion model: " + str); +} -- 2.7.4