From f32b645b96264bb9a2bcb54d4cb5f830969e6f13 Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Thu, 5 Apr 2012 09:15:13 +0000 Subject: [PATCH] Refactored videostab module, added base class for woobble suppression --- .../include/opencv2/videostab/global_motion.hpp | 9 + .../include/opencv2/videostab/stabilizer.hpp | 36 ++-- .../opencv2/videostab/wobble_suppression.hpp | 91 +++++++++++ modules/videostab/src/stabilizer.cpp | 181 ++++++++++++--------- modules/videostab/src/wobble_suppression.cpp | 60 +++++++ samples/cpp/videostab.cpp | 8 +- 6 files changed, 295 insertions(+), 90 deletions(-) create mode 100644 modules/videostab/include/opencv2/videostab/wobble_suppression.hpp create mode 100644 modules/videostab/src/wobble_suppression.cpp diff --git a/modules/videostab/include/opencv2/videostab/global_motion.hpp b/modules/videostab/include/opencv2/videostab/global_motion.hpp index 8e0bba0..5b00487 100644 --- a/modules/videostab/include/opencv2/videostab/global_motion.hpp +++ b/modules/videostab/include/opencv2/videostab/global_motion.hpp @@ -105,6 +105,15 @@ protected: MotionModel motionModel_; }; +class CV_EXPORTS EyeMotionEstimator : public GlobalMotionEstimatorBase +{ +public: + virtual Mat estimate(const Mat &/*frame0*/, const Mat &/*frame1*/) + { + return Mat::eye(3, 3, CV_32F); + } +}; + class CV_EXPORTS PyrLkRobustMotionEstimator : public GlobalMotionEstimatorBase { public: diff --git a/modules/videostab/include/opencv2/videostab/stabilizer.hpp b/modules/videostab/include/opencv2/videostab/stabilizer.hpp index 4eef2a6..0a35df3 100644 --- a/modules/videostab/include/opencv2/videostab/stabilizer.hpp +++ b/modules/videostab/include/opencv2/videostab/stabilizer.hpp @@ -52,6 +52,7 @@ #include "opencv2/videostab/log.hpp" #include "opencv2/videostab/inpainting.hpp" #include "opencv2/videostab/deblurring.hpp" +#include "opencv2/videostab/wobble_suppression.hpp" namespace cv { @@ -93,14 +94,14 @@ public: protected: StabilizerBase(); - void setUp(int cacheSize, const Mat &frame); + void reset(); Mat nextStabilizedFrame(); bool doOneIteration(); - void stabilizeFrame(const Mat &stabilizationMotion); - - virtual void setUp(Mat &firstFrame) = 0; - virtual void stabilizeFrame() = 0; - virtual void estimateMotion() = 0; + virtual void setUp(const Mat &firstFrame); + virtual Mat estimateMotion() = 0; + virtual Mat estimateStabilizationMotion() = 0; + void stabilizeFrame(); + virtual Mat postProcessFrame(const Mat &frame); Ptr log_; Ptr frameSource_; @@ -120,6 +121,7 @@ protected: Mat preProcessedFrame_; bool doInpainting_; Mat inpaintingMask_; + Mat finalFrame_; std::vector frames_; std::vector motions_; // motions_[i] is the motion from i-th to i+1-th frame std::vector blurrinessRates_; @@ -140,9 +142,10 @@ public: virtual Mat nextFrame() { return nextStabilizedFrame(); } private: - virtual void setUp(Mat &firstFrame); - virtual void estimateMotion(); - virtual void stabilizeFrame(); + virtual void setUp(const Mat &firstFrame); + virtual Mat estimateMotion(); + virtual Mat estimateStabilizationMotion(); + virtual Mat postProcessFrame(const Mat &frame); Ptr motionFilter_; }; @@ -153,7 +156,10 @@ public: TwoPassStabilizer(); void setMotionStabilizer(Ptr val) { motionStabilizer_ = val; } - Ptr motionStabilizer() const { return motionStabilizer_; } + Ptr motionStabilizer() const { return motionStabilizer_; } + + void setWobbleSuppressor(Ptr val) { wobbleSuppressor_ = val; } + Ptr wobbleSuppressor() const { return wobbleSuppressor_; } void setEstimateTrimRatio(bool val) { mustEstTrimRatio_ = val; } bool mustEstimateTrimaRatio() const { return mustEstTrimRatio_; } @@ -165,18 +171,20 @@ public: std::vector motions() const; private: - void resetImpl(); void runPrePassIfNecessary(); - virtual void setUp(Mat &firstFrame); - virtual void estimateMotion() { /* do nothing as motion was estimation in pre-pass */ } - virtual void stabilizeFrame(); + virtual void setUp(const Mat &firstFrame); + virtual Mat estimateMotion(); + virtual Mat estimateStabilizationMotion(); + virtual Mat postProcessFrame(const Mat &frame); Ptr motionStabilizer_; + Ptr wobbleSuppressor_; bool mustEstTrimRatio_; int frameCount_; bool isPrePassDone_; + Mat suppressedFrame_; }; } // namespace videostab diff --git a/modules/videostab/include/opencv2/videostab/wobble_suppression.hpp b/modules/videostab/include/opencv2/videostab/wobble_suppression.hpp new file mode 100644 index 0000000..772c29a --- /dev/null +++ b/modules/videostab/include/opencv2/videostab/wobble_suppression.hpp @@ -0,0 +1,91 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. +// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#ifndef __OPENCV_VIDEOSTAB_WOBBLE_SUPPRESSION_HPP__ +#define __OPENCV_VIDEOSTAB_WOBBLE_SUPPRESSION_HPP__ + +#include +#include "opencv2/core/core.hpp" +#include "opencv2/videostab/global_motion.hpp" + +namespace cv +{ +namespace videostab +{ + +class CV_EXPORTS WobbleSuppressorBase +{ +public: + virtual ~WobbleSuppressorBase() {} + + virtual void setFrames(const std::vector &val) { frames_ = &val; } + virtual const std::vector& frames() const { return *frames_; } + + virtual void setMotions(const std::vector &val) { motions_ = &val; } + virtual const std::vector& motions() const { return *motions_; } + + virtual void setStabilizedFrames(const std::vector &val) { stabilizedFrames_ = &val; } + virtual const std::vector& stabilizedFrames() const { return *stabilizedFrames_; } + + virtual void setStabilizationMotions(const std::vector &val) { stabilizationMotions_ = &val; } + virtual const std::vector& stabilizationMotions() const { return *stabilizationMotions_; } + + virtual void suppress(int idx, Mat &frame) = 0; + +protected: + const std::vector *frames_; + const std::vector *motions_; + const std::vector *stabilizedFrames_; + const std::vector *stabilizationMotions_; +}; + +class CV_EXPORTS NullWobbleSuppressor : public WobbleSuppressorBase +{ +public: + virtual void suppress(int idx, Mat &result); +}; + +} // namespace videostab +} // namespace cv + +#endif + diff --git a/modules/videostab/src/stabilizer.cpp b/modules/videostab/src/stabilizer.cpp index 6177d3f..bfa0646 100644 --- a/modules/videostab/src/stabilizer.cpp +++ b/modules/videostab/src/stabilizer.cpp @@ -65,52 +65,40 @@ StabilizerBase::StabilizerBase() } -void StabilizerBase::setUp(int cacheSize, const Mat &frame) +void StabilizerBase::reset() { - InpainterBase *inpainter = static_cast(inpainter_); - doInpainting_ = dynamic_cast(inpainter) == 0; - if (doInpainting_) - { - inpainter_->setMotionModel(motionEstimator_->motionModel()); - inpainter_->setFrames(frames_); - inpainter_->setMotions(motions_); - inpainter_->setStabilizedFrames(stabilizedFrames_); - inpainter_->setStabilizationMotions(stabilizationMotions_); - } - - DeblurerBase *deblurer = static_cast(deblurer_); - doDeblurring_ = dynamic_cast(deblurer) == 0; - if (doDeblurring_) - { - blurrinessRates_.resize(cacheSize); - float blurriness = calcBlurriness(frame); - for (int i = -radius_; i <= 0; ++i) - at(i, blurrinessRates_) = blurriness; - deblurer_->setFrames(frames_); - deblurer_->setMotions(motions_); - deblurer_->setBlurrinessRates(blurrinessRates_); - } - - log_->print("processing frames"); + frameSize_ = Size(0, 0); + frameMask_ = Mat(); + curPos_ = -1; + curStabilizedPos_ = -1; + doDeblurring_ = false; + preProcessedFrame_ = Mat(); + doInpainting_ = false; + inpaintingMask_ = Mat(); + frames_.clear(); + motions_.clear(); + blurrinessRates_.clear(); + stabilizedFrames_.clear(); + stabilizedMasks_.clear(); + stabilizationMotions_.clear(); } Mat StabilizerBase::nextStabilizedFrame() { + // check if we've processed all frames already if (curStabilizedPos_ == curPos_ && curStabilizedPos_ != -1) - return Mat(); // we've processed all frames already + return Mat(); bool processed; do processed = doOneIteration(); while (processed && curStabilizedPos_ == -1); + // check if frame source is empty if (curStabilizedPos_ == -1) - return Mat(); // frame source is empty + return Mat(); - const Mat &stabilizedFrame = at(curStabilizedPos_, stabilizedFrames_); - int dx = static_cast(floor(trimRatio_ * stabilizedFrame.cols)); - int dy = static_cast(floor(trimRatio_ * stabilizedFrame.rows)); - return stabilizedFrame(Rect(dx, dy, stabilizedFrame.cols - 2*dx, stabilizedFrame.rows - 2*dy)); + return postProcessFrame(at(curStabilizedPos_, stabilizedFrames_)); } @@ -128,7 +116,7 @@ bool StabilizerBase::doOneIteration() if (doDeblurring_) at(curPos_, blurrinessRates_) = calcBlurriness(frame); - estimateMotion(); + at(curPos_ - 1, motions_) = estimateMotion(); if (curPos_ >= radius_) { @@ -158,15 +146,43 @@ bool StabilizerBase::doOneIteration() } -void StabilizerBase::stabilizeFrame(const Mat &stabilizationMotion) +void StabilizerBase::setUp(const Mat &firstFrame) { - Mat stabilizationMotion_; + InpainterBase *inpainter = static_cast(inpainter_); + doInpainting_ = dynamic_cast(inpainter) == 0; + if (doInpainting_) + { + inpainter_->setMotionModel(motionEstimator_->motionModel()); + inpainter_->setFrames(frames_); + inpainter_->setMotions(motions_); + inpainter_->setStabilizedFrames(stabilizedFrames_); + inpainter_->setStabilizationMotions(stabilizationMotions_); + } + + DeblurerBase *deblurer = static_cast(deblurer_); + doDeblurring_ = dynamic_cast(deblurer) == 0; + if (doDeblurring_) + { + blurrinessRates_.resize(2*radius_ + 1); + float blurriness = calcBlurriness(firstFrame); + for (int i = -radius_; i <= 0; ++i) + at(i, blurrinessRates_) = blurriness; + deblurer_->setFrames(frames_); + deblurer_->setMotions(motions_); + deblurer_->setBlurrinessRates(blurrinessRates_); + } + + log_->print("processing frames"); +} + + +void StabilizerBase::stabilizeFrame() +{ + Mat stabilizationMotion = estimateStabilizationMotion(); if (doCorrectionForInclusion_) - stabilizationMotion_ = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_); - else - stabilizationMotion_ = stabilizationMotion.clone(); + stabilizationMotion = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_); - at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion_; + at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion; if (doDeblurring_) { @@ -181,22 +197,22 @@ void StabilizerBase::stabilizeFrame(const Mat &stabilizationMotion) if (motionEstimator_->motionModel() != HOMOGRAPHY) warpAffine( preProcessedFrame_, at(curStabilizedPos_, stabilizedFrames_), - stabilizationMotion_(Rect(0,0,3,2)), frameSize_, INTER_LINEAR, borderMode_); + stabilizationMotion(Rect(0,0,3,2)), frameSize_, INTER_LINEAR, borderMode_); else warpPerspective( preProcessedFrame_, at(curStabilizedPos_, stabilizedFrames_), - stabilizationMotion_, frameSize_, INTER_LINEAR, borderMode_); + stabilizationMotion, frameSize_, INTER_LINEAR, borderMode_); if (doInpainting_) { if (motionEstimator_->motionModel() != HOMOGRAPHY) warpAffine( frameMask_, at(curStabilizedPos_, stabilizedMasks_), - stabilizationMotion_(Rect(0,0,3,2)), frameSize_, INTER_NEAREST); + stabilizationMotion(Rect(0,0,3,2)), frameSize_, INTER_NEAREST); else warpPerspective( frameMask_, at(curStabilizedPos_, stabilizedMasks_), - stabilizationMotion_, frameSize_, INTER_NEAREST); + stabilizationMotion, frameSize_, INTER_NEAREST); erode(at(curStabilizedPos_, stabilizedMasks_), at(curStabilizedPos_, stabilizedMasks_), Mat()); @@ -209,6 +225,15 @@ void StabilizerBase::stabilizeFrame(const Mat &stabilizationMotion) } +Mat StabilizerBase::postProcessFrame(const Mat &frame) +{ + // trim frame + int dx = static_cast(floor(trimRatio_ * frame.cols)); + int dy = static_cast(floor(trimRatio_ * frame.rows)); + return frame(Rect(dx, dy, frame.cols - 2*dx, frame.rows - 2*dy)); +} + + OnePassStabilizer::OnePassStabilizer() { setMotionFilter(new GaussianMotionFilter()); @@ -218,25 +243,17 @@ OnePassStabilizer::OnePassStabilizer() void OnePassStabilizer::reset() { - curPos_ = -1; - curStabilizedPos_ = -1; - frames_.clear(); - motions_.clear(); - stabilizedFrames_.clear(); - stabilizationMotions_.clear(); - doDeblurring_ = false; - doInpainting_ = false; + StabilizerBase::reset(); } -void OnePassStabilizer::setUp(Mat &firstFrame) +void OnePassStabilizer::setUp(const Mat &firstFrame) { frameSize_ = firstFrame.size(); frameMask_.create(frameSize_, CV_8U); frameMask_.setTo(255); int cacheSize = 2*radius_ + 1; - frames_.resize(cacheSize); stabilizedFrames_.resize(cacheSize); stabilizedMasks_.resize(cacheSize); @@ -251,27 +268,32 @@ void OnePassStabilizer::setUp(Mat &firstFrame) at(0, frames_) = firstFrame; - StabilizerBase::setUp(cacheSize, firstFrame); + StabilizerBase::setUp(firstFrame); } -void OnePassStabilizer::estimateMotion() +Mat OnePassStabilizer::estimateMotion() { - at(curPos_ - 1, motions_) = motionEstimator_->estimate( - at(curPos_ - 1, frames_), at(curPos_, frames_)); + return motionEstimator_->estimate(at(curPos_ - 1, frames_), at(curPos_, frames_)); } -void OnePassStabilizer::stabilizeFrame() +Mat OnePassStabilizer::estimateStabilizationMotion() { - Mat stabilizationMotion = motionFilter_->stabilize(curStabilizedPos_, motions_, make_pair(0, curPos_)); - StabilizerBase::stabilizeFrame(stabilizationMotion); + return motionFilter_->stabilize(curStabilizedPos_, motions_, make_pair(0, curPos_)); +} + + +Mat OnePassStabilizer::postProcessFrame(const Mat &frame) +{ + return StabilizerBase::postProcessFrame(frame); } TwoPassStabilizer::TwoPassStabilizer() { setMotionStabilizer(new GaussianMotionFilter()); + setWobbleSuppressor(new NullWobbleSuppressor()); setEstimateTrimRatio(false); reset(); } @@ -279,16 +301,10 @@ TwoPassStabilizer::TwoPassStabilizer() void TwoPassStabilizer::reset() { - isPrePassDone_ = false; + StabilizerBase::reset(); frameCount_ = 0; - curPos_ = -1; - curStabilizedPos_ = -1; - frames_.clear(); - motions_.clear(); - stabilizedFrames_.clear(); - stabilizationMotions_.clear(); - doDeblurring_ = false; - doInpainting_ = false; + isPrePassDone_ = false; + suppressedFrame_ = Mat(); } @@ -360,10 +376,9 @@ void TwoPassStabilizer::runPrePassIfNecessary() } -void TwoPassStabilizer::setUp(Mat &firstFrame) +void TwoPassStabilizer::setUp(const Mat &firstFrame) { int cacheSize = 2*radius_ + 1; - frames_.resize(cacheSize); stabilizedFrames_.resize(cacheSize); stabilizedMasks_.resize(cacheSize); @@ -371,13 +386,31 @@ void TwoPassStabilizer::setUp(Mat &firstFrame) for (int i = -radius_; i <= 0; ++i) at(i, frames_) = firstFrame; - StabilizerBase::setUp(cacheSize, firstFrame); + wobbleSuppressor_->setFrames(frames_); + wobbleSuppressor_->setMotions(motions_); + wobbleSuppressor_->setStabilizedFrames(stabilizedFrames_); + wobbleSuppressor_->setStabilizationMotions(stabilizationMotions_); + + StabilizerBase::setUp(firstFrame); +} + + +Mat TwoPassStabilizer::estimateMotion() +{ + return motions_[curPos_ - 1].clone(); +} + + +Mat TwoPassStabilizer::estimateStabilizationMotion() +{ + return stabilizationMotions_[curStabilizedPos_].clone(); } -void TwoPassStabilizer::stabilizeFrame() +Mat TwoPassStabilizer::postProcessFrame(const Mat &/*frame*/) { - StabilizerBase::stabilizeFrame(stabilizationMotions_[curStabilizedPos_]); + wobbleSuppressor_->suppress(curStabilizedPos_, suppressedFrame_); + return StabilizerBase::postProcessFrame(suppressedFrame_); } } // namespace videostab diff --git a/modules/videostab/src/wobble_suppression.cpp b/modules/videostab/src/wobble_suppression.cpp new file mode 100644 index 0000000..0895dfc --- /dev/null +++ b/modules/videostab/src/wobble_suppression.cpp @@ -0,0 +1,60 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. +// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include "precomp.hpp" +#include "opencv2/videostab/wobble_suppression.hpp" +#include "opencv2/videostab/ring_buffer.hpp" + +using namespace std; + +namespace cv +{ +namespace videostab +{ + +void NullWobbleSuppressor::suppress(int idx, Mat &result) +{ + result = at(idx, *stabilizedFrames_); +} + +} // namespace videostab +} // namespace cv diff --git a/samples/cpp/videostab.cpp b/samples/cpp/videostab.cpp index 74ef1bf..a7f41bb 100644 --- a/samples/cpp/videostab.cpp +++ b/samples/cpp/videostab.cpp @@ -71,9 +71,11 @@ void run() { VideoWriter writer; Mat stabilizedFrame; + int nframes = 0; while (!(stabilizedFrame = stabilizedFrames->nextFrame()).empty()) { + nframes++; if (!saveMotionsPath.empty()) saveMotionsIfNecessary(); if (!outputPath.empty()) @@ -91,7 +93,9 @@ void run() } } - cout << "\nfinished\n"; + cout << endl + << "processed frames: " << nframes << endl + << "finished\n"; } @@ -248,7 +252,7 @@ int main(int argc, const char **argv) if (inputPath.empty()) throw runtime_error("specify video file path"); VideoFileSource *source = new VideoFileSource(inputPath); - cout << "frame count: " << source->count() << endl; + cout << "frame count (rough): " << source->count() << endl; if (arg("fps") == "auto") outputFps = source->fps(); else outputFps = argd("fps"); stabilizer->setFrameSource(source); -- 2.7.4