From fa09f3d12197183398410101165a091fcdb4874f Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Thu, 5 Apr 2012 13:23:42 +0000 Subject: [PATCH] Refactored videostab module. Added MoreAccurateMotionWobbleSuppressor class --- .../include/opencv2/videostab/deblurring.hpp | 9 +- .../include/opencv2/videostab/global_motion.hpp | 24 ++++-- .../include/opencv2/videostab/inpainting.hpp | 9 +- .../include/opencv2/videostab/stabilizer.hpp | 2 + .../opencv2/videostab/wobble_suppression.hpp | 34 ++++++-- modules/videostab/src/global_motion.cpp | 35 ++++++++ modules/videostab/src/stabilizer.cpp | 31 +++++-- modules/videostab/src/wobble_suppression.cpp | 25 +++++- samples/cpp/videostab.cpp | 95 ++++++---------------- 9 files changed, 163 insertions(+), 101 deletions(-) diff --git a/modules/videostab/include/opencv2/videostab/deblurring.hpp b/modules/videostab/include/opencv2/videostab/deblurring.hpp index 6cbc47f..d2ab438 100644 --- a/modules/videostab/include/opencv2/videostab/deblurring.hpp +++ b/modules/videostab/include/opencv2/videostab/deblurring.hpp @@ -56,13 +56,18 @@ CV_EXPORTS float calcBlurriness(const Mat &frame); class CV_EXPORTS DeblurerBase { public: - DeblurerBase() : radius_(0), frames_(0), motions_(0) {} + DeblurerBase() : radius_(0), frames_(0), motions_(0), blurrinessRates_(0) {} virtual ~DeblurerBase() {} virtual void setRadius(int val) { radius_ = val; } virtual int radius() const { return radius_; } + virtual void deblur(int idx, Mat &frame) = 0; + + + // data from stabilizer + virtual void setFrames(const std::vector &val) { frames_ = &val; } virtual const std::vector& frames() const { return *frames_; } @@ -72,8 +77,6 @@ public: virtual void setBlurrinessRates(const std::vector &val) { blurrinessRates_ = &val; } virtual const std::vector& blurrinessRates() const { return *blurrinessRates_; } - virtual void deblur(int idx, Mat &frame) = 0; - protected: int radius_; const std::vector *frames_; diff --git a/modules/videostab/include/opencv2/videostab/global_motion.hpp b/modules/videostab/include/opencv2/videostab/global_motion.hpp index 5b00487..2c46350 100644 --- a/modules/videostab/include/opencv2/videostab/global_motion.hpp +++ b/modules/videostab/include/opencv2/videostab/global_motion.hpp @@ -44,6 +44,8 @@ #define __OPENCV_VIDEOSTAB_GLOBAL_MOTION_HPP__ #include +#include +#include #include "opencv2/core/core.hpp" #include "opencv2/features2d/features2d.hpp" #include "opencv2/videostab/optical_flow.hpp" @@ -105,13 +107,25 @@ protected: MotionModel motionModel_; }; -class CV_EXPORTS EyeMotionEstimator : public GlobalMotionEstimatorBase +class CV_EXPORTS FromFileMotionReader : public GlobalMotionEstimatorBase { public: - virtual Mat estimate(const Mat &/*frame0*/, const Mat &/*frame1*/) - { - return Mat::eye(3, 3, CV_32F); - } + FromFileMotionReader(const std::string &path); + virtual Mat estimate(const Mat &frame0, const Mat &frame1); + +private: + std::ifstream file_; +}; + +class CV_EXPORTS ToFileMotionWriter : public GlobalMotionEstimatorBase +{ +public: + ToFileMotionWriter(const std::string &path, Ptr estimator); + virtual Mat estimate(const Mat &frame0, const Mat &frame1); + +private: + std::ofstream file_; + Ptr estimator_; }; class CV_EXPORTS PyrLkRobustMotionEstimator : public GlobalMotionEstimatorBase diff --git a/modules/videostab/include/opencv2/videostab/inpainting.hpp b/modules/videostab/include/opencv2/videostab/inpainting.hpp index 25b5639..193c295 100644 --- a/modules/videostab/include/opencv2/videostab/inpainting.hpp +++ b/modules/videostab/include/opencv2/videostab/inpainting.hpp @@ -59,7 +59,7 @@ class CV_EXPORTS InpainterBase { public: InpainterBase() - : radius_(0), frames_(0), motions_(0), + : radius_(0), motionModel_(UNKNOWN), frames_(0), motions_(0), stabilizedFrames_(0), stabilizationMotions_(0) {} virtual ~InpainterBase() {} @@ -70,6 +70,11 @@ public: virtual void setMotionModel(MotionModel val) { motionModel_ = val; } virtual MotionModel motionModel() const { return motionModel_; } + virtual void inpaint(int idx, Mat &frame, Mat &mask) = 0; + + + // data from stabilizer + virtual void setFrames(const std::vector &val) { frames_ = &val; } virtual const std::vector& frames() const { return *frames_; } @@ -82,8 +87,6 @@ public: virtual void setStabilizationMotions(const std::vector &val) { stabilizationMotions_ = &val; } virtual const std::vector& stabilizationMotions() const { return *stabilizationMotions_; } - virtual void inpaint(int idx, Mat &frame, Mat &mask) = 0; - protected: int radius_; MotionModel motionModel_; diff --git a/modules/videostab/include/opencv2/videostab/stabilizer.hpp b/modules/videostab/include/opencv2/videostab/stabilizer.hpp index 0a35df3..3df22b4 100644 --- a/modules/videostab/include/opencv2/videostab/stabilizer.hpp +++ b/modules/videostab/include/opencv2/videostab/stabilizer.hpp @@ -184,6 +184,8 @@ private: int frameCount_; bool isPrePassDone_; + bool doWobbleSuppression_; + std::vector motions2_; Mat suppressedFrame_; }; diff --git a/modules/videostab/include/opencv2/videostab/wobble_suppression.hpp b/modules/videostab/include/opencv2/videostab/wobble_suppression.hpp index 772c29a..1e8dd56 100644 --- a/modules/videostab/include/opencv2/videostab/wobble_suppression.hpp +++ b/modules/videostab/include/opencv2/videostab/wobble_suppression.hpp @@ -46,6 +46,7 @@ #include #include "opencv2/core/core.hpp" #include "opencv2/videostab/global_motion.hpp" +#include "opencv2/videostab/log.hpp" namespace cv { @@ -55,33 +56,48 @@ namespace videostab class CV_EXPORTS WobbleSuppressorBase { public: + WobbleSuppressorBase(); + virtual ~WobbleSuppressorBase() {} - virtual void setFrames(const std::vector &val) { frames_ = &val; } - virtual const std::vector& frames() const { return *frames_; } + void setMotionEstimator(Ptr val) { motionEstimator_ = val; } + Ptr motionEstimator() const { return motionEstimator_; } + + virtual void suppress(int idx, const Mat &frame, Mat &result) = 0; + + + // data from stabilizer + + virtual void setFrameCount(int val) { frameCount_ = val; } + virtual int frameCount() const { return frameCount_; } virtual void setMotions(const std::vector &val) { motions_ = &val; } virtual const std::vector& motions() const { return *motions_; } - virtual void setStabilizedFrames(const std::vector &val) { stabilizedFrames_ = &val; } - virtual const std::vector& stabilizedFrames() const { return *stabilizedFrames_; } + virtual void setMotions2(const std::vector &val) { motions2_ = &val; } + virtual const std::vector& motions2() const { return *motions2_; } virtual void setStabilizationMotions(const std::vector &val) { stabilizationMotions_ = &val; } virtual const std::vector& stabilizationMotions() const { return *stabilizationMotions_; } - virtual void suppress(int idx, Mat &frame) = 0; - protected: - const std::vector *frames_; + Ptr motionEstimator_; + int frameCount_; const std::vector *motions_; - const std::vector *stabilizedFrames_; + const std::vector *motions2_; const std::vector *stabilizationMotions_; }; class CV_EXPORTS NullWobbleSuppressor : public WobbleSuppressorBase { public: - virtual void suppress(int idx, Mat &result); + virtual void suppress(int idx, const Mat &frame, Mat &result); +}; + +class CV_EXPORTS MoreAccurateMotionWobbleSuppressor : public WobbleSuppressorBase +{ +public: + virtual void suppress(int idx, const Mat &frame, Mat &result); }; } // namespace videostab diff --git a/modules/videostab/src/global_motion.cpp b/modules/videostab/src/global_motion.cpp index b3d476c..ee9e640 100644 --- a/modules/videostab/src/global_motion.cpp +++ b/modules/videostab/src/global_motion.cpp @@ -288,6 +288,41 @@ Mat estimateGlobalMotionRobust( } +FromFileMotionReader::FromFileMotionReader(const string &path) +{ + file_.open(path.c_str()); + CV_Assert(file_.is_open()); +} + + +Mat FromFileMotionReader::estimate(const Mat &/*frame0*/, const Mat &/*frame1*/) +{ + Mat_ M(3, 3); + file_ >> M(0,0) >> M(0,1) >> M(0,2) + >> M(1,0) >> M(1,1) >> M(1,2) + >> M(2,0) >> M(2,1) >> M(2,2); + return M; +} + + +ToFileMotionWriter::ToFileMotionWriter(const string &path, Ptr estimator) +{ + file_.open(path.c_str()); + CV_Assert(file_.is_open()); + estimator_ = estimator; +} + + +Mat ToFileMotionWriter::estimate(const Mat &frame0, const Mat &frame1) +{ + Mat_ M = estimator_->estimate(frame0, frame1); + file_ << M(0,0) << " " << M(0,1) << " " << M(0,2) << " " + << M(1,0) << " " << M(1,1) << " " << M(1,2) << " " + << M(2,0) << " " << M(2,1) << " " << M(2,2) << endl; + return M; +} + + PyrLkRobustMotionEstimator::PyrLkRobustMotionEstimator() : ransacParams_(RansacParams::affine2dMotionStd()) { diff --git a/modules/videostab/src/stabilizer.cpp b/modules/videostab/src/stabilizer.cpp index bfa0646..2544f9c 100644 --- a/modules/videostab/src/stabilizer.cpp +++ b/modules/videostab/src/stabilizer.cpp @@ -53,7 +53,7 @@ namespace videostab StabilizerBase::StabilizerBase() { - setLog(new NullLog()); + setLog(new LogToStdout()); setFrameSource(new NullFrameSource()); setMotionEstimator(new PyrLkRobustMotionEstimator()); setDeblurer(new NullDeblurer()); @@ -304,6 +304,8 @@ void TwoPassStabilizer::reset() StabilizerBase::reset(); frameCount_ = 0; isPrePassDone_ = false; + doWobbleSuppression_ = false; + motions2_.clear(); suppressedFrame_ = Mat(); } @@ -333,10 +335,20 @@ void TwoPassStabilizer::runPrePassIfNecessary() Mat prevFrame, frame; + WobbleSuppressorBase *wobbleSuppressor = static_cast(wobbleSuppressor_); + doWobbleSuppression_ = dynamic_cast(wobbleSuppressor) == 0; + while (!(frame = frameSource_->nextFrame()).empty()) { if (frameCount_ > 0) + { motions_.push_back(motionEstimator_->estimate(prevFrame, frame)); + if (doWobbleSuppression_) + { + motions2_.push_back( + wobbleSuppressor_->motionEstimator()->estimate(prevFrame, frame)); + } + } else { frameSize_ = frame.size(); @@ -386,10 +398,15 @@ void TwoPassStabilizer::setUp(const Mat &firstFrame) for (int i = -radius_; i <= 0; ++i) at(i, frames_) = firstFrame; - wobbleSuppressor_->setFrames(frames_); - wobbleSuppressor_->setMotions(motions_); - wobbleSuppressor_->setStabilizedFrames(stabilizedFrames_); - wobbleSuppressor_->setStabilizationMotions(stabilizationMotions_); + WobbleSuppressorBase *wobbleSuppressor = static_cast(wobbleSuppressor_); + doWobbleSuppression_ = dynamic_cast(wobbleSuppressor) == 0; + if (doWobbleSuppression_) + { + wobbleSuppressor_->setFrameCount(frameCount_); + wobbleSuppressor_->setMotions(motions_); + wobbleSuppressor_->setMotions2(motions2_); + wobbleSuppressor_->setStabilizationMotions(stabilizationMotions_); + } StabilizerBase::setUp(firstFrame); } @@ -407,9 +424,9 @@ Mat TwoPassStabilizer::estimateStabilizationMotion() } -Mat TwoPassStabilizer::postProcessFrame(const Mat &/*frame*/) +Mat TwoPassStabilizer::postProcessFrame(const Mat &frame) { - wobbleSuppressor_->suppress(curStabilizedPos_, suppressedFrame_); + wobbleSuppressor_->suppress(curStabilizedPos_, frame, suppressedFrame_); return StabilizerBase::postProcessFrame(suppressedFrame_); } diff --git a/modules/videostab/src/wobble_suppression.cpp b/modules/videostab/src/wobble_suppression.cpp index 0895dfc..80c4800 100644 --- a/modules/videostab/src/wobble_suppression.cpp +++ b/modules/videostab/src/wobble_suppression.cpp @@ -51,9 +51,30 @@ namespace cv namespace videostab { -void NullWobbleSuppressor::suppress(int idx, Mat &result) +WobbleSuppressorBase::WobbleSuppressorBase() + : motions_(0), stabilizationMotions_(0) { - result = at(idx, *stabilizedFrames_); + PyrLkRobustMotionEstimator *est = new PyrLkRobustMotionEstimator(); + est->setMotionModel(HOMOGRAPHY); + est->setRansacParams(RansacParams::homography2dMotionStd()); + setMotionEstimator(est); +} + + +void NullWobbleSuppressor::suppress(int /*idx*/, const Mat &frame, Mat &result) +{ + result = frame; +} + + +void MoreAccurateMotionWobbleSuppressor::suppress(int idx, const Mat &frame, Mat &result) +{ + CV_Assert(motions_ && stabilizationMotions_); + + // TODO implement + CV_Error(CV_StsNotImplemented, "MoreAccurateMotionWobbleSuppressor"); + + result = frame; } } // namespace videostab diff --git a/samples/cpp/videostab.cpp b/samples/cpp/videostab.cpp index a7f41bb..a3e1f04 100644 --- a/samples/cpp/videostab.cpp +++ b/samples/cpp/videostab.cpp @@ -29,43 +29,6 @@ void run(); void saveMotionsIfNecessary(); void printHelp(); -class GlobalMotionReader : public GlobalMotionEstimatorBase -{ -public: - GlobalMotionReader(string path) - { - ifstream f(path.c_str()); - if (!f.is_open()) - throw runtime_error("can't open motions file: " + path); - int size; f >> size; - motions_.resize(size); - for (int i = 0; i < size; ++i) - { - Mat_ M(3, 3); - for (int l = 0; l < 3; ++l) - for (int s = 0; s < 3; ++s) - f >> M(l,s); - motions_[i] = M; - } - pos_ = 0; - } - - virtual Mat estimate(const Mat &/*frame0*/, const Mat &/*frame1*/) - { - if (pos_ >= motions_.size()) - { - stringstream text; - text << "can't load motion between frames " << pos_ << " and " << pos_+1; - throw runtime_error(text.str()); - } - return motions_[pos_++]; - } - -private: - vector motions_; - size_t pos_; -}; - void run() { @@ -76,8 +39,6 @@ void run() while (!(stabilizedFrame = stabilizedFrames->nextFrame()).empty()) { nframes++; - if (!saveMotionsPath.empty()) - saveMotionsIfNecessary(); if (!outputPath.empty()) { if (!writer.isOpened()) @@ -99,33 +60,6 @@ void run() } -void saveMotionsIfNecessary() -{ - static bool areMotionsSaved = false; - if (!areMotionsSaved) - { - IFrameSource *frameSource = static_cast(stabilizedFrames); - TwoPassStabilizer *twoPassStabilizer = dynamic_cast(frameSource); - if (twoPassStabilizer) - { - ofstream f(saveMotionsPath.c_str()); - const vector &motions = twoPassStabilizer->motions(); - f << motions.size() << endl; - for (size_t i = 0; i < motions.size(); ++i) - { - Mat_ M = motions[i]; - for (int l = 0, k = 0; l < 3; ++l) - for (int s = 0; s < 3; ++s, ++k) - f << M(l,s) << " "; - f << endl; - } - } - areMotionsSaved = true; - cout << "motions are saved"; - } -} - - void printHelp() { cout << "OpenCV video stabilizer.\n" @@ -173,6 +107,8 @@ void printHelp() " --color-inpaint-radius=\n" " Set color inpainting radius (for ns and telea options only).\n" " The default is 2.0\n\n" + " --wobble-suppress=(yes|no)\n" + " Perform wobble suppression. The default is no.\n\n" " -o, --output=(no|)\n" " Set output file path explicitely. The default is stabilized.avi.\n" " --fps=(|auto)\n" @@ -210,6 +146,7 @@ int main(int argc, const char **argv) "{ | dist-thresh | 5.0 | }" "{ | color-inpaint | no | }" "{ | color-inpaint-radius | 2 | }" + "{ | wobble-suppress | no | }" "{ o | output | stabilized.avi | }" "{ | fps | auto | }" "{ q | quiet | false | }" @@ -226,7 +163,9 @@ int main(int argc, const char **argv) StabilizerBase *stabilizer; - bool isTwoPass = arg("est-trim") == "yes" || arg("save-motions") != "no"; + bool isTwoPass = + arg("est-trim") == "yes" || arg("wobble-suppress") == "yes"; + if (isTwoPass) { TwoPassStabilizer *twoPassStabilizer = new TwoPassStabilizer(); @@ -236,6 +175,19 @@ int main(int argc, const char **argv) twoPassStabilizer->setMotionStabilizer(new GaussianMotionFilter(argi("radius"))); else twoPassStabilizer->setMotionStabilizer(new GaussianMotionFilter(argi("radius"), argf("stdev"))); + if (arg("wobble-suppress") == "yes") + { + twoPassStabilizer->setWobbleSuppressor(new MoreAccurateMotionWobbleSuppressor()); + if (arg("load-motions") != "no") + twoPassStabilizer->wobbleSuppressor()->setMotionEstimator( + new FromFileMotionReader("motions2." + arg("load-motions"))); + if (arg("save-motions") != "no") + { + Ptr est = twoPassStabilizer->wobbleSuppressor()->motionEstimator(); + twoPassStabilizer->wobbleSuppressor()->setMotionEstimator( + new ToFileMotionWriter("motions2." + arg("save-motions"), est)); + } + } } else { @@ -289,10 +241,11 @@ int main(int argc, const char **argv) stabilizer->setMotionEstimator(est_); } else - stabilizer->setMotionEstimator(new GlobalMotionReader(arg("load-motions"))); + stabilizer->setMotionEstimator(new FromFileMotionReader("motions." + arg("load-motions"))); if (arg("save-motions") != "no") - saveMotionsPath = arg("save-motions"); + stabilizer->setMotionEstimator( + new ToFileMotionWriter("motions." + arg("save-motions"), stabilizer->motionEstimator())); stabilizer->setRadius(argi("radius")); if (arg("deblur") == "yes") @@ -342,9 +295,7 @@ int main(int argc, const char **argv) { inpainters->setRadius(argi("radius")); stabilizer->setInpainter(inpainters_); - } - - stabilizer->setLog(new LogToStdout()); + } if (arg("output") != "no") outputPath = arg("output"); -- 2.7.4