From d9d47553872df363b53e542fb1cd398e4a3c0156 Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Fri, 6 Apr 2012 08:52:31 +0000 Subject: [PATCH] Updated wobble suppression code in videostab module --- .../include/opencv2/videostab/global_motion.hpp | 2 +- .../opencv2/videostab/wobble_suppression.hpp | 9 ++ modules/videostab/src/global_motion.cpp | 15 ++- modules/videostab/src/wobble_suppression.cpp | 56 +++++++- samples/cpp/videostab.cpp | 142 +++++++++++---------- 5 files changed, 151 insertions(+), 73 deletions(-) diff --git a/modules/videostab/include/opencv2/videostab/global_motion.hpp b/modules/videostab/include/opencv2/videostab/global_motion.hpp index 2c46350..c7dc474 100644 --- a/modules/videostab/include/opencv2/videostab/global_motion.hpp +++ b/modules/videostab/include/opencv2/videostab/global_motion.hpp @@ -131,7 +131,7 @@ private: class CV_EXPORTS PyrLkRobustMotionEstimator : public GlobalMotionEstimatorBase { public: - PyrLkRobustMotionEstimator(); + PyrLkRobustMotionEstimator(MotionModel model = AFFINE); void setDetector(Ptr val) { detector_ = val; } Ptr detector() const { return detector_; } diff --git a/modules/videostab/include/opencv2/videostab/wobble_suppression.hpp b/modules/videostab/include/opencv2/videostab/wobble_suppression.hpp index 1e8dd56..d0476a7 100644 --- a/modules/videostab/include/opencv2/videostab/wobble_suppression.hpp +++ b/modules/videostab/include/opencv2/videostab/wobble_suppression.hpp @@ -97,7 +97,16 @@ public: class CV_EXPORTS MoreAccurateMotionWobbleSuppressor : public WobbleSuppressorBase { public: + MoreAccurateMotionWobbleSuppressor(); + + void setPeriod(int val) { period_ = val; } + int period() const { return period_; } + virtual void suppress(int idx, const Mat &frame, Mat &result); + +private: + int period_; + Mat_ mapx_, mapy_; }; } // namespace videostab diff --git a/modules/videostab/src/global_motion.cpp b/modules/videostab/src/global_motion.cpp index ee9e640..cd1fe6c 100644 --- a/modules/videostab/src/global_motion.cpp +++ b/modules/videostab/src/global_motion.cpp @@ -323,12 +323,21 @@ Mat ToFileMotionWriter::estimate(const Mat &frame0, const Mat &frame1) } -PyrLkRobustMotionEstimator::PyrLkRobustMotionEstimator() - : ransacParams_(RansacParams::affine2dMotionStd()) +PyrLkRobustMotionEstimator::PyrLkRobustMotionEstimator(MotionModel model) { setDetector(new GoodFeaturesToTrackDetector()); setOptFlowEstimator(new SparsePyrLkOptFlowEstimator()); - setMotionModel(AFFINE); + setMotionModel(model); + if (model == TRANSLATION) + setRansacParams(RansacParams::translation2dMotionStd()); + else if (model == TRANSLATION_AND_SCALE) + setRansacParams(RansacParams::translationAndScale2dMotionStd()); + else if (model == LINEAR_SIMILARITY) + setRansacParams(RansacParams::linearSimilarity2dMotionStd()); + else if (model == AFFINE) + setRansacParams(RansacParams::affine2dMotionStd()); + else if (model == HOMOGRAPHY) + setRansacParams(RansacParams::homography2dMotionStd()); setMaxRmse(0.5f); setMinInlierRatio(0.1f); } diff --git a/modules/videostab/src/wobble_suppression.cpp b/modules/videostab/src/wobble_suppression.cpp index 80c4800..d997e0c 100644 --- a/modules/videostab/src/wobble_suppression.cpp +++ b/modules/videostab/src/wobble_suppression.cpp @@ -51,8 +51,7 @@ namespace cv namespace videostab { -WobbleSuppressorBase::WobbleSuppressorBase() - : motions_(0), stabilizationMotions_(0) +WobbleSuppressorBase::WobbleSuppressorBase() : motions_(0), stabilizationMotions_(0) { PyrLkRobustMotionEstimator *est = new PyrLkRobustMotionEstimator(); est->setMotionModel(HOMOGRAPHY); @@ -67,14 +66,61 @@ void NullWobbleSuppressor::suppress(int /*idx*/, const Mat &frame, Mat &result) } +MoreAccurateMotionWobbleSuppressor::MoreAccurateMotionWobbleSuppressor() +{ + setPeriod(30); +} + + void MoreAccurateMotionWobbleSuppressor::suppress(int idx, const Mat &frame, Mat &result) { CV_Assert(motions_ && stabilizationMotions_); - // TODO implement - CV_Error(CV_StsNotImplemented, "MoreAccurateMotionWobbleSuppressor"); + if (idx % period_ == 0) + { + result = frame; + return; + } - result = frame; + int k1 = idx / period_ * period_; + int k2 = std::min(k1 + period_, frameCount_ - 1); + + Mat S1 = (*stabilizationMotions_)[idx]; + + Mat_ ML = S1 * getMotion(k1, idx, *motions2_) * getMotion(k1, idx, *motions_).inv() * S1.inv(); + Mat_ MR = S1 * getMotion(idx, k2, *motions2_).inv() * getMotion(idx, k2, *motions_) * S1.inv(); + + mapx_.create(frame.size()); + mapy_.create(frame.size()); + + float xl, yl, zl, wl; + float xr, yr, zr, wr; + + for (int y = 0; y < frame.rows; ++y) + { + for (int x = 0; x < frame.cols; ++x) + { + xl = ML(0,0)*x + ML(0,1)*y + ML(0,2); + yl = ML(1,0)*x + ML(1,1)*y + ML(1,2); + zl = ML(2,0)*x + ML(2,1)*y + ML(2,2); + xl /= zl; yl /= zl; + wl = 1.f / (sqrt(sqr(x - xl) + sqr(y - yl)) + 1e-5f); + + xr = MR(0,0)*x + MR(0,1)*y + MR(0,2); + yr = MR(1,0)*x + MR(1,1)*y + MR(1,2); + zr = MR(2,0)*x + MR(2,1)*y + MR(2,2); + xr /= zr; yr /= zr; + wr = 1.f / (sqrt(sqr(x - xr) + sqr(y - yr)) + 1e-5f); + + mapx_(y,x) = (wl * xl + wr * xr) / (wl + wr); + mapy_(y,x) = (wl * yl + wr * yr) / (wl + wr); + } + } + + if (result.data == frame.data) + result = Mat(frame.size(), frame.type()); + + remap(frame, result, mapx_, mapy_, INTER_LINEAR); } } // namespace videostab diff --git a/samples/cpp/videostab.cpp b/samples/cpp/videostab.cpp index a3e1f04..bd0fc36f 100644 --- a/samples/cpp/videostab.cpp +++ b/samples/cpp/videostab.cpp @@ -72,52 +72,60 @@ void printHelp() " --min-inlier-ratio=\n" " Minimum inlier ratio to decide if estimated motion is OK. The default is 0.1,\n" " but you may want to increase it.\n\n" - " --save-motions=(|no)\n" + " -sm, --save-motions=(|no)\n" " Save estimated motions into file. The default is no.\n" - " --load-motions=(|no)\n" + " -lm, --load-motions=(|no)\n" " Load motions from file. The default is no.\n\n" " -r, --radius=\n" " Set sliding window radius. The default is 15.\n" " --stdev=(|auto)\n" - " Set smoothing weights standard deviation. The default is sqrt(radius),\n" - " i.e. auto.\n\n" + " Set smoothing weights standard deviation. The default is auto\n" + " (i.e. sqrt(radius)).\n" " --deblur=(yes|no)\n" " Do deblurring.\n" " --deblur-sens=\n" " Set deblurring sensitivity (from 0 to +inf). The default is 0.1.\n\n" " -t, --trim-ratio=\n" " Set trimming ratio (from 0 to 0.5). The default is 0.1.\n" - " --est-trim=(yes|no)\n" - " Estimate trim ratio automatically. The default is yes (that leads to two passes,\n" - " you can turn it off if you want to use one pass only).\n" - " --incl-constr=(yes|no)\n" + " -et, --est-trim=(yes|no)\n" + " Estimate trim ratio automatically. The default is yes.\n" + " -ic, --incl-constr=(yes|no)\n" " Ensure the inclusion constraint is always satisfied. The default is no.\n\n" - " --border-mode=(replicate|reflect|const)\n" + " -bm, --border-mode=(replicate|reflect|const)\n" " Set border extrapolation mode. The default is replicate.\n\n" " --mosaic=(yes|no)\n" " Do consistent mosaicing. The default is no.\n" " --mosaic-stdev=\n" " Consistent mosaicing stdev threshold. The default is 10.0.\n\n" - " --motion-inpaint=(yes|no)\n" + " -mi, --motion-inpaint=(yes|no)\n" " Do motion inpainting (requires GPU support). The default is no.\n" - " --dist-thresh=\n" + " --mi-dist-thresh=\n" " Estimated flow distance threshold for motion inpainting. The default is 5.0.\n\n" - " --color-inpaint=(no|average|ns|telea)\n" + " -ci, --color-inpaint=(no|average|ns|telea)\n" " Do color inpainting. The defailt is no.\n" - " --color-inpaint-radius=\n" + " --ci-radius=\n" " Set color inpainting radius (for ns and telea options only).\n" " The default is 2.0\n\n" - " --wobble-suppress=(yes|no)\n" - " Perform wobble suppression. The default is no.\n\n" + " -ws, --wobble-suppress=(yes|no)\n" + " Perform wobble suppression. The default is no.\n" + " --ws-period=\n" + " Set wobble suppression period. The default is 30.\n" + " --ws-model=(transl|transl_and_scale|linear_sim|affine|homography)\n" + " Set wobble suppression motion model (must have more DOF than motion \n" + " estimation model). The default is homography.\n" + " -sm2, --save-motions2=(|no)\n" + " Save motions estimated for wobble suppression. The default is no.\n" + " -lm2, --load-motions2=(|no)\n" + " Load motions for wobble suppression from file. The default is no.\n\n" " -o, --output=(no|)\n" " Set output file path explicitely. The default is stabilized.avi.\n" - " --fps=(|auto)\n" - " Set output video FPS explicitely. By default the source FPS is used.\n" + " --fps=(|auto)\n" + " Set output video FPS explicitely. By default the source FPS is used (auto).\n" " -q, --quiet\n" " Don't show output video frames.\n\n" " -h, --help\n" - " Print help.\n" - "\n"; + " Print help.\n\n" + "Note: some argument configurations lead to two passes, some to single pass.\n\n"; } @@ -130,23 +138,27 @@ int main(int argc, const char **argv) "{ m | model | affine| }" "{ | min-inlier-ratio | 0.1 | }" "{ | outlier-ratio | 0.5 | }" - "{ | save-motions | no | }" - "{ | load-motions | no | }" + "{ sm | save-motions | no | }" + "{ lm | load-motions | no | }" "{ r | radius | 15 | }" "{ | stdev | auto | }" "{ | deblur | no | }" "{ | deblur-sens | 0.1 | }" - "{ | est-trim | yes | }" + "{ et | est-trim | yes | }" "{ t | trim-ratio | 0.1 | }" - "{ | incl-constr | no | }" - "{ | border-mode | replicate | }" + "{ ic | incl-constr | no | }" + "{ bm | border-mode | replicate | }" "{ | mosaic | no | }" - "{ | mosaic-stdev | 10.0 | }" - "{ | motion-inpaint | no | }" - "{ | dist-thresh | 5.0 | }" - "{ | color-inpaint | no | }" - "{ | color-inpaint-radius | 2 | }" - "{ | wobble-suppress | no | }" + "{ ms | mosaic-stdev | 10.0 | }" + "{ mi | motion-inpaint | no | }" + "{ | mi-dist-thresh | 5.0 | }" + "{ ci | color-inpaint | no | }" + "{ | ci-radius | 2 | }" + "{ ws | wobble-suppress | no | }" + "{ | ws-period | 30 | }" + "{ | ws-model | homography | }" + "{ sm2 | save-motions2 | no | }" + "{ lm2 | load-motions2 | no | }" "{ o | output | stabilized.avi | }" "{ | fps | auto | }" "{ q | quiet | false | }" @@ -177,15 +189,27 @@ int main(int argc, const char **argv) twoPassStabilizer->setMotionStabilizer(new GaussianMotionFilter(argi("radius"), argf("stdev"))); if (arg("wobble-suppress") == "yes") { - twoPassStabilizer->setWobbleSuppressor(new MoreAccurateMotionWobbleSuppressor()); - if (arg("load-motions") != "no") - twoPassStabilizer->wobbleSuppressor()->setMotionEstimator( - new FromFileMotionReader("motions2." + arg("load-motions"))); - if (arg("save-motions") != "no") + MoreAccurateMotionWobbleSuppressor *ws = new MoreAccurateMotionWobbleSuppressor(); + twoPassStabilizer->setWobbleSuppressor(ws); + ws->setPeriod(argi("ws-period")); + if (arg("ws-model") == "transl") + ws->setMotionEstimator(new PyrLkRobustMotionEstimator(TRANSLATION)); + else if (arg("ws-model") == "transl_and_scale") + ws->setMotionEstimator(new PyrLkRobustMotionEstimator(TRANSLATION_AND_SCALE)); + else if (arg("ws-model") == "linear_sim") + ws->setMotionEstimator(new PyrLkRobustMotionEstimator(LINEAR_SIMILARITY)); + else if (arg("ws-model") == "affine") + ws->setMotionEstimator(new PyrLkRobustMotionEstimator(AFFINE)); + else if (arg("ws-model") == "homography") + ws->setMotionEstimator(new PyrLkRobustMotionEstimator(HOMOGRAPHY)); + else + throw runtime_error("unknown wobble suppression motion model: " + arg("ws-model")); + if (arg("load-motions2") != "no") + ws->setMotionEstimator(new FromFileMotionReader(arg("load-motions2"))); + if (arg("save-motions2") != "no") { - Ptr est = twoPassStabilizer->wobbleSuppressor()->motionEstimator(); - twoPassStabilizer->wobbleSuppressor()->setMotionEstimator( - new ToFileMotionWriter("motions2." + arg("save-motions"), est)); + Ptr est = ws->motionEstimator(); + ws->setMotionEstimator(new ToFileMotionWriter(arg("save-motions2"), est)); } } } @@ -209,43 +233,33 @@ int main(int argc, const char **argv) stabilizer->setFrameSource(source); if (arg("load-motions") == "no") - { - RansacParams ransac; - PyrLkRobustMotionEstimator *est = new PyrLkRobustMotionEstimator(); - Ptr est_(est); + { + PyrLkRobustMotionEstimator *est = 0; if (arg("model") == "transl") - { - est->setMotionModel(TRANSLATION); - ransac = RansacParams::translation2dMotionStd(); - } + est = new PyrLkRobustMotionEstimator(TRANSLATION); else if (arg("model") == "transl_and_scale") - { - est->setMotionModel(TRANSLATION_AND_SCALE); - ransac = RansacParams::translationAndScale2dMotionStd(); - } + est = new PyrLkRobustMotionEstimator(TRANSLATION_AND_SCALE); else if (arg("model") == "linear_sim") - { - est->setMotionModel(LINEAR_SIMILARITY); - ransac = RansacParams::linearSimilarity2dMotionStd(); - } + est = new PyrLkRobustMotionEstimator(LINEAR_SIMILARITY); else if (arg("model") == "affine") - { - est->setMotionModel(AFFINE); - ransac = RansacParams::affine2dMotionStd(); - } + est = new PyrLkRobustMotionEstimator(AFFINE); else + { + delete est; throw runtime_error("unknown motion model: " + arg("model")); + } + RansacParams ransac = est->ransacParams(); ransac.eps = argf("outlier-ratio"); est->setRansacParams(ransac); est->setMinInlierRatio(argf("min-inlier-ratio")); - stabilizer->setMotionEstimator(est_); + stabilizer->setMotionEstimator(est); } else - stabilizer->setMotionEstimator(new FromFileMotionReader("motions." + arg("load-motions"))); + stabilizer->setMotionEstimator(new FromFileMotionReader(arg("load-motions"))); if (arg("save-motions") != "no") stabilizer->setMotionEstimator( - new ToFileMotionWriter("motions." + arg("save-motions"), stabilizer->motionEstimator())); + new ToFileMotionWriter(arg("save-motions"), stabilizer->motionEstimator())); stabilizer->setRadius(argi("radius")); if (arg("deblur") == "yes") @@ -280,15 +294,15 @@ int main(int argc, const char **argv) if (arg("motion-inpaint") == "yes") { MotionInpainter *inp = new MotionInpainter(); - inp->setDistThreshold(argf("dist-thresh")); + inp->setDistThreshold(argf("mi-dist-thresh")); inpainters->pushBack(inp); } if (arg("color-inpaint") == "average") inpainters->pushBack(new ColorAverageInpainter()); else if (arg("color-inpaint") == "ns") - inpainters->pushBack(new ColorInpainter(INPAINT_NS, argd("color-inpaint-radius"))); + inpainters->pushBack(new ColorInpainter(INPAINT_NS, argd("ci-radius"))); else if (arg("color-inpaint") == "telea") - inpainters->pushBack(new ColorInpainter(INPAINT_TELEA, argd("color-inpaint-radius"))); + inpainters->pushBack(new ColorInpainter(INPAINT_TELEA, argd("ci-radius"))); else if (arg("color-inpaint") != "no") throw runtime_error("unknown color inpainting method: " + arg("color-inpaint")); if (!inpainters->empty()) -- 2.7.4