modules/videostab/src/stabilizer.cpp

   1 /*M///////////////////////////////////////////////////////////////////////////////////////
   2 //
   3 //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
   4 //
   5 //  By downloading, copying, installing or using the software you agree to this license.
   6 //  If you do not agree to this license, do not download, install,
   7 //  copy or use the software.
   8 //
   9 //
  10 //                           License Agreement
  11 //                For Open Source Computer Vision Library
  12 //
  13 // Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
  14 // Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved.
  15 // Third party copyrights are property of their respective owners.
  16 //
  17 // Redistribution and use in source and binary forms, with or without modification,
  18 // are permitted provided that the following conditions are met:
  19 //
  20 //   * Redistribution's of source code must retain the above copyright notice,
  21 //     this list of conditions and the following disclaimer.
  22 //
  23 //   * Redistribution's in binary form must reproduce the above copyright notice,
  24 //     this list of conditions and the following disclaimer in the documentation
  25 //     and/or other materials provided with the distribution.
  26 //
  27 //   * The name of the copyright holders may not be used to endorse or promote products
  28 //     derived from this software without specific prior written permission.
  29 //
  30 // This software is provided by the copyright holders and contributors "as is" and
  31 // any express or implied warranties, including, but not limited to, the implied
  32 // warranties of merchantability and fitness for a particular purpose are disclaimed.
  33 // In no event shall the Intel Corporation or contributors be liable for any direct,
  34 // indirect, incidental, special, exemplary, or consequential damages
  35 // (including, but not limited to, procurement of substitute goods or services;
  36 // loss of use, data, or profits; or business interruption) however caused
  37 // and on any theory of liability, whether in contract, strict liability,
  38 // or tort (including negligence or otherwise) arising in any way out of
  39 // the use of this software, even if advised of the possibility of such damage.
  40 //
  41 //M*/
  42
  43 #include "precomp.hpp"
  44 #include "opencv2/videostab/stabilizer.hpp"
  45 #include "opencv2/videostab/ring_buffer.hpp"
  46
  47 using namespace std;
  48
  49 namespace cv
  50 {
  51 namespace videostab
  52 {
  53
  54 StabilizerBase::StabilizerBase()
  55 {
  56     setLog(new NullLog());
  57     setFrameSource(new NullFrameSource());
  58     setMotionEstimator(new PyrLkRobustMotionEstimator());
  59     setDeblurer(new NullDeblurer());
  60     setInpainter(new NullInpainter());
  61     setRadius(15);
  62     setTrimRatio(0);
  63     setCorrectionForInclusion(false);
  64     setBorderMode(BORDER_REPLICATE);
  65 }
  66
  67
  68 void StabilizerBase::reset()
  69 {
  70     frameSize_ = Size(0, 0);
  71     frameMask_ = Mat();
  72     curPos_ = -1;
  73     curStabilizedPos_ = -1;
  74     doDeblurring_ = false;
  75     preProcessedFrame_ = Mat();
  76     doInpainting_ = false;
  77     inpaintingMask_ = Mat();
  78     frames_.clear();
  79     motions_.clear();
  80     blurrinessRates_.clear();
  81     stabilizedFrames_.clear();
  82     stabilizedMasks_.clear();
  83     stabilizationMotions_.clear();
  84 }
  85
  86
  87 Mat StabilizerBase::nextStabilizedFrame()
  88 {
  89     // check if we've processed all frames already
  90     if (curStabilizedPos_ == curPos_ && curStabilizedPos_ != -1)
  91         return Mat();
  92
  93     bool processed;
  94     do processed = doOneIteration();
  95     while (processed && curStabilizedPos_ == -1);
  96
  97     // check if frame source is empty
  98     if (curStabilizedPos_ == -1)
  99         return Mat();
 100
 101     return postProcessFrame(at(curStabilizedPos_, stabilizedFrames_));
 102 }
 103
 104
 105 bool StabilizerBase::doOneIteration()
 106 {
 107     Mat frame = frameSource_->nextFrame();
 108     if (!frame.empty())
 109     {
 110         curPos_++;
 111
 112         if (curPos_ > 0)
 113         {
 114             at(curPos_, frames_) = frame;
 115
 116             if (doDeblurring_)
 117                 at(curPos_, blurrinessRates_) = calcBlurriness(frame);
 118
 119             at(curPos_ - 1, motions_) = estimateMotion();
 120
 121             if (curPos_ >= radius_)
 122             {
 123                 curStabilizedPos_ = curPos_ - radius_;
 124                 stabilizeFrame();
 125             }
 126         }
 127         else
 128             setUp(frame);
 129
 130         log_->print(".");
 131         return true;
 132     }
 133
 134     if (curStabilizedPos_ < curPos_)
 135     {
 136         curStabilizedPos_++;
 137         at(curStabilizedPos_ + radius_, frames_) = at(curPos_, frames_);
 138         at(curStabilizedPos_ + radius_ - 1, motions_) = Mat::eye(3, 3, CV_32F);
 139         stabilizeFrame();
 140
 141         log_->print(".");
 142         return true;
 143     }
 144
 145     return false;
 146 }
 147
 148
 149 void StabilizerBase::setUp(const Mat &firstFrame)
 150 {
 151     InpainterBase *inpainter = static_cast<InpainterBase*>(inpainter_);
 152     doInpainting_ = dynamic_cast<NullInpainter*>(inpainter) == 0;
 153     if (doInpainting_)
 154     {
 155         inpainter_->setMotionModel(motionEstimator_->motionModel());
 156         inpainter_->setFrames(frames_);
 157         inpainter_->setMotions(motions_);
 158         inpainter_->setStabilizedFrames(stabilizedFrames_);
 159         inpainter_->setStabilizationMotions(stabilizationMotions_);
 160     }
 161
 162     DeblurerBase *deblurer = static_cast<DeblurerBase*>(deblurer_);
 163     doDeblurring_ = dynamic_cast<NullDeblurer*>(deblurer) == 0;
 164     if (doDeblurring_)
 165     {
 166         blurrinessRates_.resize(2*radius_ + 1);
 167         float blurriness = calcBlurriness(firstFrame);
 168         for (int i  = -radius_; i <= 0; ++i)
 169             at(i, blurrinessRates_) = blurriness;
 170         deblurer_->setFrames(frames_);
 171         deblurer_->setMotions(motions_);
 172         deblurer_->setBlurrinessRates(blurrinessRates_);
 173     }
 174
 175     log_->print("processing frames");
 176 }
 177
 178
 179 void StabilizerBase::stabilizeFrame()
 180 {
 181     Mat stabilizationMotion = estimateStabilizationMotion();
 182     if (doCorrectionForInclusion_)
 183         stabilizationMotion = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_);
 184
 185     at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion;
 186
 187     if (doDeblurring_)
 188     {
 189         at(curStabilizedPos_, frames_).copyTo(preProcessedFrame_);
 190         deblurer_->deblur(curStabilizedPos_, preProcessedFrame_);
 191     }
 192     else
 193         preProcessedFrame_ = at(curStabilizedPos_, frames_);
 194
 195     // apply stabilization transformation
 196
 197     if (motionEstimator_->motionModel() != HOMOGRAPHY)
 198         warpAffine(
 199                 preProcessedFrame_, at(curStabilizedPos_, stabilizedFrames_),
 200                 stabilizationMotion(Rect(0,0,3,2)), frameSize_, INTER_LINEAR, borderMode_);
 201     else
 202         warpPerspective(
 203                 preProcessedFrame_, at(curStabilizedPos_, stabilizedFrames_),
 204                 stabilizationMotion, frameSize_, INTER_LINEAR, borderMode_);
 205
 206     if (doInpainting_)
 207     {
 208         if (motionEstimator_->motionModel() != HOMOGRAPHY)
 209             warpAffine(
 210                     frameMask_, at(curStabilizedPos_, stabilizedMasks_),
 211                     stabilizationMotion(Rect(0,0,3,2)), frameSize_, INTER_NEAREST);
 212         else
 213             warpPerspective(
 214                     frameMask_, at(curStabilizedPos_, stabilizedMasks_),
 215                     stabilizationMotion, frameSize_, INTER_NEAREST);
 216
 217         erode(at(curStabilizedPos_, stabilizedMasks_), at(curStabilizedPos_, stabilizedMasks_),
 218               Mat());
 219
 220         at(curStabilizedPos_, stabilizedMasks_).copyTo(inpaintingMask_);
 221
 222         inpainter_->inpaint(
 223             curStabilizedPos_, at(curStabilizedPos_, stabilizedFrames_), inpaintingMask_);
 224     }
 225 }
 226
 227
 228 Mat StabilizerBase::postProcessFrame(const Mat &frame)
 229 {
 230     // trim frame
 231     int dx = static_cast<int>(floor(trimRatio_ * frame.cols));
 232     int dy = static_cast<int>(floor(trimRatio_ * frame.rows));
 233     return frame(Rect(dx, dy, frame.cols - 2*dx, frame.rows - 2*dy));
 234 }
 235
 236
 237 OnePassStabilizer::OnePassStabilizer()
 238 {
 239     setMotionFilter(new GaussianMotionFilter());
 240     reset();
 241 }
 242
 243
 244 void OnePassStabilizer::reset()
 245 {
 246     StabilizerBase::reset();
 247 }
 248
 249
 250 void OnePassStabilizer::setUp(const Mat &firstFrame)
 251 {
 252     frameSize_ = firstFrame.size();
 253     frameMask_.create(frameSize_, CV_8U);
 254     frameMask_.setTo(255);
 255
 256     int cacheSize = 2*radius_ + 1;
 257     frames_.resize(cacheSize);
 258     stabilizedFrames_.resize(cacheSize);
 259     stabilizedMasks_.resize(cacheSize);
 260     motions_.resize(cacheSize);
 261     stabilizationMotions_.resize(cacheSize);
 262
 263     for (int i = -radius_; i < 0; ++i)
 264     {
 265         at(i, motions_) = Mat::eye(3, 3, CV_32F);
 266         at(i, frames_) = firstFrame;
 267     }
 268
 269     at(0, frames_) = firstFrame;
 270
 271     StabilizerBase::setUp(firstFrame);
 272 }
 273
 274
 275 Mat OnePassStabilizer::estimateMotion()
 276 {
 277     return motionEstimator_->estimate(at(curPos_ - 1, frames_), at(curPos_, frames_));
 278 }
 279
 280
 281 Mat OnePassStabilizer::estimateStabilizationMotion()
 282 {
 283     return motionFilter_->stabilize(curStabilizedPos_, motions_, make_pair(0, curPos_));
 284 }
 285
 286
 287 Mat OnePassStabilizer::postProcessFrame(const Mat &frame)
 288 {
 289     return StabilizerBase::postProcessFrame(frame);
 290 }
 291
 292
 293 TwoPassStabilizer::TwoPassStabilizer()
 294 {
 295     setMotionStabilizer(new GaussianMotionFilter());
 296     setWobbleSuppressor(new NullWobbleSuppressor());
 297     setEstimateTrimRatio(false);
 298     reset();
 299 }
 300
 301
 302 void TwoPassStabilizer::reset()
 303 {
 304     StabilizerBase::reset();
 305     frameCount_ = 0;
 306     isPrePassDone_ = false;
 307     suppressedFrame_ = Mat();
 308 }
 309
 310
 311 Mat TwoPassStabilizer::nextFrame()
 312 {
 313     runPrePassIfNecessary();
 314     return StabilizerBase::nextStabilizedFrame();
 315 }
 316
 317
 318 vector<Mat> TwoPassStabilizer::motions() const
 319 {
 320     if (frameCount_ == 0)
 321         return vector<Mat>();
 322     vector<Mat> res(frameCount_ - 1);
 323     copy(motions_.begin(), motions_.begin() + frameCount_ - 1, res.begin());
 324     return res;
 325 }
 326
 327
 328 void TwoPassStabilizer::runPrePassIfNecessary()
 329 {
 330     if (!isPrePassDone_)
 331     {
 332         log_->print("first pass: estimating motions");
 333
 334         Mat prevFrame, frame;
 335
 336         while (!(frame = frameSource_->nextFrame()).empty())
 337         {
 338             if (frameCount_ > 0)
 339                 motions_.push_back(motionEstimator_->estimate(prevFrame, frame));
 340             else
 341             {
 342                 frameSize_ = frame.size();
 343                 frameMask_.create(frameSize_, CV_8U);
 344                 frameMask_.setTo(255);
 345             }
 346
 347             prevFrame = frame;
 348             frameCount_++;
 349
 350             log_->print(".");
 351         }
 352
 353         for (int i = 0; i < radius_; ++i)
 354             motions_.push_back(Mat::eye(3, 3, CV_32F));
 355         log_->print("\n");
 356
 357         stabilizationMotions_.resize(frameCount_);
 358
 359         motionStabilizer_->stabilize(
 360             frameCount_, motions_, make_pair(0, frameCount_ - 1), &stabilizationMotions_[0]);
 361
 362         if (mustEstTrimRatio_)
 363         {
 364             trimRatio_ = 0;
 365             for (int i = 0; i < frameCount_; ++i)
 366             {
 367                 Mat S = stabilizationMotions_[i];
 368                 trimRatio_ = std::max(trimRatio_, estimateOptimalTrimRatio(S, frameSize_));
 369             }
 370             log_->print("estimated trim ratio: %f\n", static_cast<double>(trimRatio_));
 371         }
 372
 373         isPrePassDone_ = true;
 374         frameSource_->reset();
 375     }
 376 }
 377
 378
 379 void TwoPassStabilizer::setUp(const Mat &firstFrame)
 380 {
 381     int cacheSize = 2*radius_ + 1;
 382     frames_.resize(cacheSize);
 383     stabilizedFrames_.resize(cacheSize);
 384     stabilizedMasks_.resize(cacheSize);
 385
 386     for (int i = -radius_; i <= 0; ++i)
 387         at(i, frames_) = firstFrame;
 388
 389     wobbleSuppressor_->setFrames(frames_);
 390     wobbleSuppressor_->setMotions(motions_);
 391     wobbleSuppressor_->setStabilizedFrames(stabilizedFrames_);
 392     wobbleSuppressor_->setStabilizationMotions(stabilizationMotions_);
 393
 394     StabilizerBase::setUp(firstFrame);
 395 }
 396
 397
 398 Mat TwoPassStabilizer::estimateMotion()
 399 {
 400     return motions_[curPos_ - 1].clone();
 401 }
 402
 403
 404 Mat TwoPassStabilizer::estimateStabilizationMotion()
 405 {
 406     return stabilizationMotions_[curStabilizedPos_].clone();
 407 }
 408
 409
 410 Mat TwoPassStabilizer::postProcessFrame(const Mat &/*frame*/)
 411 {
 412     wobbleSuppressor_->suppress(curStabilizedPos_, suppressedFrame_);
 413     return StabilizerBase::postProcessFrame(suppressedFrame_);
 414 }
 415
 416 } // namespace videostab
 417 } // namespace cv