From 7a77cf584a64a5a929c1928bc2da3e442d2be083 Mon Sep 17 00:00:00 2001 From: Petr Glotov Date: Fri, 16 May 2014 22:52:07 -0700 Subject: [PATCH] added timelapse --- .../include/opencv2/stitching/detail/matchers.hpp | 14 +++ .../opencv2/stitching/detail/timelapsers.hpp | 86 +++++++++++++++++ .../include/opencv2/stitching/detail/util.hpp | 1 + modules/stitching/src/matchers.cpp | 34 +++++++ modules/stitching/src/precomp.hpp | 1 + modules/stitching/src/timelapsers.cpp | 107 +++++++++++++++++++++ modules/stitching/src/util.cpp | 15 +++ samples/cpp/stitching_detailed.cpp | 74 ++++++++++++-- 8 files changed, 322 insertions(+), 10 deletions(-) create mode 100644 modules/stitching/include/opencv2/stitching/detail/timelapsers.hpp create mode 100644 modules/stitching/src/timelapsers.cpp diff --git a/modules/stitching/include/opencv2/stitching/detail/matchers.hpp b/modules/stitching/include/opencv2/stitching/detail/matchers.hpp index 8c3284a..d87a1ff 100644 --- a/modules/stitching/include/opencv2/stitching/detail/matchers.hpp +++ b/modules/stitching/include/opencv2/stitching/detail/matchers.hpp @@ -183,6 +183,20 @@ protected: Ptr impl_; }; +class CV_EXPORTS BestOf2NearestRangeMatcher : public BestOf2NearestMatcher +{ +public: + BestOf2NearestRangeMatcher(int range_width = 5, bool try_use_gpu = false, float match_conf = 0.3f, + int num_matches_thresh1 = 6, int num_matches_thresh2 = 6); + + void operator ()(const std::vector &features, std::vector &pairwise_matches, + const cv::UMat &mask = cv::UMat()); + + +protected: + int range_width_; +}; + } // namespace detail } // namespace cv diff --git a/modules/stitching/include/opencv2/stitching/detail/timelapsers.hpp b/modules/stitching/include/opencv2/stitching/detail/timelapsers.hpp new file mode 100644 index 0000000..f881a9b --- /dev/null +++ b/modules/stitching/include/opencv2/stitching/detail/timelapsers.hpp @@ -0,0 +1,86 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. +// Copyright (C) 2009, Willow Garage Inc., all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + + +#ifndef __OPENCV_STITCHING_TIMELAPSERS_HPP__ +#define __OPENCV_STITCHING_TIMELAPSERS_HPP__ + +#include "opencv2/core.hpp" + +namespace cv { +namespace detail { + +// Base Timelapser class, takes a sequence of images, applies appropriate shift, stores result in dst_. + +class CV_EXPORTS Timelapser +{ +public: + + enum {AS_IS, CROP}; + + virtual ~Timelapser() {} + + static Ptr createDefault(int type); + + virtual void initialize(const std::vector &corners, const std::vector &sizes); + virtual void process(InputArray img, InputArray mask, Point tl); + virtual const UMat& getDst() {return dst_;} + +protected: + + virtual bool test_point(Point pt); + + UMat dst_; + Rect dst_roi_; +}; + + +class CV_EXPORTS TimelapserCrop : public Timelapser +{ +public: + virtual void initialize(const std::vector &corners, const std::vector &sizes); +}; + +} // namespace detail +} // namespace cv + +#endif // __OPENCV_STITCHING_TIMELAPSERS_HPP__ diff --git a/modules/stitching/include/opencv2/stitching/detail/util.hpp b/modules/stitching/include/opencv2/stitching/detail/util.hpp index 6b1c5f3..23c413d 100644 --- a/modules/stitching/include/opencv2/stitching/detail/util.hpp +++ b/modules/stitching/include/opencv2/stitching/detail/util.hpp @@ -148,6 +148,7 @@ private: CV_EXPORTS bool overlapRoi(Point tl1, Point tl2, Size sz1, Size sz2, Rect &roi); CV_EXPORTS Rect resultRoi(const std::vector &corners, const std::vector &images); CV_EXPORTS Rect resultRoi(const std::vector &corners, const std::vector &sizes); +CV_EXPORTS Rect resultRoiIntersection(const std::vector &corners, const std::vector &sizes); CV_EXPORTS Point resultTl(const std::vector &corners); // Returns random 'count' element subset of the {0,1,...,size-1} set diff --git a/modules/stitching/src/matchers.cpp b/modules/stitching/src/matchers.cpp index c303c4a..a512e35 100644 --- a/modules/stitching/src/matchers.cpp +++ b/modules/stitching/src/matchers.cpp @@ -646,5 +646,39 @@ void BestOf2NearestMatcher::collectGarbage() impl_->collectGarbage(); } + +BestOf2NearestRangeMatcher::BestOf2NearestRangeMatcher(int range_width, bool try_use_gpu, float match_conf, int num_matches_thresh1, int num_matches_thresh2): BestOf2NearestMatcher(try_use_gpu, match_conf, num_matches_thresh1, num_matches_thresh2) +{ + range_width_ = range_width; +} + + +void BestOf2NearestRangeMatcher::operator ()(const std::vector &features, std::vector &pairwise_matches, + const UMat &mask) +{ + const int num_images = static_cast(features.size()); + + CV_Assert(mask.empty() || (mask.type() == CV_8U && mask.cols == num_images && mask.rows)); + Mat_ mask_(mask.getMat(ACCESS_READ)); + if (mask_.empty()) + mask_ = Mat::ones(num_images, num_images, CV_8U); + + std::vector > near_pairs; + for (int i = 0; i < num_images - 1; ++i) + for (int j = i + 1; j < std::min(num_images, i + range_width_); ++j) + if (features[i].keypoints.size() > 0 && features[j].keypoints.size() > 0 && mask_(i, j)) + near_pairs.push_back(std::make_pair(i, j)); + + pairwise_matches.resize(num_images * num_images); + MatchPairsBody body(*this, features, pairwise_matches, near_pairs); + + if (is_thread_safe_) + parallel_for_(Range(0, static_cast(near_pairs.size())), body); + else + body(Range(0, static_cast(near_pairs.size()))); + LOGLN_CHAT(""); +} + + } // namespace detail } // namespace cv diff --git a/modules/stitching/src/precomp.hpp b/modules/stitching/src/precomp.hpp index 18ce413..759d036 100644 --- a/modules/stitching/src/precomp.hpp +++ b/modules/stitching/src/precomp.hpp @@ -59,6 +59,7 @@ #include "opencv2/stitching.hpp" #include "opencv2/stitching/detail/autocalib.hpp" #include "opencv2/stitching/detail/blenders.hpp" +#include "opencv2/stitching/detail/timelapsers.hpp" #include "opencv2/stitching/detail/camera.hpp" #include "opencv2/stitching/detail/exposure_compensate.hpp" #include "opencv2/stitching/detail/matchers.hpp" diff --git a/modules/stitching/src/timelapsers.cpp b/modules/stitching/src/timelapsers.cpp new file mode 100644 index 0000000..d78ad86 --- /dev/null +++ b/modules/stitching/src/timelapsers.cpp @@ -0,0 +1,107 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. +// Copyright (C) 2009, Willow Garage Inc., all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include "precomp.hpp" +#include "opencl_kernels.hpp" + +namespace cv { +namespace detail { + +Ptr Timelapser::createDefault(int type) +{ + if (type == AS_IS) + return makePtr(); + if (type == CROP) + return makePtr(); + CV_Error(Error::StsBadArg, "unsupported timelapsing method"); + return Ptr(); +} + + +void Timelapser::initialize(const std::vector &corners, const std::vector &sizes) +{ + dst_roi_ = resultRoi(corners, sizes); + dst_.create(dst_roi_.size(), CV_16SC3); +} + +void Timelapser::process(InputArray _img, InputArray /*_mask*/, Point tl) +{ + dst_.setTo(Scalar::all(0)); + + Mat img = _img.getMat(); + Mat dst = dst_.getMat(ACCESS_RW); + + CV_Assert(img.type() == CV_16SC3); + int dx = tl.x - dst_roi_.x; + int dy = tl.y - dst_roi_.y; + + for (int y = 0; y < img.rows; ++y) + { + const Point3_ *src_row = img.ptr >(y); + + for (int x = 0; x < img.cols; ++x) + { + if (test_point(Point(tl.x + x, tl.y + y))) + { + Point3_ *dst_row = dst.ptr >(dy + y); + dst_row[dx + x] = src_row[x]; + } + } + } +} + + +bool Timelapser::test_point(Point pt) +{ + return dst_roi_.contains(pt); +} + + +void TimelapserCrop::initialize(const std::vector &corners, const std::vector &sizes) +{ + dst_roi_ = resultRoiIntersection(corners, sizes); + dst_.create(dst_roi_.size(), CV_16SC3); +} + + +} // namespace detail +} // namespace cv diff --git a/modules/stitching/src/util.cpp b/modules/stitching/src/util.cpp index 5e026f0..ce36d9d 100644 --- a/modules/stitching/src/util.cpp +++ b/modules/stitching/src/util.cpp @@ -137,6 +137,21 @@ Rect resultRoi(const std::vector &corners, const std::vector &sizes return Rect(tl, br); } +Rect resultRoiIntersection(const std::vector &corners, const std::vector &sizes) +{ + CV_Assert(sizes.size() == corners.size()); + Point tl(std::numeric_limits::min(), std::numeric_limits::min()); + Point br(std::numeric_limits::max(), std::numeric_limits::max()); + for (size_t i = 0; i < corners.size(); ++i) + { + tl.x = std::max(tl.x, corners[i].x); + tl.y = std::max(tl.y, corners[i].y); + br.x = std::min(br.x, corners[i].x + sizes[i].width); + br.y = std::min(br.y, corners[i].y + sizes[i].height); + } + return Rect(tl, br); +} + Point resultTl(const std::vector &corners) { diff --git a/samples/cpp/stitching_detailed.cpp b/samples/cpp/stitching_detailed.cpp index df0a9ab..bc66694 100644 --- a/samples/cpp/stitching_detailed.cpp +++ b/samples/cpp/stitching_detailed.cpp @@ -49,6 +49,7 @@ #include "opencv2/highgui.hpp" #include "opencv2/stitching/detail/autocalib.hpp" #include "opencv2/stitching/detail/blenders.hpp" +#include "opencv2/stitching/detail/timelapsers.hpp" #include "opencv2/stitching/detail/camera.hpp" #include "opencv2/stitching/detail/exposure_compensate.hpp" #include "opencv2/stitching/detail/matchers.hpp" @@ -116,7 +117,9 @@ static void printUsage() " --blend_strength \n" " Blending strength from [0,100] range. The default is 5.\n" " --output \n" - " The default is 'result.jpg'.\n"; + " The default is 'result.jpg'.\n" + " --timelapse (as_is|crop) (range_width)\n" + " Output warped images separately as frames of a time lapse movie, with 'fixed_' prepended to input file names.\n"; } @@ -140,8 +143,12 @@ int expos_comp_type = ExposureCompensator::GAIN_BLOCKS; float match_conf = 0.3f; string seam_find_type = "gc_color"; int blend_type = Blender::MULTI_BAND; +int timelapse_type = Timelapser::AS_IS; float blend_strength = 5; string result_name = "result.jpg"; +bool timelapse = false; +int timelapse_range = 5; + static int parseCmdArgs(int argc, char** argv) { @@ -304,6 +311,24 @@ static int parseCmdArgs(int argc, char** argv) } i++; } + else if (string(argv[i]) == "--timelapse") + { + timelapse = true; + + if (string(argv[i + 1]) == "as_is") + timelapse_type = Timelapser::AS_IS; + else if (string(argv[i + 1]) == "crop") + timelapse_type = Timelapser::CROP; + else + { + cout << "Bad timelapse method\n"; + return -1; + } + i++; + + timelapse_range = atoi(argv[i + 1]); + i++; + } else if (string(argv[i]) == "--blend_strength") { blend_strength = static_cast(atof(argv[i + 1])); @@ -432,9 +457,19 @@ int main(int argc, char* argv[]) t = getTickCount(); #endif vector pairwise_matches; - BestOf2NearestMatcher matcher(try_cuda, match_conf); - matcher(features, pairwise_matches); - matcher.collectGarbage(); + if (!timelapse) + { + BestOf2NearestMatcher matcher(try_cuda, match_conf); + matcher(features, pairwise_matches); + matcher.collectGarbage(); + } + else + { + BestOf2NearestRangeMatcher matcher(timelapse_range, try_cuda, match_conf); + matcher(features, pairwise_matches); + matcher.collectGarbage(); + } + LOGLN("Pairwise matching, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); // Check if we should save matches graph @@ -679,6 +714,7 @@ int main(int argc, char* argv[]) Mat img_warped, img_warped_s; Mat dilated_mask, seam_mask, mask, mask_warped; Ptr blender; + Ptr timelapser; //double compose_seam_aspect = 1; double compose_work_aspect = 1; @@ -755,7 +791,7 @@ int main(int argc, char* argv[]) resize(dilated_mask, seam_mask, mask_warped.size()); mask_warped = seam_mask & mask_warped; - if (!blender) + if (!blender && !timelapse) { blender = Blender::createDefault(blend_type, try_cuda); Size dst_sz = resultRoi(corners, sizes).size(); @@ -776,17 +812,35 @@ int main(int argc, char* argv[]) } blender->prepare(corners, sizes); } + else if (!timelapser) + { + CV_Assert(timelapse); + timelapser = Timelapser::createDefault(timelapse_type); + timelapser->initialize(corners, sizes); + } // Blend the current image - blender->feed(img_warped_s, mask_warped, corners[img_idx]); + if (timelapse) + { + timelapser->process(img_warped_s, Mat::ones(img_warped_s.size(), CV_8UC1), corners[img_idx]); + + imwrite("fixed_" + img_names[img_idx], timelapser->getDst()); + } + else + { + blender->feed(img_warped_s, mask_warped, corners[img_idx]); + } } - Mat result, result_mask; - blender->blend(result, result_mask); + if (!timelapse) + { + Mat result, result_mask; + blender->blend(result, result_mask); - LOGLN("Compositing, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); + LOGLN("Compositing, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); - imwrite(result_name, result); + imwrite(result_name, result); + } LOGLN("Finished, total time: " << ((getTickCount() - app_start_time) / getTickFrequency()) << " sec"); return 0; -- 2.7.4