From f95e71ea3a63cf27e6060e5ea5dd28e2656f9338 Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Mon, 15 Aug 2011 08:19:57 +0000 Subject: [PATCH] added saving of matches graph into opencv_stitching (in DOT format) --- modules/stitching/main.cpp | 21 ++++ modules/stitching/matchers.hpp | 198 ++++++++++++++++---------------- modules/stitching/motion_estimators.cpp | 65 +++++++++++ modules/stitching/motion_estimators.hpp | 186 +++++++++++++++--------------- modules/stitching/util.hpp | 3 +- 5 files changed, 282 insertions(+), 191 deletions(-) diff --git a/modules/stitching/main.cpp b/modules/stitching/main.cpp index 76a4c44..573f685 100644 --- a/modules/stitching/main.cpp +++ b/modules/stitching/main.cpp @@ -48,6 +48,7 @@ // 3) Automatic Panoramic Image Stitching using Invariant Features. // Matthew Brown and David G. Lowe. 2007. +#include #include "precomp.hpp" #include "util.hpp" #include "warpers.hpp" @@ -83,6 +84,10 @@ void printUsage() " Bundle adjustment cost function. The default is 'focal_ray'.\n" " --wave_correct (no|yes)\n" " Perform wave effect correction. The default is 'yes'.\n" + " --save_graph \n" + " Save matches graph represented in DOT language to file.\n" + " Labels description: Nm is number of matches, Ni is number of inliers,\n" + " C is confidence.\n" "\nCompositing Flags:\n" " --warp (plane|cylindrical|spherical)\n" " Warp surface type. The default is 'spherical'.\n" @@ -114,6 +119,8 @@ double compose_megapix = -1; int ba_space = BundleAdjuster::FOCAL_RAY_SPACE; float conf_thresh = 1.f; bool wave_correct = true; +bool save_graph = false; +std::string save_graph_to; int warp_type = Warper::SPHERICAL; int expos_comp_type = ExposureCompensator::GAIN_BLOCKS; float match_conf = 0.65f; @@ -209,6 +216,12 @@ int parseCmdArgs(int argc, char** argv) } i++; } + else if (string(argv[i]) == "--save_graph") + { + save_graph = true; + save_graph_to = argv[i + 1]; + i++; + } else if (string(argv[i]) == "--warp") { if (string(argv[i + 1]) == "plane") @@ -378,6 +391,14 @@ int main(int argc, char* argv[]) matcher.releaseMemory(); LOGLN("Pairwise matching, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); + // Check if we should save matches graph + if (save_graph) + { + LOGLN("Saving matches graph..."); + ofstream f(save_graph_to.c_str()); + f << matchesGraphAsString(img_names, pairwise_matches, conf_thresh); + } + // Leave only images we are sure are from the same panorama vector indices = leaveBiggestComponent(features, pairwise_matches, conf_thresh); vector img_subset; diff --git a/modules/stitching/matchers.hpp b/modules/stitching/matchers.hpp index fe7fbe0..f35dcd4 100644 --- a/modules/stitching/matchers.hpp +++ b/modules/stitching/matchers.hpp @@ -38,102 +38,102 @@ // or tort (including negligence or otherwise) arising in any way out of // the use of this software, even if advised of the possibility of such damage. // -//M*/ -#ifndef __OPENCV_MATCHERS_HPP__ -#define __OPENCV_MATCHERS_HPP__ - -#include "precomp.hpp" - -struct ImageFeatures -{ - int img_idx; - cv::Size img_size; - std::vector keypoints; - cv::Mat descriptors; -}; - - -class FeaturesFinder -{ -public: - virtual ~FeaturesFinder() {} - void operator ()(const cv::Mat &image, ImageFeatures &features); - - virtual void releaseMemory() {} - -protected: - virtual void find(const cv::Mat &image, ImageFeatures &features) = 0; -}; - - -class SurfFeaturesFinder : public FeaturesFinder -{ -public: - SurfFeaturesFinder(bool try_use_gpu = true, double hess_thresh = 300.0, - int num_octaves = 3, int num_layers = 4, - int num_octaves_descr = 4, int num_layers_descr = 2); - - void releaseMemory(); - -protected: - void find(const cv::Mat &image, ImageFeatures &features); - - cv::Ptr impl_; -}; - - -struct MatchesInfo -{ - MatchesInfo(); - MatchesInfo(const MatchesInfo &other); - const MatchesInfo& operator =(const MatchesInfo &other); - - int src_img_idx, dst_img_idx; // Images indices (optional) - std::vector matches; - std::vector inliers_mask; // Geometrically consistent matches mask - int num_inliers; // Number of geometrically consistent matches - cv::Mat H; // Estimated homography - double confidence; // Confidence two images are from the same panorama -}; - - -class FeaturesMatcher -{ -public: - virtual ~FeaturesMatcher() {} - - void operator ()(const ImageFeatures &features1, const ImageFeatures &features2, - MatchesInfo& matches_info) { match(features1, features2, matches_info); } - void operator ()(const std::vector &features, std::vector &pairwise_matches); - - bool isThreadSafe() const { return is_thread_safe_; } - - virtual void releaseMemory() {} - -protected: - FeaturesMatcher(bool is_thread_safe = false) : is_thread_safe_(is_thread_safe) {} - - virtual void match(const ImageFeatures &features1, const ImageFeatures &features2, - MatchesInfo& matches_info) = 0; - - bool is_thread_safe_; -}; - - -class BestOf2NearestMatcher : public FeaturesMatcher -{ -public: - BestOf2NearestMatcher(bool try_use_gpu = true, float match_conf = 0.55f, int num_matches_thresh1 = 6, - int num_matches_thresh2 = 6); - - void releaseMemory(); - -protected: - void match(const ImageFeatures &features1, const ImageFeatures &features2, MatchesInfo &matches_info); - - int num_matches_thresh1_; - int num_matches_thresh2_; - cv::Ptr impl_; -}; - -#endif // __OPENCV_MATCHERS_HPP__ \ No newline at end of file +//M*/ +#ifndef __OPENCV_MATCHERS_HPP__ +#define __OPENCV_MATCHERS_HPP__ + +#include "precomp.hpp" + +struct ImageFeatures +{ + int img_idx; + cv::Size img_size; + std::vector keypoints; + cv::Mat descriptors; +}; + + +class FeaturesFinder +{ +public: + virtual ~FeaturesFinder() {} + void operator ()(const cv::Mat &image, ImageFeatures &features); + + virtual void releaseMemory() {} + +protected: + virtual void find(const cv::Mat &image, ImageFeatures &features) = 0; +}; + + +class SurfFeaturesFinder : public FeaturesFinder +{ +public: + SurfFeaturesFinder(bool try_use_gpu = true, double hess_thresh = 300.0, + int num_octaves = 3, int num_layers = 4, + int num_octaves_descr = 4, int num_layers_descr = 2); + + void releaseMemory(); + +protected: + void find(const cv::Mat &image, ImageFeatures &features); + + cv::Ptr impl_; +}; + + +struct MatchesInfo +{ + MatchesInfo(); + MatchesInfo(const MatchesInfo &other); + const MatchesInfo& operator =(const MatchesInfo &other); + + int src_img_idx, dst_img_idx; // Images indices (optional) + std::vector matches; + std::vector inliers_mask; // Geometrically consistent matches mask + int num_inliers; // Number of geometrically consistent matches + cv::Mat H; // Estimated homography + double confidence; // Confidence two images are from the same panorama +}; + + +class FeaturesMatcher +{ +public: + virtual ~FeaturesMatcher() {} + + void operator ()(const ImageFeatures &features1, const ImageFeatures &features2, + MatchesInfo& matches_info) { match(features1, features2, matches_info); } + void operator ()(const std::vector &features, std::vector &pairwise_matches); + + bool isThreadSafe() const { return is_thread_safe_; } + + virtual void releaseMemory() {} + +protected: + FeaturesMatcher(bool is_thread_safe = false) : is_thread_safe_(is_thread_safe) {} + + virtual void match(const ImageFeatures &features1, const ImageFeatures &features2, + MatchesInfo& matches_info) = 0; + + bool is_thread_safe_; +}; + + +class BestOf2NearestMatcher : public FeaturesMatcher +{ +public: + BestOf2NearestMatcher(bool try_use_gpu = true, float match_conf = 0.55f, int num_matches_thresh1 = 6, + int num_matches_thresh2 = 6); + + void releaseMemory(); + +protected: + void match(const ImageFeatures &features1, const ImageFeatures &features2, MatchesInfo &matches_info); + + int num_matches_thresh1_; + int num_matches_thresh2_; + cv::Ptr impl_; +}; + +#endif // __OPENCV_MATCHERS_HPP__ diff --git a/modules/stitching/motion_estimators.cpp b/modules/stitching/motion_estimators.cpp index d354456..cf359f1 100644 --- a/modules/stitching/motion_estimators.cpp +++ b/modules/stitching/motion_estimators.cpp @@ -40,6 +40,7 @@ // //M*/ #include +#include #include "autocalib.hpp" #include "motion_estimators.hpp" #include "util.hpp" @@ -407,6 +408,70 @@ void waveCorrect(vector &rmats) ////////////////////////////////////////////////////////////////////////////// +string matchesGraphAsString(vector &pathes, vector &pairwise_matches, + float conf_threshold) +{ + stringstream str; + str << "graph matches_graph{\n"; + + set added_imgs; + + // Add matches + for (size_t i = 0; i < pairwise_matches.size(); ++i) + { + if (pairwise_matches[i].src_img_idx < pairwise_matches[i].dst_img_idx && + pairwise_matches[i].confidence > conf_threshold) + { + string name_src = pathes[pairwise_matches[i].src_img_idx]; + size_t prefix_len = name_src.find_last_of("/\\"); + if (prefix_len != string::npos) prefix_len++; else prefix_len = 0; + name_src = name_src.substr(prefix_len, name_src.size() - prefix_len); + + string name_dst = pathes[pairwise_matches[i].dst_img_idx]; + prefix_len = name_dst.find_last_of("/\\"); + if (prefix_len != string::npos) prefix_len++; else prefix_len = 0; + name_dst = name_dst.substr(prefix_len, name_dst.size() - prefix_len); + + added_imgs.insert(pairwise_matches[i].src_img_idx); + added_imgs.insert(pairwise_matches[i].dst_img_idx); + + str << "\"" << name_src << "\" -- \"" << name_dst << "\"" + << "[label=\"Nm=" << pairwise_matches[i].matches.size() + << ", Ni=" << pairwise_matches[i].num_inliers + << ", C=" << pairwise_matches[i].confidence << "\"];\n"; + } + } + + // Add unmatched images + for (size_t i = 0; i < pairwise_matches.size(); ++i) + { + if (pairwise_matches[i].src_img_idx < pairwise_matches[i].dst_img_idx) + { + if (added_imgs.find(pairwise_matches[i].src_img_idx) == added_imgs.end()) + { + added_imgs.insert(pairwise_matches[i].src_img_idx); + string name = pathes[pairwise_matches[i].src_img_idx]; + size_t prefix_len = name.find_last_of("/\\"); + if (prefix_len != string::npos) prefix_len++; else prefix_len = 0; + name = name.substr(prefix_len, name.size() - prefix_len); + str << "\"" << name << "\";\n"; + } + if (added_imgs.find(pairwise_matches[i].dst_img_idx) == added_imgs.end()) + { + added_imgs.insert(pairwise_matches[i].dst_img_idx); + string name = pathes[pairwise_matches[i].dst_img_idx]; + size_t prefix_len = name.find_last_of("/\\"); + if (prefix_len != string::npos) prefix_len++; else prefix_len = 0; + name = name.substr(prefix_len, name.size() - prefix_len); + str << "\"" << name << "\";\n"; + } + } + } + + str << "}"; + return str.str(); +} + vector leaveBiggestComponent(vector &features, vector &pairwise_matches, float conf_threshold) { diff --git a/modules/stitching/motion_estimators.hpp b/modules/stitching/motion_estimators.hpp index 8682a4c..369f725 100644 --- a/modules/stitching/motion_estimators.hpp +++ b/modules/stitching/motion_estimators.hpp @@ -38,94 +38,98 @@ // or tort (including negligence or otherwise) arising in any way out of // the use of this software, even if advised of the possibility of such damage. // -//M*/ -#ifndef __OPENCV_MOTION_ESTIMATORS_HPP__ -#define __OPENCV_MOTION_ESTIMATORS_HPP__ - -#include "precomp.hpp" -#include "matchers.hpp" -#include "util.hpp" - -struct CameraParams -{ - CameraParams(); - CameraParams(const CameraParams& other); - const CameraParams& operator =(const CameraParams& other); - - double focal; // Focal length - cv::Mat R; // Rotation - cv::Mat t; // Translation -}; - - -class Estimator -{ -public: - void operator ()(const std::vector &features, const std::vector &pairwise_matches, - std::vector &cameras) - { - estimate(features, pairwise_matches, cameras); - } - -protected: - virtual void estimate(const std::vector &features, const std::vector &pairwise_matches, - std::vector &cameras) = 0; -}; - - -class HomographyBasedEstimator : public Estimator -{ -public: - HomographyBasedEstimator() : is_focals_estimated_(false) {} - bool isFocalsEstimated() const { return is_focals_estimated_; } - -private: - void estimate(const std::vector &features, const std::vector &pairwise_matches, - std::vector &cameras); - - bool is_focals_estimated_; -}; - - -class BundleAdjuster : public Estimator -{ -public: - enum { RAY_SPACE, FOCAL_RAY_SPACE }; - - BundleAdjuster(int cost_space = FOCAL_RAY_SPACE, float conf_thresh = 1.f) - : cost_space_(cost_space), conf_thresh_(conf_thresh) {} - -private: - void estimate(const std::vector &features, const std::vector &pairwise_matches, - std::vector &cameras); - - void calcError(cv::Mat &err); - void calcJacobian(); - - int num_images_; - int total_num_matches_; - const ImageFeatures *features_; - const MatchesInfo *pairwise_matches_; - cv::Mat cameras_; - std::vector > edges_; - - int cost_space_; - float conf_thresh_; - cv::Mat err_, err1_, err2_; - cv::Mat J_; -}; - - -void waveCorrect(std::vector &rmats); - - -////////////////////////////////////////////////////////////////////////////// -// Auxiliary functions - -std::vector leaveBiggestComponent(std::vector &features, std::vector &pairwise_matches, - float conf_threshold); - -void findMaxSpanningTree(int num_images, const std::vector &pairwise_matches, - Graph &span_tree, std::vector ¢ers); - -#endif // __OPENCV_MOTION_ESTIMATORS_HPP__ +//M*/ +#ifndef __OPENCV_MOTION_ESTIMATORS_HPP__ +#define __OPENCV_MOTION_ESTIMATORS_HPP__ + +#include "precomp.hpp" +#include "matchers.hpp" +#include "util.hpp" + +struct CameraParams +{ + CameraParams(); + CameraParams(const CameraParams& other); + const CameraParams& operator =(const CameraParams& other); + + double focal; // Focal length + cv::Mat R; // Rotation + cv::Mat t; // Translation +}; + + +class Estimator +{ +public: + void operator ()(const std::vector &features, const std::vector &pairwise_matches, + std::vector &cameras) + { + estimate(features, pairwise_matches, cameras); + } + +protected: + virtual void estimate(const std::vector &features, const std::vector &pairwise_matches, + std::vector &cameras) = 0; +}; + + +class HomographyBasedEstimator : public Estimator +{ +public: + HomographyBasedEstimator() : is_focals_estimated_(false) {} + bool isFocalsEstimated() const { return is_focals_estimated_; } + +private: + void estimate(const std::vector &features, const std::vector &pairwise_matches, + std::vector &cameras); + + bool is_focals_estimated_; +}; + + +class BundleAdjuster : public Estimator +{ +public: + enum { RAY_SPACE, FOCAL_RAY_SPACE }; + + BundleAdjuster(int cost_space = FOCAL_RAY_SPACE, float conf_thresh = 1.f) + : cost_space_(cost_space), conf_thresh_(conf_thresh) {} + +private: + void estimate(const std::vector &features, const std::vector &pairwise_matches, + std::vector &cameras); + + void calcError(cv::Mat &err); + void calcJacobian(); + + int num_images_; + int total_num_matches_; + const ImageFeatures *features_; + const MatchesInfo *pairwise_matches_; + cv::Mat cameras_; + std::vector > edges_; + + int cost_space_; + float conf_thresh_; + cv::Mat err_, err1_, err2_; + cv::Mat J_; +}; + + +void waveCorrect(std::vector &rmats); + + +////////////////////////////////////////////////////////////////////////////// +// Auxiliary functions + +// Returns matches graph representation in DOT language +std::string matchesGraphAsString(std::vector &pathes, std::vector &pairwise_matches, + float conf_threshold); + +std::vector leaveBiggestComponent(std::vector &features, std::vector &pairwise_matches, + float conf_threshold); + +void findMaxSpanningTree(int num_images, const std::vector &pairwise_matches, + Graph &span_tree, std::vector ¢ers); + +#endif // __OPENCV_MOTION_ESTIMATORS_HPP__ diff --git a/modules/stitching/util.hpp b/modules/stitching/util.hpp index 6bec221..4c2db0d 100644 --- a/modules/stitching/util.hpp +++ b/modules/stitching/util.hpp @@ -108,8 +108,9 @@ bool overlapRoi(cv::Point tl1, cv::Point tl2, cv::Size sz1, cv::Size sz2, cv::Re cv::Rect resultRoi(const std::vector &corners, const std::vector &images); cv::Rect resultRoi(const std::vector &corners, const std::vector &sizes); cv::Point resultTl(const std::vector &corners); -void selectRandomSubset(int count, int size, std::vector &subset); +// Returns random 'count' element subset of the {0,1,...,size-1} set +void selectRandomSubset(int count, int size, std::vector &subset); #include "util_inl.hpp" -- 2.7.4