samples/cpp/video_homography.cpp

   1 /*
   2 * video_homography.cpp
   3 *
   4 *  Created on: Oct 18, 2010
   5 *      Author: erublee
   6 */
   7
   8 #include "opencv2/calib3d/calib3d.hpp"
   9 #include "opencv2/highgui/highgui.hpp"
  10 #include "opencv2/imgproc/imgproc.hpp"
  11 #include "opencv2/features2d/features2d.hpp"
  12 #include <iostream>
  13 #include <list>
  14 #include <vector>
  15
  16 using namespace std;
  17 using namespace cv;
  18
  19 void help(char **av)
  20 {
  21     cout << "\nThis program demonstrated the use of features2d with the Fast corner detector and brief descriptors\n"
  22         << "to track planar objects by computing their homography from the key (training) image to the query (test) image\n\n" << endl;
  23     cout << "usage: " << av[0] << " <video device number>\n" << endl;
  24     cout << "The following keys do stuff:" << endl;
  25     cout << "  t : grabs a reference frame to match against" << endl;
  26     cout << "  l : makes the reference frame new every frame" << endl;
  27     cout << "  q or escape: quit" << endl;
  28 }
  29
  30 namespace
  31 {
  32     void drawMatchesRelative(const vector<KeyPoint>& train, const vector<KeyPoint>& query,
  33         std::vector<cv::DMatch>& matches, Mat& img, const vector<unsigned char>& mask = vector<
  34         unsigned char> ())
  35     {
  36         for (int i = 0; i < (int)matches.size(); i++)
  37         {
  38             if (mask.empty() || mask[i])
  39             {
  40                 Point2f pt_new = query[matches[i].queryIdx].pt;
  41                 Point2f pt_old = train[matches[i].trainIdx].pt;
  42
  43                 cv::line(img, pt_new, pt_old, Scalar(125, 255, 125), 1);
  44                 cv::circle(img, pt_new, 2, Scalar(255, 0, 125), 1);
  45
  46             }
  47         }
  48     }
  49
  50     //Takes a descriptor and turns it into an xy point
  51     void keypoints2points(const vector<KeyPoint>& in, vector<Point2f>& out)
  52     {
  53         out.clear();
  54         out.reserve(in.size());
  55         for (size_t i = 0; i < in.size(); ++i)
  56         {
  57             out.push_back(in[i].pt);
  58         }
  59     }
  60
  61     //Takes an xy point and appends that to a keypoint structure
  62     void points2keypoints(const vector<Point2f>& in, vector<KeyPoint>& out)
  63     {
  64         out.clear();
  65         out.reserve(in.size());
  66         for (size_t i = 0; i < in.size(); ++i)
  67         {
  68             out.push_back(KeyPoint(in[i], 1));
  69         }
  70     }
  71
  72     //Uses computed homography H to warp original input points to new planar position
  73     void warpKeypoints(const Mat& H, const vector<KeyPoint>& in, vector<KeyPoint>& out)
  74     {
  75         vector<Point2f> pts;
  76         keypoints2points(in, pts);
  77         vector<Point2f> pts_w(pts.size());
  78         Mat m_pts_w(pts_w);
  79         perspectiveTransform(Mat(pts), m_pts_w, H);
  80         points2keypoints(pts_w, out);
  81     }
  82
  83     //Converts matching indices to xy points
  84     void matches2points(const vector<KeyPoint>& train, const vector<KeyPoint>& query,
  85         const std::vector<cv::DMatch>& matches, std::vector<cv::Point2f>& pts_train,
  86         std::vector<Point2f>& pts_query)
  87     {
  88
  89         pts_train.clear();
  90         pts_query.clear();
  91         pts_train.reserve(matches.size());
  92         pts_query.reserve(matches.size());
  93
  94         size_t i = 0;
  95
  96         for (; i < matches.size(); i++)
  97         {
  98
  99             const DMatch & dmatch = matches[i];
 100
 101             pts_query.push_back(query[dmatch.queryIdx].pt);
 102             pts_train.push_back(train[dmatch.trainIdx].pt);
 103
 104         }
 105
 106     }
 107
 108     void resetH(Mat&H)
 109     {
 110         H = Mat::eye(3, 3, CV_32FC1);
 111     }
 112 }
 113
 114 int main(int ac, char ** av)
 115 {
 116
 117     if (ac != 2)
 118     {
 119         help(av);
 120         return 1;
 121     }
 122
 123     BriefDescriptorExtractor brief(32);
 124
 125     VideoCapture capture;
 126     capture.open(atoi(av[1]));
 127     if (!capture.isOpened())
 128     {
 129         help(av);
 130         cout << "capture device " << atoi(av[1]) << " failed to open!" << endl;
 131         return 1;
 132     }
 133
 134     cout << "following keys do stuff:" << endl;
 135     cout << "t : grabs a reference frame to match against" << endl;
 136     cout << "l : makes the reference frame new every frame" << endl;
 137     cout << "q or escape: quit" << endl;
 138
 139     Mat frame;
 140
 141     vector<DMatch> matches;
 142
 143     BFMatcher desc_matcher(NORM_HAMMING);
 144
 145     vector<Point2f> train_pts, query_pts;
 146     vector<KeyPoint> train_kpts, query_kpts;
 147     vector<unsigned char> match_mask;
 148
 149     Mat gray;
 150
 151     bool ref_live = true;
 152
 153     Mat train_desc, query_desc;
 154     const int DESIRED_FTRS = 500;
 155     GridAdaptedFeatureDetector detector(new FastFeatureDetector(10, true), DESIRED_FTRS, 4, 4);
 156
 157     Mat H_prev = Mat::eye(3, 3, CV_32FC1);
 158     for (;;)
 159     {
 160         capture >> frame;
 161         if (frame.empty())
 162             break;
 163
 164         cvtColor(frame, gray, CV_RGB2GRAY);
 165
 166         detector.detect(gray, query_kpts); //Find interest points
 167
 168         brief.compute(gray, query_kpts, query_desc); //Compute brief descriptors at each keypoint location
 169
 170         if (!train_kpts.empty())
 171         {
 172
 173             vector<KeyPoint> test_kpts;
 174             warpKeypoints(H_prev.inv(), query_kpts, test_kpts);
 175
 176             Mat mask = windowedMatchingMask(test_kpts, train_kpts, 25, 25);
 177             desc_matcher.match(query_desc, train_desc, matches, mask);
 178             drawKeypoints(frame, test_kpts, frame, Scalar(255, 0, 0), DrawMatchesFlags::DRAW_OVER_OUTIMG);
 179
 180             matches2points(train_kpts, query_kpts, matches, train_pts, query_pts);
 181
 182             if (matches.size() > 5)
 183             {
 184                 Mat H = findHomography(train_pts, query_pts, RANSAC, 4, match_mask);
 185                 if (countNonZero(Mat(match_mask)) > 15)
 186                 {
 187                     H_prev = H;
 188                 }
 189                 else
 190                     resetH(H_prev);
 191                 drawMatchesRelative(train_kpts, query_kpts, matches, frame, match_mask);
 192             }
 193             else
 194                 resetH(H_prev);
 195
 196         }
 197         else
 198         {
 199             H_prev = Mat::eye(3, 3, CV_32FC1);
 200             Mat out;
 201             drawKeypoints(gray, query_kpts, out);
 202             frame = out;
 203         }
 204
 205         imshow("frame", frame);
 206
 207         if (ref_live)
 208         {
 209             train_kpts = query_kpts;
 210             query_desc.copyTo(train_desc);
 211         }
 212         char key = (char)waitKey(2);
 213         switch (key)
 214         {
 215         case 'l':
 216             ref_live = true;
 217             resetH(H_prev);
 218             break;
 219         case 't':
 220             ref_live = false;
 221             train_kpts = query_kpts;
 222             query_desc.copyTo(train_desc);
 223             resetH(H_prev);
 224             break;
 225         case 27:
 226         case 'q':
 227             return 0;
 228             break;
 229         }
 230
 231     }
 232     return 0;
 233 }