1 /*M///////////////////////////////////////////////////////////////////////////////////////
3 // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
5 // By downloading, copying, installing or using the software you agree to this license.
6 // If you do not agree to this license, do not download, install,
7 // copy or use the software.
11 // For Open Source Computer Vision Library
13 // Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
14 // Copyright (C) 2008-2012, Willow Garage Inc., all rights reserved.
15 // Third party copyrights are property of their respective owners.
17 // Redistribution and use in source and binary forms, with or without modification,
18 // are permitted provided that the following conditions are met:
20 // * Redistribution's of source code must retain the above copyright notice,
21 // this list of conditions and the following disclaimer.
23 // * Redistribution's in binary form must reproduce the above copyright notice,
24 // this list of conditions and the following disclaimer in the documentation
25 // and/or other materials provided with the distribution.
27 // * The name of the copyright holders may not be used to endorse or promote products
28 // derived from this software without specific prior written permission.
30 // This software is provided by the copyright holders and contributors "as is" and
31 // any express or implied warranties, including, but not limited to, the implied
32 // warranties of merchantability and fitness for a particular purpose are disclaimed.
33 // In no event shall the Intel Corporation or contributors be liable for any direct,
34 // indirect, incidental, special, exemplary, or consequential damages
35 // (including, but not limited to, procurement of substitute goods or services;
36 // loss of use, data, or profits; or business interruption) however caused
37 // and on any theory of liability, whether in contract, strict liability,
38 // or tort (including negligence or otherwise) arising in any way out of
39 // the use of this software, even if advised of the possibility of such damage.
43 #include "precomp.hpp"
45 #if !defined (HAVE_CUDA)
47 cv::softcascade::SCascade::SCascade(const double, const double, const int, const int) { throw_no_cuda(); }
49 cv::softcascade::SCascade::~SCascade() { throw_no_cuda(); }
51 bool cv::softcascade::SCascade::load(const FileNode&) { throw_no_cuda(); return false;}
53 void cv::softcascade::SCascade::detect(InputArray, InputArray, OutputArray, cv::cuda::Stream&) const { throw_no_cuda(); }
55 void cv::softcascade::SCascade::read(const FileNode& fn) { Algorithm::read(fn); }
57 cv::softcascade::ChannelsProcessor::ChannelsProcessor() { throw_no_cuda(); }
58 cv::softcascade::ChannelsProcessor::~ChannelsProcessor() { throw_no_cuda(); }
60 cv::Ptr<cv::softcascade::ChannelsProcessor> cv::softcascade::ChannelsProcessor::create(const int, const int, const int)
61 { throw_no_cuda(); return cv::Ptr<cv::softcascade::ChannelsProcessor>(); }
65 # include "cuda_invoker.hpp"
67 cv::softcascade::cudev::Level::Level(int idx, const Octave& oct, const float scale, const int w, const int h)
68 : octave(idx), step(oct.stages), relScale(scale / oct.scale)
70 workRect.x = (unsigned char)cvRound(w / (float)oct.shrinkage);
71 workRect.y = (unsigned char)cvRound(h / (float)oct.shrinkage);
73 objSize.x = cv::saturate_cast<uchar>(oct.size.x * relScale);
74 objSize.y = cv::saturate_cast<uchar>(oct.size.y * relScale);
76 // according to R. Benenson, M. Mathias, R. Timofte and L. Van Gool's and Dallal's papers
77 if (fabs(relScale - 1.f) < FLT_EPSILON)
78 scaling[0] = scaling[1] = 1.f;
81 scaling[0] = (relScale < 1.f) ? 0.89f * ::pow(relScale, 1.099f / ::log(2.0f)) : 1.f;
82 scaling[1] = relScale * relScale;
86 namespace cv { namespace softcascade { namespace cudev {
88 void fillBins(cv::cuda::PtrStepSzb hogluv, const cv::cuda::PtrStepSzf& nangle,
89 const int fw, const int fh, const int bins, cudaStream_t stream);
91 void suppress(const cv::cuda::PtrStepSzb& objects, cv::cuda::PtrStepSzb overlaps, cv::cuda::PtrStepSzi ndetections,
92 cv::cuda::PtrStepSzb suppressed, cudaStream_t stream);
94 void bgr2Luv(const cv::cuda::PtrStepSzb& bgr, cv::cuda::PtrStepSzb luv);
95 void transform(const cv::cuda::PtrStepSz<uchar3>& bgr, cv::cuda::PtrStepSzb gray);
96 void gray2hog(const cv::cuda::PtrStepSzb& gray, cv::cuda::PtrStepSzb mag, const int bins);
97 void shrink(const cv::cuda::PtrStepSzb& channels, cv::cuda::PtrStepSzb shrunk);
99 void shfl_integral(const cv::cuda::PtrStepSzb& img, cv::cuda::PtrStepSz<unsigned int> integral, cudaStream_t stream);
102 struct cv::softcascade::SCascade::Fields
104 static Fields* parseCascade(const FileNode &root, const float mins, const float maxs, const int totals, const int method)
106 static const char *const SC_STAGE_TYPE = "stageType";
107 static const char *const SC_BOOST = "BOOST";
108 static const char *const SC_FEATURE_TYPE = "featureType";
109 static const char *const SC_ICF = "ICF";
110 static const char *const SC_ORIG_W = "width";
111 static const char *const SC_ORIG_H = "height";
112 static const char *const SC_FEATURE_FORMAT = "featureFormat";
113 static const char *const SC_SHRINKAGE = "shrinkage";
114 static const char *const SC_OCTAVES = "octaves";
115 static const char *const SC_OCT_SCALE = "scale";
116 static const char *const SC_OCT_WEAKS = "weaks";
117 static const char *const SC_TREES = "trees";
118 static const char *const SC_WEAK_THRESHOLD = "treeThreshold";
119 static const char *const SC_FEATURES = "features";
120 static const char *const SC_INTERNAL = "internalNodes";
121 static const char *const SC_LEAF = "leafValues";
122 static const char *const SC_F_CHANNEL = "channel";
123 static const char *const SC_F_RECT = "rect";
125 // only Ada Boost supported
126 String stageTypeStr = (String)root[SC_STAGE_TYPE];
127 CV_Assert(stageTypeStr == SC_BOOST);
129 // only HOG-like integral channel features supported
130 String featureTypeStr = (String)root[SC_FEATURE_TYPE];
131 CV_Assert(featureTypeStr == SC_ICF);
133 int origWidth = (int)root[SC_ORIG_W];
134 int origHeight = (int)root[SC_ORIG_H];
136 String fformat = (String)root[SC_FEATURE_FORMAT];
137 bool useBoxes = (fformat == "BOX");
138 ushort shrinkage = cv::saturate_cast<ushort>((int)root[SC_SHRINKAGE]);
140 FileNode fn = root[SC_OCTAVES];
141 if (fn.empty()) return 0;
143 std::vector<cudev::Octave> voctaves;
144 std::vector<float> vstages;
145 std::vector<cudev::Node> vnodes;
146 std::vector<float> vleaves;
148 FileNodeIterator it = fn.begin(), it_end = fn.end();
149 for (ushort octIndex = 0; it != it_end; ++it, ++octIndex)
152 float scale = powf(2.f,saturate_cast<float>((int)fns[SC_OCT_SCALE]));
153 bool isUPOctave = scale >= 1;
155 ushort nweaks = saturate_cast<ushort>((int)fns[SC_OCT_WEAKS]);
158 size.x = (unsigned short)cvRound(origWidth * scale);
159 size.y = (unsigned short)cvRound(origHeight * scale);
161 cudev::Octave octave(octIndex, nweaks, shrinkage, size, scale);
162 CV_Assert(octave.stages > 0);
163 voctaves.push_back(octave);
165 FileNode ffs = fns[SC_FEATURES];
166 if (ffs.empty()) return 0;
168 std::vector<cv::Rect> feature_rects;
169 std::vector<int> feature_channels;
171 FileNodeIterator ftrs = ffs.begin(), ftrs_end = ffs.end();
172 int feature_offset = 0;
173 for (; ftrs != ftrs_end; ++ftrs, ++feature_offset )
175 cv::FileNode ftn = (*ftrs)[SC_F_RECT];
176 cv::FileNodeIterator r_it = ftn.begin();
177 int x = (int)*(r_it++);
178 int y = (int)*(r_it++);
179 int w = (int)*(r_it++);
180 int h = (int)*(r_it++);
198 feature_rects.push_back(cv::Rect(x, y, w, h));
199 feature_channels.push_back((int)(*ftrs)[SC_F_CHANNEL]);
203 if (fn.empty()) return 0;
205 // for each stage (~ decision tree with H = 2)
206 FileNodeIterator st = fns.begin(), st_end = fns.end();
207 for (; st != st_end; ++st )
209 FileNode octfn = *st;
210 float threshold = (float)octfn[SC_WEAK_THRESHOLD];
211 vstages.push_back(threshold);
213 FileNode intfns = octfn[SC_INTERNAL];
214 FileNodeIterator inIt = intfns.begin(), inIt_end = intfns.end();
215 for (; inIt != inIt_end;)
218 int featureIdx = (int)(*(inIt++));
220 float orig_threshold = (float)(*(inIt++));
221 unsigned int th = saturate_cast<unsigned int>((int)orig_threshold);
222 cv::Rect& r = feature_rects[featureIdx];
224 rect.x = saturate_cast<uchar>(r.x);
225 rect.y = saturate_cast<uchar>(r.y);
226 rect.z = saturate_cast<uchar>(r.width);
227 rect.w = saturate_cast<uchar>(r.height);
229 unsigned int channel = saturate_cast<unsigned int>(feature_channels[featureIdx]);
230 vnodes.push_back(cudev::Node(rect, channel, th));
233 intfns = octfn[SC_LEAF];
234 inIt = intfns.begin(), inIt_end = intfns.end();
235 for (; inIt != inIt_end; ++inIt)
237 vleaves.push_back((float)(*inIt));
242 cv::Mat hoctaves(1, (int) (voctaves.size() * sizeof(cudev::Octave)), CV_8UC1, (uchar*)&(voctaves[0]));
243 CV_Assert(!hoctaves.empty());
245 cv::Mat hstages(cv::Mat(vstages).reshape(1,1));
246 CV_Assert(!hstages.empty());
248 cv::Mat hnodes(1, (int) (vnodes.size() * sizeof(cudev::Node)), CV_8UC1, (uchar*)&(vnodes[0]) );
249 CV_Assert(!hnodes.empty());
251 cv::Mat hleaves(cv::Mat(vleaves).reshape(1,1));
252 CV_Assert(!hleaves.empty());
254 Fields* fields = new Fields(mins, maxs, totals, origWidth, origHeight, shrinkage, 0,
255 hoctaves, hstages, hnodes, hleaves, method);
256 fields->voctaves = voctaves;
257 fields->createLevels(DEFAULT_FRAME_HEIGHT, DEFAULT_FRAME_WIDTH);
262 bool check(float mins,float maxs, int scales)
264 bool updated = ((minScale == mins) || (maxScale == maxs) || (totals == scales));
273 int createLevels(const int fh, const int fw)
275 std::vector<cudev::Level> vlevels;
276 float logFactor = (::log(maxScale) - ::log(minScale)) / (totals -1);
278 float scale = minScale;
280 for (int sc = 0; sc < totals; ++sc)
282 int width = (int)::std::max(0.0f, fw - (origObjWidth * scale));
283 int height = (int)::std::max(0.0f, fh - (origObjHeight * scale));
285 float logScale = ::log(scale);
286 int fit = fitOctave(voctaves, logScale);
288 cudev::Level level(fit, voctaves[fit], scale, width, height);
290 if (!width || !height)
294 vlevels.push_back(level);
295 if (voctaves[fit].scale < 1) ++dcs;
298 if (::fabs(scale - maxScale) < FLT_EPSILON) break;
299 scale = ::std::min(maxScale, ::expf(::log(scale) + logFactor));
302 cv::Mat hlevels = cv::Mat(1, (int) (vlevels.size() * sizeof(cudev::Level)), CV_8UC1, (uchar*)&(vlevels[0]) );
303 CV_Assert(!hlevels.empty());
304 levels.upload(hlevels);
309 bool update(int fh, int fw, int shr)
311 shrunk.create(fh / shr * HOG_LUV_BINS, fw / shr, CV_8UC1);
312 integralBuffer.create(shrunk.rows, shrunk.cols, CV_32SC1);
314 hogluv.create((fh / shr) * HOG_LUV_BINS + 1, fw / shr + 1, CV_32SC1);
315 hogluv.setTo(cv::Scalar::all(0));
317 overlaps.create(1, 5000, CV_8UC1);
318 suppressed.create(1, sizeof(Detection) * 51, CV_8UC1);
323 Fields( const float mins, const float maxs, const int tts, const int ow, const int oh, const int shr, const int ds,
324 cv::Mat hoctaves, cv::Mat hstages, cv::Mat hnodes, cv::Mat hleaves, int method)
325 : minScale(mins), maxScale(maxs), totals(tts), origObjWidth(ow), origObjHeight(oh), shrinkage(shr), downscales(ds)
327 update(DEFAULT_FRAME_HEIGHT, DEFAULT_FRAME_WIDTH, shr);
328 octaves.upload(hoctaves);
329 stages.upload(hstages);
330 nodes.upload(hnodes);
331 leaves.upload(hleaves);
333 preprocessor = ChannelsProcessor::create(shrinkage, 6, method);
336 void detect(cv::cuda::GpuMat& objects, cv::cuda::Stream& s) const
338 objects.setTo(Scalar::all(0), s);
340 cudaSafeCall( cudaGetLastError());
342 cudev::CascadeInvoker<cudev::GK107PolicyX4> invoker
343 = cudev::CascadeInvoker<cudev::GK107PolicyX4>(levels, stages, nodes, leaves);
345 cudaStream_t stream = cv::cuda::StreamAccessor::getStream(s);
346 invoker(mask, hogluv, objects, downscales, stream);
349 void suppress(cv::cuda::GpuMat& objects, cv::cuda::Stream& s)
351 cv::cuda::GpuMat ndetections = cv::cuda::GpuMat(objects, cv::Rect(0, 0, sizeof(Detection), 1));
352 ensureSizeIsEnough(objects.rows, objects.cols, CV_8UC1, overlaps);
354 overlaps.setTo(0, s);
355 suppressed.setTo(0, s);
357 cudaStream_t stream = cv::cuda::StreamAccessor::getStream(s);
358 cudev::suppress(objects, overlaps, ndetections, suppressed, stream);
363 typedef std::vector<cudev::Octave>::const_iterator octIt_t;
364 static int fitOctave(const std::vector<cudev::Octave>& octs, const float& logFactor)
366 float minAbsLog = FLT_MAX;
368 for (int oct = 0; oct < (int)octs.size(); ++oct)
370 const cudev::Octave& octave =octs[oct];
371 float logOctave = ::log(octave.scale);
372 float logAbsScale = ::fabs(logFactor - logOctave);
374 if(logAbsScale < minAbsLog)
377 minAbsLog = logAbsScale;
385 cv::Ptr<ChannelsProcessor> preprocessor;
401 cv::cuda::GpuMat shrunk;
403 // temporal mat for integral
404 cv::cuda::GpuMat integralBuffer;
407 cv::cuda::GpuMat hogluv;
410 // used for suppression
411 cv::cuda::GpuMat suppressed;
412 // used for area overlap computing during
413 cv::cuda::GpuMat overlaps;
417 cv::cuda::GpuMat octaves;
418 cv::cuda::GpuMat stages;
419 cv::cuda::GpuMat nodes;
420 cv::cuda::GpuMat leaves;
421 cv::cuda::GpuMat levels;
425 cv::cuda::GpuMat mask;
426 cv::cuda::GpuMat genRoiTmp;
428 // cv::cuda::GpuMat collected;
431 std::vector<cudev::Octave> voctaves;
438 DEFAULT_FRAME_WIDTH = 640,
439 DEFAULT_FRAME_HEIGHT = 480,
444 cv::softcascade::SCascade::Fields& operator=( const cv::softcascade::SCascade::Fields & );
447 cv::softcascade::SCascade::SCascade(const double mins, const double maxs, const int sc, const int fl)
448 : fields(0), minScale(mins), maxScale(maxs), scales(sc), flags(fl) {}
450 cv::softcascade::SCascade::~SCascade() { delete fields; }
452 bool cv::softcascade::SCascade::load(const FileNode& fn)
454 if (fields) delete fields;
455 fields = Fields::parseCascade(fn, (float)minScale, (float)maxScale, scales, flags);
461 void integral(const cv::cuda::GpuMat& src, cv::cuda::GpuMat& sum, cv::cuda::GpuMat& buffer, cv::cuda::Stream& s)
463 CV_Assert(src.type() == CV_8UC1);
465 cudaStream_t stream = cv::cuda::StreamAccessor::getStream(s);
470 src.locateROI(whole, offset);
472 if (cv::cuda::deviceSupports(cv::cuda::WARP_SHUFFLE_FUNCTIONS) && src.cols <= 2048
473 && offset.x % 16 == 0 && ((src.cols + 63) / 64) * 64 <= (static_cast<int>(src.step) - offset.x))
475 ensureSizeIsEnough(((src.rows + 7) / 8) * 8, ((src.cols + 63) / 64) * 64, CV_32SC1, buffer);
477 cv::softcascade::cudev::shfl_integral(src, buffer, stream);
479 sum.create(src.rows + 1, src.cols + 1, CV_32SC1);
480 sum.setTo(cv::Scalar::all(0), s);
482 cv::cuda::GpuMat inner = sum(cv::Rect(1, 1, src.cols, src.rows));
483 cv::cuda::GpuMat res = buffer(cv::Rect(0, 0, src.cols, src.rows));
485 res.copyTo(inner, s);
487 else {CV_Error(cv::Error::GpuNotSupported, ": CC 3.x required.");}
492 void cv::softcascade::SCascade::detect(InputArray _image, InputArray _rois, OutputArray _objects, cv::cuda::Stream& s) const
496 // only color images and precomputed integrals are supported
497 int type = _image.type();
498 CV_Assert(type == CV_8UC3 || type == CV_32SC1 || (!_rois.empty()));
500 const cv::cuda::GpuMat image = _image.getGpuMat();
502 if (_objects.empty()) _objects.create(1, 4096 * sizeof(Detection), CV_8UC1);
504 cv::cuda::GpuMat rois = _rois.getGpuMat(), objects = _objects.getGpuMat();
507 Fields& flds = *fields;
508 int shr = flds.shrinkage;
510 flds.mask.create( rois.cols / shr, rois.rows / shr, rois.type());
512 cudev::shrink(rois, flds.mask);
513 //cv::cuda::transpose(flds.genRoiTmp, flds.mask, s);
517 flds.update(image.rows, image.cols, flds.shrinkage);
519 if (flds.check((float)minScale, (float)maxScale, scales))
520 flds.createLevels(image.rows, image.cols);
522 flds.preprocessor->apply(image, flds.shrunk);
523 ::integral(flds.shrunk, flds.hogluv, flds.integralBuffer, s);
527 image.copyTo(flds.hogluv, s);
530 flds.detect(objects, s);
532 if ( (flags && NMS_MASK) != NO_REJECT)
534 cv::cuda::GpuMat spr(objects, cv::Rect(0, 0, flds.suppressed.cols, flds.suppressed.rows));
535 flds.suppress(objects, s);
536 flds.suppressed.copyTo(spr);
540 void cv::softcascade::SCascade::read(const FileNode& fn)
547 using cv::InputArray;
548 using cv::OutputArray;
549 using cv::cuda::Stream;
550 using cv::cuda::GpuMat;
552 inline void setZero(cv::cuda::GpuMat& m, cv::cuda::Stream& s)
557 struct SeparablePreprocessor : public cv::softcascade::ChannelsProcessor
559 SeparablePreprocessor(const int s, const int b) : cv::softcascade::ChannelsProcessor(), shrinkage(s), bins(b) {}
560 virtual ~SeparablePreprocessor() {}
562 virtual void apply(InputArray _frame, OutputArray _shrunk, cv::cuda::Stream& s = cv::cuda::Stream::Null())
564 bgr = _frame.getGpuMat();
565 //cv::cuda::GaussianBlur(frame, bgr, cv::Size(3, 3), -1.0);
567 _shrunk.create(bgr.rows * (4 + bins) / shrinkage, bgr.cols / shrinkage, CV_8UC1);
568 cv::cuda::GpuMat shrunk = _shrunk.getGpuMat();
570 channels.create(bgr.rows * (4 + bins), bgr.cols, CV_8UC1);
571 setZero(channels, s);
573 gray.create(bgr.size(), CV_8UC1);
574 cv::softcascade::cudev::transform(bgr, gray); //cv::cuda::cvtColor(bgr, gray, CV_BGR2GRAY);
575 cv::softcascade::cudev::gray2hog(gray, channels(cv::Rect(0, 0, bgr.cols, bgr.rows * (bins + 1))), bins);
577 cv::cuda::GpuMat luv(channels, cv::Rect(0, bgr.rows * (bins + 1), bgr.cols, bgr.rows * 3));
578 cv::softcascade::cudev::bgr2Luv(bgr, luv);
579 cv::softcascade::cudev::shrink(channels, shrunk);
586 cv::cuda::GpuMat bgr;
587 cv::cuda::GpuMat gray;
588 cv::cuda::GpuMat channels;
589 SeparablePreprocessor& operator=( const SeparablePreprocessor& );
594 cv::Ptr<cv::softcascade::ChannelsProcessor> cv::softcascade::ChannelsProcessor::create(const int s, const int b, const int m)
596 CV_Assert((m && SEPARABLE));
597 return makePtr<SeparablePreprocessor>(s, b);
600 cv::softcascade::ChannelsProcessor::ChannelsProcessor() { }
601 cv::softcascade::ChannelsProcessor::~ChannelsProcessor() { }