1 /*M///////////////////////////////////////////////////////////////////////////////////////
3 // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
5 // By downloading, copying, installing or using the software you agree to this license.
6 // If you do not agree to this license, do not download, install,
7 // copy or use the software.
11 // For Open Source Computer Vision Library
13 // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
14 // Copyright (C) 2017, Intel Corporation, all rights reserved.
15 // Third party copyrights are property of their respective owners.
17 // Redistribution and use in source and binary forms, with or without modification,
18 // are permitted provided that the following conditions are met:
20 // * Redistribution's of source code must retain the above copyright notice,
21 // this list of conditions and the following disclaimer.
23 // * Redistribution's in binary form must reproduce the above copyright notice,
24 // this list of conditions and the following disclaimer in the documentation
25 // and/or other materials provided with the distribution.
27 // * The name of the copyright holders may not be used to endorse or promote products
28 // derived from this software without specific prior written permission.
30 // This software is provided by the copyright holders and contributors "as is" and
31 // any express or implied warranties, including, but not limited to, the implied
32 // warranties of merchantability and fitness for a particular purpose are disclaimed.
33 // In no event shall the Intel Corporation or contributors be liable for any direct,
34 // indirect, incidental, special, exemplary, or consequential damages
35 // (including, but not limited to, procurement of substitute goods or services;
36 // loss of use, data, or profits; or business interruption) however caused
37 // and on any theory of liability, whether in contract, strict liability,
38 // or tort (including negligence or otherwise) arising in any way out of
39 // the use of this software, even if advised of the possibility of such damage.
43 #include "../precomp.hpp"
44 #include "layers_common.hpp"
45 #include "../op_halide.hpp"
46 #include "../op_inf_engine.hpp"
47 #include "../op_vkcom.hpp"
48 #include "opencv2/imgproc.hpp"
49 #include "opencv2/dnn/shape_utils.hpp"
50 #include "opencv2/core/hal/hal.hpp"
54 #include "opencl_kernels_dnn.hpp"
55 using namespace cv::dnn::ocl4dnn;
63 class LRNLayerImpl CV_FINAL : public LRNLayer
66 LRNLayerImpl(const LayerParams& params)
68 setParamsFrom(params);
70 String nrmType = params.get<String>("norm_region", "ACROSS_CHANNELS");
71 if (nrmType == "ACROSS_CHANNELS")
73 else if (nrmType == "WITHIN_CHANNEL")
76 CV_Error(Error::StsBadArg, "Unknown region type \"" + nrmType + "\"");
78 size = params.get<int>("local_size", 5);
79 if (size % 2 != 1 || size <= 0)
80 CV_Error(Error::StsBadArg, "LRN layer supports only positive odd values for local_size");
82 alpha = params.get<double>("alpha", 1);
83 beta = params.get<double>("beta", 0.75);
84 bias = params.get<double>("bias", 1);
85 normBySize = params.get<bool>("norm_by_size", true);
89 Ptr<OCL4DNNLRN<float> > lrnOp;
92 virtual bool supportBackend(int backendId) CV_OVERRIDE
94 if (backendId == DNN_BACKEND_INFERENCE_ENGINE)
95 return bias == (int)bias;
96 return backendId == DNN_BACKEND_OPENCV ||
97 backendId == DNN_BACKEND_HALIDE ||
98 (backendId == DNN_BACKEND_VKCOM && haveVulkan() && (size % 2 == 1) && (type == CHANNEL_NRM));
102 virtual void finalize(InputArrayOfArrays, OutputArrayOfArrays) CV_OVERRIDE
107 bool forward_ocl(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals)
109 std::vector<UMat> inputs;
110 std::vector<UMat> outputs;
112 bool use_half = (inps.depth() == CV_16S);
113 inps.getUMatVector(inputs);
114 outs.getUMatVector(outputs);
118 OCL4DNNLRNConfig config;
119 config.lrn_type = type == CHANNEL_NRM ?
120 LRNParameter_NormRegion_ACROSS_CHANNELS :
121 LRNParameter_NormRegion_WITHIN_CHANNEL;
123 CHECK_EQ(size % 2, 1)<< "LRN only supports odd values for local_size";
124 config.local_size = size;
125 config.alpha = alpha;
128 CHECK_EQ(4, inputs[0].dims) << "Input must have 4 axes, "
129 << "corresponding to (num, channels, height, width)";
130 config.batch_size = inputs[0].size[0];
131 config.channels = inputs[0].size[1];
132 config.height = inputs[0].size[2];
133 config.width = inputs[0].size[3];
134 config.norm_by_size = normBySize;
135 config.use_half = use_half;
137 lrnOp = Ptr<OCL4DNNLRN<float> >(new OCL4DNNLRN<float>(config));
140 if (!lrnOp->Forward(inputs[0], outputs[0]))
147 void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE
150 CV_TRACE_ARG_VALUE(name, "name", name.c_str());
152 CV_Assert(inputs_arr.total() == outputs_arr.total());
154 CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget),
155 forward_ocl(inputs_arr, outputs_arr, internals_arr))
157 if (inputs_arr.depth() == CV_16S)
159 forward_fallback(inputs_arr, outputs_arr, internals_arr);
163 std::vector<Mat> inputs, outputs;
164 inputs_arr.getMatVector(inputs);
165 outputs_arr.getMatVector(outputs);
167 CV_Assert(inputs.size() == outputs.size());
169 for (int i = 0; i < inputs.size(); i++)
171 CV_Assert(inputs[i].dims == 4);
173 Mat &src = inputs[i];
174 Mat &dst = outputs[i];
179 channelNormalization(src, dst);
182 spatialNormalization(src, dst);
185 CV_Error(Error::StsNotImplemented, "Unimplemented mode of LRN layer");
191 class ChannelLRN : public ParallelLoopBody
194 ChannelLRN(const float* src, float* dst, int channels, int ksize,
195 float alpha1, float bias1, float beta1,
196 size_t planeSize, int nsamples, int nstripes)
198 src_ = src; dst_ = dst;
199 channels_ = channels;
201 alpha1_ = alpha1; bias1_ = bias1; beta1_ = beta1;
202 planeSize_ = planeSize; nsamples_ = nsamples; nstripes_ = nstripes;
205 void operator()(const Range& r) const CV_OVERRIDE
207 int nsamples = nsamples_, nstripes = nstripes_;
208 size_t planeSize = planeSize_, planeSize_n = planeSize * nsamples;
209 size_t elemsPerStripe = (planeSize_n + nstripes - 1)/nstripes;
210 size_t rstart = r.start*elemsPerStripe;
211 size_t rend = r.end == nstripes ? planeSize_n : r.end*elemsPerStripe;
212 rstart = std::min(rstart, planeSize_n);
213 rend = std::min(rend, planeSize_n);
214 float alpha1 = alpha1_, bias1 = bias1_, beta1 = beta1_;
215 int k, channels = channels_, ksize = ksize_;
217 AutoBuffer<float> buf_((channels + ksize + 1)*2);
218 float* acc = buf_.data();
219 float* buf = acc + channels + ksize + 1;
220 for( k = 0; k <= ksize; k++ )
221 buf[-k-1] = buf[channels + k] = 0.f;
223 for( size_t ofs = rstart; ofs < rend; )
225 int sampleIdx = (int)(ofs/planeSize);
226 if( sampleIdx >= nsamples )
228 size_t ofs0 = ofs - sampleIdx*planeSize;
229 size_t ofs1 = std::min(planeSize - ofs0, rend - ofs) + ofs;
230 const float* src = src_ + sampleIdx*planeSize*channels + ofs0;
231 float* dst = dst_ + sampleIdx*planeSize*channels + ofs0;
233 for( ; ofs < ofs1; ofs++, src++, dst++ )
235 for( k = 0; k < channels; k++ )
236 buf[k] = src[k*planeSize];
238 for( k = 0; k < ksize; k++ )
240 for( k = 0; k < channels; k++ )
242 float x1 = buf[k + ksize];
243 float x0 = buf[k - ksize - 1];
244 s = std::max(s + (x1 + x0)*(x1 - x0), 0.f);
245 acc[k] = (float)(alpha1*s + bias1);
248 hal::log32f(acc, acc, channels);
249 for( k = 0; k < channels; k++ )
251 hal::exp32f(acc, acc, channels);
253 for( k = 0; k < channels; k++ )
254 dst[k*planeSize] = buf[k]*acc[k];
261 float alpha1_, bias1_, beta1_;
263 int channels_, ksize_, nsamples_, nstripes_;
266 void channelNormalization(Mat &srcBlob, Mat &dstBlob)
268 int num = srcBlob.size[0];
269 int channels = srcBlob.size[1];
270 int ksize = (size - 1) / 2;
271 int sizeNormFactor = normBySize ? size : 1;
272 size_t planeSize = srcBlob.size[2]*srcBlob.size[3];
274 int nstripes = std::max(getNumThreads(), 1);
276 ChannelLRN clrn(srcBlob.ptr<float>(), dstBlob.ptr<float>(), channels,
277 ksize, alpha/sizeNormFactor, bias, -beta, planeSize, num, nstripes);
278 parallel_for_(Range(0, nstripes), clrn, nstripes);
281 void sqrBoxFilter_(const Mat &src, Mat &dst)
283 Mat srcRawWrapper(src.rows, src.cols, src.type(), src.data, src.step[0]);
284 cv::sqrBoxFilter(srcRawWrapper, dst, dst.depth(), Size(size, size), Point(-1, -1), false, BORDER_CONSTANT);
287 void spatialNormalization(Mat &srcBlob, Mat &dstBlob)
289 int num = srcBlob.size[0];
290 int channels = srcBlob.size[1];
291 int sizeNormFactor = normBySize ? size*size : 1;
293 Mat srcMat = srcBlob;
294 Mat dstMat = dstBlob;
296 for (int n = 0; n < num; n++)
298 for (int cn = 0; cn < channels; cn++)
300 Mat src = getPlane(srcMat, n, cn);
301 Mat dst = getPlane(dstMat, n, cn);
303 sqrBoxFilter_(src, dst);
305 dst.convertTo(dst, dst.type(), alpha/sizeNormFactor, bias);
306 cv::pow(dst, beta, dst);
307 cv::divide(src, dst, dst);
312 virtual Ptr<BackendNode> initVkCom(const std::vector<Ptr<BackendWrapper> > &inputs) CV_OVERRIDE
315 std::shared_ptr<vkcom::OpBase> op(new vkcom::OpLRN(size / 2, bias, alpha, beta, normBySize));
316 return Ptr<BackendNode>(new VkComBackendNode(inputs, op));
318 return Ptr<BackendNode>();
321 virtual Ptr<BackendNode> initHalide(const std::vector<Ptr<BackendWrapper> > &inputs) CV_OVERRIDE
324 float alphaSize = alpha;
326 alphaSize /= (type == CHANNEL_NRM ? size : size * size);
327 int width, height, channels, numImgs;
328 Halide::Buffer<float> inputBuffer = halideBuffer(inputs[0]);
329 getCanonicalSize(inputBuffer, &width, &height, &channels, &numImgs);
331 Halide::Var x("x"), y("y"), c("c"), n("n");
332 Halide::Func top = (name.empty() ? Halide::Func() : Halide::Func(name));
333 Halide::Func padded_sq(name + "_padded_sq");
334 Halide::Func sq("sq");
335 sq(x, y, c, n) = inputBuffer(x, y, c, n) * inputBuffer(x, y, c, n);
337 Halide::Func bounded =
338 Halide::BoundaryConditions::constant_exterior(sq, 0, 0, width,
342 padded_sq(x, y, c, n) = bounded(x, y, c, n);
345 if (type == CHANNEL_NRM)
347 Halide::RDom r((1 - size) / 2, size);
348 base = alphaSize * sum(padded_sq(x, y, c + r, n));
352 Halide::RDom r((1 - size) / 2, size, (1 - size) / 2, size);
353 base = alphaSize * sum(padded_sq(x + r.x, y + r.y, c, n));
355 base += static_cast<float>(bias);
356 top(x, y, c, n) = inputBuffer(x, y, c, n) / pow(base, beta);
357 return Ptr<BackendNode>(new HalideBackendNode({ padded_sq, top }));
358 #endif // HAVE_HALIDE
359 return Ptr<BackendNode>();
362 virtual void applyHalideScheduler(Ptr<BackendNode>& node,
363 const std::vector<Mat*> &inputs,
364 const std::vector<Mat> &outputs,
365 int targetId) const CV_OVERRIDE
368 if (targetId != DNN_TARGET_CPU)
370 Layer::applyHalideScheduler(node, inputs, outputs, targetId);
373 int outW, outH, outC, outN;
374 getCanonicalSize(outputs[0].size, &outW, &outH, &outC, &outN);
376 Halide::Var x("x"), y("y"), c("c"), n("n"), yo("yo"), yi("yi"), tile("tile");
377 Halide::Func& top = node.dynamicCast<HalideBackendNode>()->funcs[1];
378 Halide::Func& padded_sq = node.dynamicCast<HalideBackendNode>()->funcs[0];
380 if (outW < 8 || outH <= 2)
383 top.reorder(x, c, y, n)
389 padded_sq.store_at(top, tile)
390 .compute_at(top, yi);
391 #endif // HAVE_HALIDE
394 #ifdef HAVE_INF_ENGINE
395 virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> >&) CV_OVERRIDE
397 float alphaSize = alpha;
399 alphaSize *= (type == SPATIAL_NRM ? size*size : size);
401 InferenceEngine::Builder::NormLayer ieLayer(name);
402 ieLayer.setSize(size);
403 ieLayer.setAlpha(alphaSize);
404 ieLayer.setBeta(beta);
405 ieLayer.setAcrossMaps(type == CHANNEL_NRM);
407 InferenceEngine::Builder::Layer l = ieLayer;
408 l.getParameters()["k"] = bias;
409 return Ptr<BackendNode>(new InfEngineBackendNode(l));
411 #endif // HAVE_INF_ENGINE
413 virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
414 const std::vector<MatShape> &outputs) const CV_OVERRIDE
416 CV_UNUSED(outputs); // suppress unused variable warning
417 CV_Assert(inputs.size() > 0);
420 for(int i = 0; i < inputs.size(); i++)
422 if (type == CHANNEL_NRM)
424 int channels = inputs[i][1];
425 int ksize = (size - 1) / 2;
427 flops += inputs[i][0]*(std::min(ksize, channels)*2*total(inputs[i], 2) + channels*4*total(inputs[i], 2));
429 if (ksize < channels)
431 flops += (size + 2*(channels - size))*total(inputs[i], 2);
436 flops += total(inputs[i])*(2*size*size + 2);
450 Ptr<LRNLayer> LRNLayer::create(const LayerParams& params)
452 return Ptr<LRNLayer>(new LRNLayerImpl(params));