modules/dnn/src/layers/convolution_layer.cpp

   1 /*M///////////////////////////////////////////////////////////////////////////////////////
   2 //
   3 //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
   4 //
   5 //  By downloading, copying, installing or using the software you agree to this license.
   6 //  If you do not agree to this license, do not download, install,
   7 //  copy or use the software.
   8 //
   9 //
  10 //                           License Agreement
  11 //                For Open Source Computer Vision Library
  12 //
  13 // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
  14 // Copyright (C) 2017, Intel Corporation, all rights reserved.
  15 // Third party copyrights are property of their respective owners.
  16 //
  17 // Redistribution and use in source and binary forms, with or without modification,
  18 // are permitted provided that the following conditions are met:
  19 //
  20 //   * Redistribution's of source code must retain the above copyright notice,
  21 //     this list of conditions and the following disclaimer.
  22 //
  23 //   * Redistribution's in binary form must reproduce the above copyright notice,
  24 //     this list of conditions and the following disclaimer in the documentation
  25 //     and/or other materials provided with the distribution.
  26 //
  27 //   * The name of the copyright holders may not be used to endorse or promote products
  28 //     derived from this software without specific prior written permission.
  29 //
  30 // This software is provided by the copyright holders and contributors "as is" and
  31 // any express or implied warranties, including, but not limited to, the implied
  32 // warranties of merchantability and fitness for a particular purpose are disclaimed.
  33 // In no event shall the Intel Corporation or contributors be liable for any direct,
  34 // indirect, incidental, special, exemplary, or consequential damages
  35 // (including, but not limited to, procurement of substitute goods or services;
  36 // loss of use, data, or profits; or business interruption) however caused
  37 // and on any theory of liability, whether in contract, strict liability,
  38 // or tort (including negligence or otherwise) arising in any way out of
  39 // the use of this software, even if advised of the possibility of such damage.
  40 //
  41 //M*/
  42
  43 #include "../precomp.hpp"
  44 #include "layers_common.hpp"
  45 #include "../op_halide.hpp"
  46 #include "../op_inf_engine.hpp"
  47 #include "../op_vkcom.hpp"
  48 #include "opencv2/core/hal/hal.hpp"
  49 #include "opencv2/core/hal/intrin.hpp"
  50 #include <iostream>
  51 #include <numeric>
  52
  53 #ifdef HAVE_OPENCL
  54 #include "opencl_kernels_dnn.hpp"
  55 using namespace cv::dnn::ocl4dnn;
  56 #endif
  57
  58 namespace cv
  59 {
  60 namespace dnn
  61 {
  62
  63 class BaseConvolutionLayerImpl : public ConvolutionLayer
  64 {
  65 public:
  66     bool fusedWeights, fusedBias;
  67     std::vector<double> weightsMultipliers;
  68     BaseConvolutionLayerImpl(const LayerParams &params)
  69     {
  70         setParamsFrom(params);
  71         getConvolutionKernelParams(params, kernel_size, pads_begin, pads_end, strides, dilations, padMode, adjust_pads);
  72
  73         numOutput = params.get<int>("num_output");
  74         int ngroups = params.get<int>("group", 1);
  75         CV_Assert(numOutput % ngroups == 0);
  76
  77         if (kernel_size.size() == 2) {
  78             kernel = Size(kernel_size[1], kernel_size[0]);
  79             stride = Size(strides[1], strides[0]);
  80             for (int i = 0; i < pads_begin.size(); i++) {
  81                 if (pads_begin[i] != pads_end[i])
  82                     CV_Error(Error::StsNotImplemented, "Unsupported asymmetric padding in convolution layer");
  83             }
  84             pad = Size(pads_begin[1], pads_begin[0]);
  85             dilation = Size(dilations[1], dilations[0]);
  86
  87             adjustPad.height = adjust_pads[0];
  88             adjustPad.width = adjust_pads[1];
  89         }
  90
  91         for (int i = 0; i < adjust_pads.size(); i++) {
  92             CV_Assert(adjust_pads[i] < strides[i]);
  93         }
  94
  95         fusedWeights = false;
  96         fusedBias = false;
  97     }
  98
  99     virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE
 100     {
 101         std::vector<Mat> inputs, outputs;
 102         inputs_arr.getMatVector(inputs);
 103         outputs_arr.getMatVector(outputs);
 104
 105         CV_Assert(inputs.size() > 0);
 106
 107         CV_Assert(blobs.size() == 1 || blobs.size() == 2);
 108         CV_Assert(inputs[0].dims == outputs[0].dims);
 109         CV_Assert(blobs[0].dims == kernel_size.size() + 2);
 110         for (int i = 0; i < kernel_size.size(); i++) {
 111             CV_Assert(blobs[0].size[i + 2] == kernel_size[i]);
 112         }
 113
 114         const Mat &input = inputs[0];
 115         CV_Assert((input.dims == 4 || input.dims == 5) && (input.type() == CV_32F || input.type() == CV_16S));
 116         for (size_t i = 0; i < inputs.size(); i++)
 117         {
 118             CV_Assert(inputs[i].type() == input.type());
 119             CV_Assert((inputs[i].dims == 4 || inputs[i].dims == 5) && inputs[i].size[1] == input.size[1]);
 120             for (int j = 0; j < inputs[i].dims; j++) {
 121                 CV_Assert(inputs[i].size[j] == input.size[j]);
 122             }
 123         }
 124
 125         std::vector<int> inpShape;
 126         std::vector<int> outShape;
 127         for (int i = 2; i < inputs[0].dims; i++) {
 128             inpShape.push_back(inputs[0].size[i]);
 129             outShape.push_back(outputs[0].size[i]);
 130         }
 131         getConvPoolPaddings(inpShape, kernel_size, strides, padMode, pads_begin, pads_end);
 132         if (pads_begin.size() == 2) {
 133             for (int i = 0; i < pads_begin.size(); i++) {
 134                 if (pads_begin[i] != pads_end[i])
 135                     CV_Error(Error::StsNotImplemented, "Unsupported asymmetric padding in convolution layer");
 136             }
 137             pad = Size(pads_begin[1], pads_begin[0]);
 138         }
 139         fusedWeights = false;
 140         fusedBias = false;
 141     }
 142
 143     bool hasBias() const
 144     {
 145         return blobs.size() >= 2;
 146     }
 147
 148     virtual MatShape computeColRowShape(const MatShape &inpShape, const MatShape &outShape) const = 0;
 149     bool is1x1() const
 150     {
 151         return (kernel.height == 1 && kernel.width == 1) &&
 152                (stride.height == 1 && stride.width == 1) &&
 153                (dilation.height == 1 && dilation.width == 1);
 154     }
 155
 156     virtual bool tryFuse(Ptr<Layer>& top) CV_OVERRIDE
 157     {
 158         Mat w, b;
 159         top->getScaleShift(w, b);
 160         if (!w.empty() || !b.empty())
 161         {
 162             fuseWeights(w, b);
 163             fusedWeights = fusedWeights || !w.empty();
 164             fusedBias = fusedBias || (hasBias() && !w.empty()) || !b.empty();
 165             return true;
 166         }
 167         return false;
 168     }
 169
 170     virtual void fuseWeights(const Mat& w_, const Mat& b_) = 0;
 171
 172     virtual void applyHalideScheduler(Ptr<BackendNode>& node,
 173                                       const std::vector<Mat*> &inputs,
 174                                       const std::vector<Mat> &outputs,
 175                                       int targetId) const CV_OVERRIDE
 176     {
 177 #ifdef HAVE_HALIDE
 178         if (targetId != DNN_TARGET_CPU)
 179         {
 180             Layer::applyHalideScheduler(node, inputs, outputs, targetId);
 181             return;
 182         }
 183         Halide::Var x("x"), y("y"), c("c"), n("n"), tile("tile"), yi("yi"), yo("yo"), co("co"), ci("ci");
 184         Halide::Func& top = node.dynamicCast<HalideBackendNode>()->funcs[1];
 185         Halide::Func& padded_input = node.dynamicCast<HalideBackendNode>()->funcs[0];
 186
 187         int outW, outH, outC, outN;
 188         getCanonicalSize(outputs[0].size, &outW, &outH, &outC, &outN);
 189
 190         if (outW == 1 || outH <= 2)
 191             return;
 192
 193         if (is1x1() || outC <= 16)
 194             top.reorder(x, c, y)
 195                .split(y, yo, yi, 2)
 196                .fuse(yo, n, tile)
 197                .parallel(tile)
 198                .unroll(yi)
 199                .vectorize(x, outW >= 16 ? 16 : outW);
 200         else
 201             top.reorder(x, c, y)
 202                .split(y, yo, yi, 2)
 203                .split(c, co, ci, 16)
 204                .fuse(yo, co, tile).fuse(n, tile, tile)
 205                .parallel(tile)
 206                .unroll(yi)
 207                .vectorize(x, outW >= 16 ? 16 : outW);
 208         padded_input.compute_at(top, yi);
 209 #endif  // HAVE_HALIDE
 210     }
 211 };
 212
 213
 214 #define IS_POWER_LAYER(layer) \
 215             (!layer.empty() && !layer->type.compare("Power"))
 216 //TODO: simultaneously convolution and bias addition for cache optimization
 217 class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl
 218 {
 219 public:
 220     enum { VEC_ALIGN = 8, DFT_TYPE = CV_32F };
 221     Mat weightsMat;
 222     std::vector<float> biasvec;
 223     std::vector<float> reluslope;
 224     Ptr<ActivationLayer> activ;
 225
 226 #ifdef HAVE_OPENCL
 227     Ptr<OCL4DNNConvSpatial<float> > convolutionOp;
 228     std::vector<UMat> umat_blobs;
 229     bool newActiv;
 230     ocl4dnnFusedActiv_t activType;
 231     float power;
 232 #endif
 233     ConvolutionLayerImpl(const LayerParams &params) : BaseConvolutionLayerImpl(params)
 234     {
 235 #ifdef HAVE_OPENCL
 236         newActiv = false;
 237         activType = OCL4DNN_CONV_FUSED_ACTIV_NONE;
 238         power = 0.f;
 239 #endif
 240     }
 241
 242     MatShape computeColRowShape(const MatShape &inpShape, const MatShape &outShape) const CV_OVERRIDE
 243     {
 244         int dims = inpShape.size();
 245         int inpD = dims == 5 ? inpShape[2] : 1;
 246         int inpH = inpShape[dims - 2];
 247         int inpW = inpShape.back();
 248         int inpGroupCn = blobs[0].size[1];
 249         int ksize = inpGroupCn * std::accumulate(kernel_size.begin(), kernel_size.end(),
 250                                                  1, std::multiplies<size_t>());
 251         return shape(inpD * inpH * inpW, ksize);
 252     }
 253
 254     virtual bool supportBackend(int backendId) CV_OVERRIDE
 255     {
 256 #ifdef HAVE_INF_ENGINE
 257         if (backendId == DNN_BACKEND_INFERENCE_ENGINE)
 258         {
 259             if (kernel_size.size() == 3)
 260                 return preferableTarget == DNN_TARGET_CPU;
 261             return (preferableTarget != DNN_TARGET_MYRIAD || dilation.width == dilation.height);
 262         }
 263         else
 264 #endif
 265         {
 266             if (kernel_size.size() == 3)
 267                 return (preferableTarget == DNN_TARGET_CPU && backendId == DNN_BACKEND_OPENCV);
 268             else if (kernel_size.size() == 2)
 269                 return backendId == DNN_BACKEND_OPENCV ||
 270                        backendId == DNN_BACKEND_HALIDE ||
 271                        (backendId == DNN_BACKEND_VKCOM && haveVulkan());
 272             else
 273                 return false;
 274         }
 275     }
 276
 277     bool getMemoryShapes(const std::vector<MatShape> &inputs,
 278                          const int requiredOutputs,
 279                          std::vector<MatShape> &outputs,
 280                          std::vector<MatShape> &internals) const CV_OVERRIDE
 281     {
 282         CV_Assert(blobs.size() != 0);
 283         CV_Assert(!hasBias() || blobs[1].total() == (size_t)blobs[0].size[0]);
 284         CV_Assert(inputs.size() == (size_t)1);
 285
 286         internals.clear();
 287
 288         CV_Assert(inputs.size() != 0);
 289         std::vector<int> inpShape(inputs[0].begin() + 2, inputs[0].end());
 290
 291         int outCn = blobs[0].size[0];
 292         std::vector<int> outShape;
 293         outShape.push_back(inputs[0][0]);
 294         outShape.push_back(outCn);
 295
 296         int inpCn = inputs[0][1];
 297         if (padMode.empty())
 298         {
 299             for (int i = 0; i < inpShape.size(); i++)
 300                 outShape.push_back((inpShape[i] + pads_begin[i] + pads_end[i] - dilations[i] * (kernel_size[i] - 1) - 1) / strides[i] + 1);
 301         }
 302         else
 303         {
 304             getConvPoolOutParams(inpShape, kernel_size, strides, padMode, dilations, outShape);
 305         }
 306
 307         int ngroups = inpCn / blobs[0].size[1];
 308         if (ngroups == 0 || ngroups * blobs[0].size[1] != inpCn)
 309             CV_Error(Error::StsError, format("Number of input channels should "
 310                      "be multiple of %d but got %d", blobs[0].size[1], inpCn));
 311         CV_Assert(ngroups > 0 && inpCn % ngroups == 0 && outCn % ngroups == 0);
 312
 313         outputs.resize(1, outShape);
 314
 315         return false;
 316     }
 317
 318     virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE
 319     {
 320         BaseConvolutionLayerImpl::finalize(inputs_arr, outputs_arr);
 321
 322         CV_Assert(!blobs.empty());
 323         const int outCn = blobs[0].size[0];
 324         // prepare weightsMat where each row is aligned and has enough zero padding on the right to
 325         // use vectorized (i.e. with intrinsics) loops without tail processing
 326         Mat wm = blobs[0].reshape(1, outCn);
 327         if( wm.step1() % VEC_ALIGN != 0 )
 328         {
 329             int newcols = (int)alignSize(wm.step1(), VEC_ALIGN);
 330             Mat wm_buffer = Mat(outCn, newcols, wm.type());
 331             Mat wm_padding = wm_buffer.colRange(wm.cols, newcols);
 332             wm_padding.setTo(Scalar::all(0.));
 333             Mat wm_aligned = wm_buffer.colRange(0, wm.cols);
 334             wm.copyTo(wm_aligned);
 335             wm = wm_aligned;
 336         }
 337         weightsMat = wm;
 338         weightsMultipliers.assign(outCn, 1.0);
 339
 340         Mat biasMat = hasBias() ? blobs[1].reshape(1, outCn) : Mat();
 341         biasvec.resize(outCn+2);
 342         if( biasMat.empty() )
 343         {
 344             for(int i = 0; i < outCn; i++ )
 345                 biasvec[i] = 0.f;
 346         }
 347         else
 348         {
 349             for(int i = 0; i < outCn; i++ )
 350                 biasvec[i] = biasMat.at<float>(i);
 351         }
 352 #ifdef HAVE_OPENCL
 353         convolutionOp.release();
 354 #endif
 355     }
 356
 357     bool setActivation(const Ptr<ActivationLayer>& layer) CV_OVERRIDE
 358     {
 359         if (!activ.empty() && !layer.empty())
 360             return false;
 361
 362         activ = layer;
 363         if (activ.empty())
 364             reluslope.clear();
 365 #ifdef HAVE_OPENCL
 366         newActiv = true;
 367         activType = OCL4DNN_CONV_FUSED_ACTIV_NONE;
 368
 369         if (IS_DNN_OPENCL_TARGET(preferableTarget))
 370         {
 371             Ptr<PowerLayer> activ_power = activ.dynamicCast<PowerLayer>();
 372             if (!activ_power.empty())
 373             {
 374                 if (activ_power->scale != 1.f || activ_power->shift != 0.f)
 375                 {
 376                     const int outCh = blobs[0].size[0];
 377                     fuseWeights(Mat(1, outCh, CV_32F, Scalar(activ_power->scale)),
 378                                 Mat(1, outCh, CV_32F, Scalar(activ_power->shift)));
 379                 }
 380
 381                 power = activ_power->power;
 382                 activType = OCL4DNN_CONV_FUSED_ACTIV_POWER;
 383             }
 384             Ptr<TanHLayer> activ_tanh = activ.dynamicCast<TanHLayer>();
 385             if (!activ_tanh.empty())
 386             {
 387                 activType = OCL4DNN_CONV_FUSED_ACTIV_TANH;
 388             }
 389         }
 390 #endif
 391         return !activ.empty();
 392     }
 393
 394     void fuseWeights(const Mat& w_, const Mat& b_) CV_OVERRIDE
 395     {
 396         // Convolution weights have OIHW data layout. Parameters fusion in case of
 397         // (conv(I) + b1 ) * w + b2
 398         // means to replace convolution's weights to [w*conv(I)] and bias to [b1 * w + b2]
 399         const int outCn = weightsMat.size[0];
 400         Mat w = w_.total() == 1 ? Mat(1, outCn, CV_32F, Scalar(w_.at<float>(0))) : w_;
 401         Mat b = b_.total() == 1 ? Mat(1, outCn, CV_32F, Scalar(b_.at<float>(0))) : b_;
 402         CV_Assert_N(!weightsMat.empty(), biasvec.size() == outCn + 2,
 403                     w.empty() || outCn == w.total(), b.empty() || outCn == b.total());
 404
 405         if (!w.empty())
 406         {
 407             // Keep origin weights unchanged.
 408             if (weightsMat.data == blobs[0].data)
 409                 weightsMat = weightsMat.clone();
 410
 411             Mat originWeights = blobs[0].reshape(1, outCn);
 412             for (int i = 0; i < outCn; ++i)
 413             {
 414                 double wi = w.at<float>(i);
 415                 weightsMultipliers[i] *= wi;
 416                 cv::multiply(originWeights.row(i), weightsMultipliers[i], weightsMat.row(i));
 417                 biasvec[i] *= wi;
 418             }
 419         }
 420
 421         if (!b.empty())
 422         {
 423             for (int i = 0; i < outCn; ++i)
 424                 biasvec[i] += b.at<float>(i);
 425         }
 426         biasvec[outCn] = biasvec[outCn+1] = biasvec[outCn-1];
 427     }
 428
 429     virtual Ptr<BackendNode> initVkCom(const std::vector<Ptr<BackendWrapper> > &inputs) CV_OVERRIDE
 430     {
 431 #ifdef HAVE_VULKAN
 432         int out_channel = blobs[0].size[0];
 433         bool has_bias = hasBias() || fusedBias;
 434         int filter_size[2] = {kernel.height, kernel.width};
 435         int pad_size[2] = {pad.height, pad.width};
 436         int stride_size[2] = {stride.height, stride.width};
 437         int dilation_size[2] = {dilation.height, dilation.width};
 438         int activation = 0;
 439         vkcom::Tensor input_tensor = VkComTensor(inputs[0]);
 440         int in_channel = input_tensor.dimSize(1);
 441         int group = in_channel / blobs[0].size[1];
 442
 443         // TODO: support group > 1
 444         if (group != 1)
 445             return Ptr<BackendNode>();
 446
 447         int padding_mode;
 448         if (padMode.empty())
 449         {
 450             padding_mode = vkcom::kPaddingModeCaffe;
 451         }
 452         else if (padMode == "VALID")
 453         {
 454             padding_mode = vkcom::kPaddingModeValid;
 455         }
 456         else if (padMode == "SAME")
 457         {
 458             padding_mode = vkcom::kPaddingModeSame;
 459         }
 460         else
 461             CV_Error(Error::StsError, "Unsupported padding mode " + padMode);
 462
 463         std::shared_ptr<vkcom::OpBase> op(new vkcom::OpConv(out_channel, has_bias,
 464                     filter_size, pad_size,
 465                     stride_size, dilation_size,
 466                     activation, group,
 467                     padding_mode));
 468
 469         std::vector<Ptr<BackendWrapper> > blobsWrapper;
 470
 471         if (fusedWeights)
 472         {
 473             Mat wm;
 474             weightsMat.copyTo(wm); // to handle the case of isContinuous() == false
 475             wm = wm.reshape(1, blobs[0].dims, blobs[0].size);
 476             blobsWrapper.push_back(Ptr<BackendWrapper>(new VkComBackendWrapper(wm)));
 477         }
 478         else
 479         {
 480             blobsWrapper.push_back(Ptr<BackendWrapper>(new VkComBackendWrapper(blobs[0])));
 481         }
 482
 483         if (has_bias)
 484         {
 485             Mat biasesMat({out_channel}, CV_32F, &biasvec[0]);
 486             blobsWrapper.push_back(Ptr<BackendWrapper>(new VkComBackendWrapper(biasesMat)));
 487         }
 488
 489         return Ptr<BackendNode>(new VkComBackendNode(inputs, op, blobsWrapper));
 490 #endif  // HAVE_VULKAN
 491         return Ptr<BackendNode>();
 492     }
 493
 494
 495
 496     virtual Ptr<BackendNode> initHalide(const std::vector<Ptr<BackendWrapper> > &inputs) CV_OVERRIDE
 497     {
 498 #ifdef HAVE_HALIDE
 499         Halide::Buffer<float> inputBuffer = halideBuffer(inputs[0]);
 500
 501         const int inpCn = inputBuffer.channels();
 502         const int outCn = blobs[0].size[0];
 503         const int inpGroupCn = blobs[0].size[1];
 504         const int group = inpCn / inpGroupCn;
 505         const int outGroupCn = outCn / group;
 506
 507         Halide::Buffer<float> weights = wrapToHalideBuffer(blobs[0]);
 508
 509         Halide::Var x("x"), y("y"), c("c"), n("n");
 510         Halide::Func top = (name.empty() ? Halide::Func() : Halide::Func(name));
 511         Halide::Func padded_input(name + "_constant_exterior");
 512         if (pad.width || pad.height)
 513         {
 514             Halide::Func bounded =
 515                 Halide::BoundaryConditions::constant_exterior(inputBuffer, 0);
 516             padded_input(x, y, c, n) = bounded(x, y, c, n);
 517         }
 518         else
 519         {
 520             padded_input(x, y, c, n) = inputBuffer(x, y, c, n);
 521         }
 522
 523         Halide::RDom r(0, kernel.width, 0, kernel.height, 0, inpGroupCn);
 524         Halide::Expr kx = x * stride.width - pad.width + r.x * dilation.width;
 525         Halide::Expr ky = y * stride.height - pad.height + r.y * dilation.height;
 526         Halide::Expr kc = r.z;
 527         for (int i = 1; i < group; ++i)
 528         {
 529             kc = select(c < outGroupCn * i, kc, inpGroupCn * i + r.z);
 530         }
 531         Halide::Expr topExpr = sum(padded_input(kx, ky, kc, n) *
 532                                    weights(r.x, r.y, r.z, c));
 533         if (hasBias())
 534         {
 535             Halide::Buffer<float> bias = wrapToHalideBuffer(blobs[1], {outCn});
 536             topExpr += bias(c);
 537         }
 538         top(x, y, c, n) = topExpr;
 539         return Ptr<BackendNode>(new HalideBackendNode({ padded_input, top }));
 540 #endif  // HAVE_HALIDE
 541         return Ptr<BackendNode>();
 542     }
 543
 544 #ifdef HAVE_INF_ENGINE
 545     virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> > &inputs) CV_OVERRIDE
 546     {
 547         InferenceEngine::DataPtr input = infEngineDataNode(inputs[0]);
 548         std::vector<size_t> dims = input->getDims();
 549         CV_Assert(dims.size() == 4 || dims.size() == 5);
 550         const int inpCn = dims[1];
 551         const int outCn = blobs[0].size[0];
 552         const int inpGroupCn = blobs[0].size[1];
 553         const int group = inpCn / inpGroupCn;
 554         InferenceEngine::Layout layout = (dims.size() == 4) ? InferenceEngine::Layout::OIHW :
 555                                                               InferenceEngine::Layout::NCDHW;
 556
 557         auto ieWeights = wrapToInfEngineBlob(blobs[0], layout);
 558         if (fusedWeights)
 559         {
 560             if (weightsMat.isContinuous())
 561             {
 562                 Mat cvWeights = weightsMat.reshape(1, blobs[0].dims, blobs[0].size);
 563                 ieWeights = wrapToInfEngineBlob(cvWeights, layout);
 564             }
 565             else
 566             {
 567                 ieWeights = InferenceEngine::make_shared_blob<float>({
 568                                 InferenceEngine::Precision::FP32,
 569                                 ieWeights->getTensorDesc().getDims(), layout
 570                             });
 571                 ieWeights->allocate();
 572
 573                 Mat newWeights = infEngineBlobToMat(ieWeights).reshape(1, outCn);
 574                 Mat cvWeights = weightsMat.colRange(0, newWeights.cols);
 575                 cvWeights.copyTo(newWeights);
 576             }
 577         }
 578         InferenceEngine::Blob::Ptr ieBiases;
 579         if (hasBias() || fusedBias)
 580         {
 581             Mat biasesMat({outCn}, CV_32F, &biasvec[0]);
 582             ieBiases = wrapToInfEngineBlob(biasesMat, {(size_t)outCn}, InferenceEngine::Layout::C);
 583         }
 584
 585         InferenceEngine::Builder::ConvolutionLayer ieLayer(name);
 586
 587         ieLayer.setKernel(kernel_size);
 588         ieLayer.setStrides(strides);
 589         ieLayer.setDilation(dilations);
 590         ieLayer.setPaddingsBegin(pads_begin);
 591         ieLayer.setPaddingsEnd(pads_end);
 592         ieLayer.setGroup((size_t)group);
 593         ieLayer.setOutDepth((size_t)outCn);
 594
 595         InferenceEngine::Builder::Layer l = ieLayer;
 596         addConstantData("weights", ieWeights, l);
 597         if (ieBiases)
 598             addConstantData("biases", ieBiases, l);
 599
 600         if (!padMode.empty())
 601             l.getParameters()["auto_pad"] = padMode == "VALID" ? std::string("valid") : std::string("same_upper");
 602
 603         return Ptr<BackendNode>(new InfEngineBackendNode(l));
 604     }
 605 #endif  // HAVE_INF_ENGINE
 606
 607     class ParallelConv : public cv::ParallelLoopBody
 608     {
 609     public:
 610         enum { BLK_SIZE = 32, BLK_SIZE_CN = 64 };
 611
 612         const Mat* input_;
 613         const Mat* weights_;
 614         Mat* output_;
 615         int outShape[4]; // used only for conv2d
 616         std::vector<size_t> kernel_size, pads_begin, pads_end, strides, dilations;
 617         int ngroups_, nstripes_;
 618         std::vector<int> ofstab_;
 619         const std::vector<float>* biasvec_;
 620         const std::vector<float>* reluslope_;
 621         const ActivationLayer* activ_;
 622         bool is1x1_;
 623         bool useAVX;
 624         bool useAVX2;
 625         bool useAVX512;
 626
 627         ParallelConv()
 628             : input_(0), weights_(0), output_(0), ngroups_(0), nstripes_(0),
 629               biasvec_(0), reluslope_(0), activ_(0), is1x1_(false), useAVX(false), useAVX2(false), useAVX512(false)
 630         {}
 631
 632         static void run( const Mat& input, Mat& output, const Mat& weights,
 633                          const std::vector<float>& biasvec,
 634                          const std::vector<float>& reluslope,
 635                          const std::vector<size_t>& kernel_size, const std::vector<size_t>& strides,
 636                          const std::vector<size_t>& pads_begin, const std::vector<size_t>& pads_end,
 637                          const std::vector<size_t>& dilations,
 638                          const ActivationLayer* activ, int ngroups, int nstripes )
 639         {
 640             size_t karea = std::accumulate(kernel_size.begin(), kernel_size.end(),
 641                                            1, std::multiplies<size_t>());
 642             CV_Assert_N(
 643                        (input.dims == 4 || input.dims == 5) && (input.dims == output.dims),
 644                        input.size[0] == output.size[0],
 645                        weights.rows == output.size[1],
 646                        weights.cols == (input.size[1]/ngroups)*karea,
 647                        input.type() == output.type(),
 648                        input.type() == weights.type(),
 649                        input.type() == CV_32FC1,
 650                        input.isContinuous(),
 651                        output.isContinuous(),
 652                        biasvec.size() == (size_t)output.size[1]+2);
 653             ParallelConv p;
 654
 655             p.input_ = &input;
 656             p.weights_ = &weights;
 657             p.output_ = &output;
 658             for( int i = 0; i < 4; i++ ) p.outShape[i] = output.size[i];
 659             p.outShape[1] /= ngroups;
 660
 661             p.kernel_size = kernel_size; p.strides = strides; p.dilations = dilations;
 662             p.pads_begin = pads_begin; p.pads_end = pads_end;
 663
 664             p.ngroups_ = ngroups;
 665             p.nstripes_ = nstripes;
 666
 667             int inpCnAll = input.size[1];
 668             int depth = (input.dims == 5) ? input.size[2] : 1;
 669             int width = input.size[input.dims - 1];
 670             int height = input.size[input.dims - 2];
 671             int inpCn = inpCnAll / ngroups;
 672
 673             bool isConv2D = kernel_size.size() == 2;
 674
 675             p.is1x1_ = isConv2D && kernel_size[0] == 1 && kernel_size[1] == 1 &&
 676                        pads_begin[0] == 0  && pads_begin[1] == 0;
 677
 678             p.useAVX    = checkHardwareSupport(CPU_AVX)  && isConv2D;
 679             p.useAVX2   = checkHardwareSupport(CPU_AVX2) && isConv2D;
 680             p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX  && isConv2D;
 681
 682             int ncn = std::min(inpCn, (int)BLK_SIZE_CN);
 683
 684             int kernel_d = !isConv2D? kernel_size[0] : 1;
 685             int kernel_h = kernel_size[kernel_size.size() - 2];
 686             int kernel_w = kernel_size.back();
 687
 688             int dil_d = !isConv2D? dilations[0] : 1;
 689             int dil_h = dilations[dilations.size() - 2];
 690             int dil_w = dilations.back();
 691
 692             p.ofstab_.resize(karea * ncn);
 693             int* ofstab = &p.ofstab_[0];
 694
 695             if (isConv2D)
 696             {
 697                 for( int k = 0; k < ncn; k++ )
 698                     for( int k_r = 0; k_r < kernel_h; k_r++ )
 699                         for( int k_c = 0; k_c < kernel_w; k_c++ )
 700                             ofstab[(k*kernel_h + k_r)*kernel_w + k_c] =
 701                                    (k*height + k_r*dil_h)*width + k_c*dil_w;
 702             }
 703             else
 704             {
 705                 for( int k = 0; k < ncn; k++ )
 706                     for (int k_d = 0; k_d < kernel_d; k_d++)
 707                         for( int k_r = 0; k_r < kernel_h; k_r++ )
 708                             for( int k_c = 0; k_c < kernel_w; k_c++ )
 709                                 ofstab[(k*kernel_d*kernel_h + k_d*kernel_h + k_r)*kernel_w + k_c] =
 710                                        (k*depth*height + k_d*dil_d*height + k_r*dil_h)*width + k_c*dil_w;
 711             }
 712
 713             p.biasvec_ = &biasvec;
 714             p.reluslope_ = &reluslope;
 715             p.activ_ = p.reluslope_->empty() ? activ : 0;
 716
 717             parallel_for_(Range(0, nstripes), p, nstripes);
 718         }
 719
 720         virtual void operator ()(const Range &r0) const CV_OVERRIDE
 721         {
 722             const int valign = ConvolutionLayerImpl::VEC_ALIGN;
 723             int ngroups = ngroups_, batchSize = input_->size[0]*ngroups;
 724             bool isConv2D = input_->dims == 4;
 725
 726             int outW = output_->size[output_->dims - 1];
 727             int outH = output_->size[output_->dims - 2];
 728             int outCn = output_->size[1]/ngroups;
 729
 730             int depth = !isConv2D? input_->size[2] : 1;
 731             int height = input_->size[input_->dims - 2];
 732             int width = input_->size[input_->dims - 1];
 733             int inpCn = input_->size[1]/ngroups;
 734
 735             const int nstripes = nstripes_;
 736
 737             int kernel_d = !isConv2D? kernel_size[0] : 1;
 738             int kernel_h = kernel_size[kernel_size.size() - 2];
 739             int kernel_w = kernel_size.back();
 740             int karea = kernel_w*kernel_h*kernel_d;
 741
 742             int pad_d = !isConv2D? pads_begin[0] : 0;
 743             int pad_t = pads_begin[pads_begin.size() - 2];
 744             int pad_l = pads_begin.back();
 745
 746             int stride_d = !isConv2D? strides[0] : 0;
 747             int stride_h = strides[strides.size() - 2];
 748             int stride_w = strides.back();
 749
 750             int dilation_d = !isConv2D? dilations[0] : 1;
 751             int dilation_h = dilations[dilations.size() - 2];
 752             int dilation_w = dilations.back();
 753
 754             int i, j, k, d;
 755             size_t inpPlaneSize = input_->total(2);
 756             size_t outPlaneSize = output_->total(2);
 757             bool is1x1 = is1x1_;
 758
 759             int stripesPerSample;
 760             size_t stripeSize;
 761             Range r = r0;
 762
 763             if( nstripes >= batchSize*2 )
 764             {
 765                 stripesPerSample = nstripes/batchSize;
 766                 stripeSize = alignSize((outPlaneSize + stripesPerSample - 1)/stripesPerSample, valign);
 767                 stripeSize = std::min(stripeSize, outPlaneSize);
 768             }
 769             else
 770             {
 771                 stripesPerSample = 1;
 772                 int samplesPerStripe = std::max((batchSize + nstripes - 1)/nstripes, 1);
 773                 r.start *= samplesPerStripe;
 774                 r.end *= samplesPerStripe;
 775                 stripeSize = outPlaneSize;
 776             }
 777
 778             const float* data_inp0_ = input_->ptr<float>();
 779             const int* ofstab = &ofstab_[0];
 780             const float* wptr_orig_ = weights_->ptr<float>();
 781             size_t wstep = weights_->step1();
 782             const float* biasptr_ = &biasvec_->at(0);
 783             const float* reluptr_ = reluslope_->empty() ? 0 : &reluslope_->at(0);
 784             float* data_out0_ = output_->ptr<float>();
 785             size_t rowbufsz = (size_t)karea*BLK_SIZE_CN*BLK_SIZE;
 786             AutoBuffer<float> rowbuf0_(rowbufsz + valign);
 787             float* rowbuf0 = alignPtr(rowbuf0_.data(), (int)(valign*sizeof(float)));
 788
 789             // we clear the buffer once; ultimately, it lets us to avoid
 790             // tail processing after running the unrolled/vectorized loop.
 791             // the main idea is to make sure that the tail (a.k.a. padding) of each row
 792             // (i.e. the elements with indices between vsz=karea*ncn and vsz_a)
 793             // does not contain NaNs or Infs. Because the padding in the weights
 794             // matrix is explicitly initialized with 0's, we handle all other
 795             // cases nicely, i.e. we can skip expliciting re-initialization
 796             // of the padding - we just retain elements from the previous iteration
 797             // of the loop over channels (cn0).
 798             memset(rowbuf0, 0, rowbufsz*sizeof(rowbuf0[0]) );
 799
 800             for( int stripe = r.start; stripe < r.end; stripe++ )
 801             {
 802                 int subsampleIdx = stripe/stripesPerSample;
 803                 if( subsampleIdx >= batchSize )
 804                     break;
 805                 int stripeStart = (int)((stripe - subsampleIdx*stripesPerSample)*stripeSize);
 806                 int stripeEnd = (int)std::min(stripeStart + stripeSize, outPlaneSize);
 807                 const float* data_inp0 = data_inp0_ + subsampleIdx*inpPlaneSize*inpCn;
 808                 float* data_out0 = data_out0_ + subsampleIdx*outPlaneSize*outCn;
 809                 int startOutCn = (subsampleIdx % ngroups)*outCn;
 810                 const float* wptr_orig = wptr_orig_ + wstep*startOutCn;
 811                 const float* biasptr = biasptr_ + startOutCn;
 812
 813                 for( int cn0 = 0; cn0 < inpCn; cn0 += BLK_SIZE_CN )
 814                 {
 815                     int cn1 = std::min(cn0 + BLK_SIZE_CN, inpCn);
 816                     int ncn = cn1 - cn0, vsz = karea*ncn;
 817                     int vsz_a = (int)alignSize(vsz, valign);
 818                     const float* wptr = wptr_orig + cn0*karea;
 819                     // we apply [Channels][P]ReLU (if any) during the final pass only.
 820                     const float* relu = cn1 == inpCn && reluptr_ ? reluptr_ + startOutCn : 0;
 821
 822                     for( int ofs0 = stripeStart; ofs0 < stripeEnd; ofs0 += BLK_SIZE )
 823                     {
 824                         int ofs, ofs1 = std::min(ofs0 + BLK_SIZE, stripeEnd);
 825
 826                         int out_d = ofs0 / (outH * outW);
 827                         int out_i = (ofs0 - out_d * outH * outW) / outW;
 828                         int out_j = ofs0 % outW;
 829
 830                         // do im2row for a part of input tensor
 831                         float* rowbuf = rowbuf0;
 832
 833                         if (isConv2D)
 834                         {
 835                             for( ofs = ofs0; ofs < ofs1; out_j = 0, ++out_i )
 836                             {
 837                                 int delta = std::min(ofs1 - ofs, outW - out_j);
 838                                 int out_j1 = out_j + delta;
 839
 840                                 int in_i = out_i * stride_h - pad_t;
 841                                 int in_j = out_j * stride_w - pad_l;
 842                                 const float* imgptr = data_inp0 + (cn0*height + in_i)*width + in_j;
 843                                 ofs += delta;
 844
 845                                 // do im2row for a part of input tensor
 846                                 if( is1x1 )
 847                                 {
 848                                     for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w )
 849                                     {
 850                                         for( k = 0; k < vsz; k++ )
 851                                             rowbuf[k] = imgptr[k*inpPlaneSize];
 852                                     }
 853                                 }
 854                                 else
 855                                 {
 856                                     bool ok_i = 0 <= in_i && in_i < height - (kernel_h-1)*dilation_h;
 857                                     int i0 = std::max(0, (-in_i + dilation_h-1)/dilation_h);
 858                                     int i1 = std::min(kernel_h, (height - in_i + dilation_h-1)/dilation_h);
 859
 860                                     for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w, in_j += stride_w )
 861                                     {
 862                                         // this condition should be true for most of the tensor elements, i.e.
 863                                         // most of the time the kernel aperture is inside the tensor X-Y plane.
 864                                         if( ok_i && out_j + 2 <= out_j1 && 0 <= in_j && in_j + stride_w*2 <= width - (kernel_w-1)*dilation_w )
 865                                         {
 866                                             for( k = 0; k < vsz; k++ )
 867                                             {
 868                                                 int k1 = ofstab[k];
 869                                                 float v0 = imgptr[k1];
 870                                                 float v1 = imgptr[k1 + stride_w];
 871                                                 rowbuf[k] = v0;
 872                                                 rowbuf[k+vsz_a] = v1;
 873                                             }
 874                                             out_j++;
 875                                             rowbuf += vsz_a;
 876                                             imgptr += stride_w;
 877                                             in_j += stride_w;
 878                                         }
 879                                         else
 880                                         {
 881                                             int j0 = std::max(0, (-in_j + dilation_w-1)/dilation_w);
 882                                             int j1 = std::min(kernel_w, (width - in_j + dilation_w-1)/dilation_w);
 883
 884                                             // here some non-continuous sub-row of the row will not be
 885                                             // filled from the tensor; we need to make sure that the uncovered
 886                                             // elements are explicitly set to 0's. the easiest way is to
 887                                             // set all the elements to 0's before the loop.
 888                                             memset(rowbuf, 0, vsz*sizeof(rowbuf[0]));
 889                                             for( k = 0; k < ncn; k++ )
 890                                             {
 891                                                 for( i = i0; i < i1; i++ )
 892                                                 {
 893                                                     for( j = j0; j < j1; j++ )
 894                                                     {
 895                                                         int imgofs = k*(width*height) + i*(dilation_h*width) + j*dilation_w;
 896                                                         rowbuf[(k*kernel_h + i)*kernel_w + j] = imgptr[imgofs];
 897                                                     }
 898                                                 }
 899                                             }
 900                                         }
 901                                     }
 902                                 }
 903                             }
 904                         }
 905                         else
 906                         {
 907                             for( ofs = ofs0; ofs < ofs1; out_d += (out_i + 1) / outH, out_i = (out_i + 1) % outH, out_j = 0 )
 908                             {
 909                                 int delta = std::min(ofs1 - ofs, outW - out_j);
 910                                 int out_j1 = out_j + delta;
 911
 912                                 int in_d = out_d * stride_d - pad_d;
 913                                 int in_i = out_i * stride_h - pad_t;
 914                                 int in_j = out_j * stride_w - pad_l;
 915                                 const float* imgptr = data_inp0 + (cn0*depth*height + in_d*height + in_i)*width + in_j;
 916                                 ofs += delta;
 917
 918                                 int d0 = std::max(0, (-in_d + dilation_d - 1) / dilation_d);
 919                                 int d1 = std::min(kernel_d, (depth - in_d + dilation_d - 1) / dilation_d);
 920
 921                                 int i0 = std::max(0, (-in_i + dilation_h-1)/dilation_h);
 922                                 int i1 = std::min(kernel_h, (height - in_i + dilation_h-1)/dilation_h);
 923
 924                                 for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w, in_j += stride_w )
 925                                 {
 926                                     int j0 = std::max(0, (-in_j + dilation_w-1)/dilation_w);
 927                                     int j1 = std::min(kernel_w, (width - in_j + dilation_w-1)/dilation_w);
 928
 929                                     // here some non-continuous sub-row of the row will not be
 930                                     // filled from the tensor; we need to make sure that the uncovered
 931                                     // elements are explicitly set to 0's. the easiest way is to
 932                                     // set all the elements to 0's before the loop.
 933                                     memset(rowbuf, 0, vsz*sizeof(rowbuf[0]));
 934                                     for( k = 0; k < ncn; k++ )
 935                                     {
 936                                         for ( d = d0; d < d1; d++)
 937                                         {
 938                                             for( i = i0; i < i1; i++ )
 939                                             {
 940                                                 for( j = j0; j < j1; j++ )
 941                                                 {
 942                                                     int imgofs = k*(depth*width*height) + d*dilation_d*width*height + i*(dilation_h*width) + j*dilation_w;
 943                                                     rowbuf[(k*kernel_d*kernel_h + d*kernel_h + i)*kernel_w + j] = imgptr[imgofs];
 944                                                 }
 945                                             }
 946                                         }
 947                                     }
 948                                 }
 949                             }
 950                         }
 951
 952                         // now compute dot product of the weights
 953                         // and im2row-transformed part of the tensor
 954                         int bsz = ofs1 - ofs0;
 955                     #if CV_TRY_AVX512_SKX
 956                         /* AVX512 convolution requires an alignment of 16, and ROI is only there for larger vector sizes */
 957                         if(useAVX512)
 958                             opt_AVX512_SKX::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0,
 959                                           outShape, bsz, vsz, vsz_a, relu, cn0 == 0);
 960                         else
 961                     #endif
 962                     #if CV_TRY_AVX2
 963                         if(useAVX2)
 964                             opt_AVX2::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0,
 965                                           outShape, bsz, vsz, vsz_a, relu, cn0 == 0);
 966                         else
 967                     #endif
 968                     #if CV_TRY_AVX
 969                         if(useAVX)
 970                             opt_AVX::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0,
 971                                          outShape, bsz, vsz, vsz_a, relu, cn0 == 0);
 972                         else
 973                     #endif
 974                         for( int i = 0; i < outCn; i += 2 )
 975                         {
 976                             const float* wptr0 = wptr + i*wstep;
 977                             const float* wptr1 = wptr0 + wstep;
 978                             float* outptr0 = data_out0 + ofs0 + i*outPlaneSize;
 979                             float* outptr1 = outptr0 + outPlaneSize;
 980                             float bias0 = biasptr[i], bias1 = biasptr[i+1];
 981                             float r0 = 1.f, r1 = 1.f;
 982
 983                             if( i+1 >= outCn )
 984                             {
 985                                 wptr1 = wptr0;
 986                                 outptr1 = outptr0;
 987                                 bias1 = bias0;
 988                             }
 989
 990                             if( relu )
 991                             {
 992                                 r0 = relu[i]; r1 = relu[i+1];
 993                                 if( i+1 >= outCn )
 994                                     r1 = r0;
 995                             }
 996
 997                             int j = 0;
 998                         #if CV_SIMD128
 999                             v_float32x4 vr0 = v_setall_f32(r0), vr1 = v_setall_f32(r1), z = v_setzero_f32();
1000
1001                             for( ; j <= bsz - 4; j += 4 )
1002                             {
1003                                 const float* rptr = rowbuf0 + j*vsz_a;
1004                                 v_float32x4 s0, s1;
1005
1006                                 if( cn0 == 0 )
1007                                 {
1008                                     s0 = v_setall_f32(bias0);
1009                                     s1 = v_setall_f32(bias1);
1010                                 }
1011                                 else
1012                                 {
1013                                     s0 = v_load(outptr0 + j);
1014                                     s1 = v_load(outptr1 + j);
1015                                 }
1016
1017                                 v_float32x4 vs00 = v_setzero_f32(), vs01 = v_setzero_f32(),
1018                                             vs02 = v_setzero_f32(), vs03 = v_setzero_f32(),
1019                                             vs10 = v_setzero_f32(), vs11 = v_setzero_f32(),
1020                                             vs12 = v_setzero_f32(), vs13 = v_setzero_f32();
1021                                 for( k = 0; k < vsz; k += 4, rptr += 4 )
1022                                 {
1023                                     v_float32x4 w0 = v_load_aligned(wptr0 + k), w1 = v_load_aligned(wptr1 + k);
1024                                     v_float32x4 r0 = v_load_aligned(rptr), r1 = v_load_aligned(rptr + vsz_a),
1025                                                 r2 = v_load_aligned(rptr + vsz_a*2), r3 = v_load_aligned(rptr + vsz_a*3);
1026
1027                                     vs00 += w0*r0;
1028                                     vs01 += w0*r1;
1029                                     vs02 += w0*r2;
1030                                     vs03 += w0*r3;
1031
1032                                     vs10 += w1*r0;
1033                                     vs11 += w1*r1;
1034                                     vs12 += w1*r2;
1035                                     vs13 += w1*r3;
1036                                 }
1037                                 s0 += v_reduce_sum4(vs00, vs01, vs02, vs03);
1038                                 s1 += v_reduce_sum4(vs10, vs11, vs12, vs13);
1039                                 if( relu )
1040                                 {
1041                                     s0 = v_select(s0 > z, s0, s0*vr0);
1042                                     s1 = v_select(s1 > z, s1, s1*vr1);
1043                                 }
1044
1045                                 v_store(outptr0 + j, s0);
1046                                 v_store(outptr1 + j, s1);
1047                             }
1048                         #endif
1049                             for( ; j < bsz; j++ )
1050                             {
1051                                 const float* rptr = rowbuf0 + j*vsz_a;
1052                                 float s00, s10;
1053
1054                                 if( cn0 == 0 )
1055                                 {
1056                                     s00 = bias0;
1057                                     s10 = bias1;
1058                                 }
1059                                 else
1060                                 {
1061                                     s00 = outptr0[j];
1062                                     s10 = outptr1[j];
1063                                 }
1064
1065                                 for( k = 0; k < vsz; k++ )
1066                                 {
1067                                     float r0 = rptr[k];
1068                                     s00 += wptr0[k]*r0;
1069                                     s10 += wptr1[k]*r0;
1070                                 }
1071                                 if( relu )
1072                                 {
1073                                     s00 = s00 > 0.f ? s00 : s00*r0;
1074                                     s10 = s10 > 0.f ? s10 : s10*r1;
1075                                 }
1076
1077                                 outptr0[j] = s00;
1078                                 outptr1[j] = s10;
1079                             }
1080                         }
1081                     }
1082                 }
1083
1084                 if( activ_ )
1085                     activ_->forwardSlice(data_out0 + stripeStart, data_out0 + stripeStart,
1086                                          (int)(stripeEnd - stripeStart),
1087                                          outPlaneSize, startOutCn, startOutCn + outCn);
1088             }
1089         }
1090     };
1091
1092 #ifdef HAVE_OPENCL
1093     bool forward_ocl(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals)
1094     {
1095         std::vector<UMat> inputs;
1096         std::vector<UMat> outputs;
1097
1098         bool use_half = (inps.depth() == CV_16S);
1099         inps.getUMatVector(inputs);
1100         outs.getUMatVector(outputs);
1101
1102         CV_Assert(outputs.size() == 1);
1103         for (int i = 0; i < inputs.size(); ++i)
1104             CV_Assert(inputs[i].u != outputs[0].u);
1105
1106         if (umat_blobs.empty())
1107         {
1108             size_t n = blobs.size();
1109             umat_blobs.resize(n);
1110             for (size_t i = 0; i < n; i++)
1111             {
1112                 blobs[i].copyTo(umat_blobs[i]);
1113             }
1114         }
1115
1116         if (convolutionOp.empty())
1117         {
1118             OCL4DNNConvConfig config;
1119             config.in_shape = shape(inputs[0]);
1120             config.out_shape = shape(outputs[0]);
1121             config.kernel = kernel;
1122             config.pad = pad;
1123             config.stride = stride;
1124             config.dilation = dilation;
1125             config.group = inputs[0].size[1] / umat_blobs[0].size[1];
1126             config.bias_term = (hasBias()) ? true : false;
1127             config.use_half = use_half;
1128
1129             convolutionOp = Ptr<OCL4DNNConvSpatial<float> >(new OCL4DNNConvSpatial<float>(config));
1130         }
1131
1132         int outCn = umat_blobs[0].size[0];
1133
1134         reluslope.clear();
1135         if( activ )
1136         {
1137             Ptr<ReLULayer> activ_relu = activ.dynamicCast<ReLULayer>();
1138             if( !activ_relu.empty() )
1139             {
1140                 reluslope.assign(outCn+2, activ_relu->negativeSlope);
1141                 activType = OCL4DNN_CONV_FUSED_ACTIV_RELU;
1142             }
1143
1144             Ptr<ReLU6Layer> activ_relu6 = activ.dynamicCast<ReLU6Layer>();
1145             if( !activ_relu6.empty() )
1146             {
1147                 reluslope.resize(2);
1148                 reluslope[0] = activ_relu6->minValue;
1149                 reluslope[1] = activ_relu6->maxValue;
1150                 activType = OCL4DNN_CONV_FUSED_ACTIV_RELU6;
1151             }
1152
1153             Ptr<ChannelsPReLULayer> activ_chprelu = activ.dynamicCast<ChannelsPReLULayer>();
1154             if( !activ_chprelu.empty() )
1155             {
1156                 const Mat& m = activ_chprelu->blobs[0];
1157                 CV_Assert(m.isContinuous() && m.type() == CV_32F && (int)m.total() == outCn);
1158                 const float* mdata = m.ptr<float>();
1159                 reluslope.resize(outCn+2);
1160                 std::copy(mdata, mdata + outCn, reluslope.begin());
1161                 reluslope[outCn] = reluslope[outCn+1] = reluslope[outCn-1];
1162                 activType = OCL4DNN_CONV_FUSED_ACTIV_PRELU;
1163             }
1164         }
1165
1166         if (fusedWeights)
1167         {
1168             weightsMat.copyTo(umat_blobs[0]);
1169             fusedWeights = false;
1170         }
1171         if (fusedBias)
1172         {
1173             if ( umat_blobs.size() < 2 )
1174                 umat_blobs.resize(2);
1175             umat_blobs[1] = UMat(biasvec, true);
1176             convolutionOp->setBias(true);
1177             fusedBias = false;
1178         }
1179
1180         if ( newActiv )
1181         {
1182             if ( activType == OCL4DNN_CONV_FUSED_ACTIV_RELU )
1183             {
1184                 CV_Assert(!reluslope.empty());
1185                 convolutionOp->setActivReLU(true, reluslope[0]);
1186             }
1187             else if ( activType == OCL4DNN_CONV_FUSED_ACTIV_PRELU)
1188             {
1189                 CV_Assert(!reluslope.empty());
1190                 convolutionOp->setActivPReLU(true, reluslope);
1191             }
1192             else if ( activType == OCL4DNN_CONV_FUSED_ACTIV_POWER)
1193             {
1194                 convolutionOp->setActivPower(true, power);
1195             }
1196             else if ( activType == OCL4DNN_CONV_FUSED_ACTIV_TANH)
1197             {
1198                 convolutionOp->setActivTanh(true);
1199             }
1200             else if ( activType == OCL4DNN_CONV_FUSED_ACTIV_RELU6)
1201             {
1202                 convolutionOp->setActivReLU6(true, reluslope[0], reluslope[1]);
1203             }
1204             else
1205             {
1206                 convolutionOp->setActivReLU(false, 0);
1207                 convolutionOp->setActivPReLU(false, reluslope);
1208                 convolutionOp->setActivPower(false, 1.f);
1209                 convolutionOp->setActivTanh(false);
1210                 convolutionOp->setActivReLU6(false, 0, 0);
1211             }
1212             newActiv = false;
1213         }
1214
1215         UMat& inpMat = inputs[0];
1216         UMat& outMat = outputs[0];
1217         int batch_size = inpMat.size[0];
1218
1219         return convolutionOp->Forward(inpMat,
1220                                       inputs.size() == 2 ? inputs[1] : UMat(),
1221                                       umat_blobs[0],
1222                                       umat_blobs.size() > 1 ? umat_blobs[1] : UMat(),
1223                                       outMat,
1224                                       batch_size);
1225     }
1226 #endif
1227
1228     void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE
1229     {
1230         CV_TRACE_FUNCTION();
1231         CV_TRACE_ARG_VALUE(name, "name", name.c_str());
1232
1233         CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget),
1234                    forward_ocl(inputs_arr, outputs_arr, internals_arr))
1235
1236         if (inputs_arr.depth() == CV_16S)
1237         {
1238             forward_fallback(inputs_arr, outputs_arr, internals_arr);
1239             return;
1240         }
1241
1242         std::vector<Mat> inputs, outputs;
1243         inputs_arr.getMatVector(inputs);
1244         outputs_arr.getMatVector(outputs);
1245
1246         /*printf("conv %s: input (%d x %d x %d x %d), kernel (%d x %d), pad (%d x %d), stride (%d x %d), dilation (%d x %d)\n",
1247                name.c_str(), inputs[0].size[0], inputs[0].size[1], inputs[0].size[2], inputs[0].size[3],
1248                kernel.width, kernel.height, pad.width, pad.height,
1249                stride.width, stride.height, dilation.width, dilation.height);*/
1250         CV_Assert_N(inputs.size() == (size_t)1, inputs[0].size[1] % blobs[0].size[1] == 0,
1251                     outputs.size() == 1, inputs[0].data != outputs[0].data);
1252
1253         int ngroups = inputs[0].size[1]/blobs[0].size[1];
1254         CV_Assert(outputs[0].size[1] % ngroups == 0);
1255         int outCn = blobs[0].size[0];
1256
1257         reluslope.clear();
1258         if( activ )
1259         {
1260             Ptr<ReLULayer> activ_relu = activ.dynamicCast<ReLULayer>();
1261             if( !activ_relu.empty() )
1262             {
1263                 reluslope.assign(outCn+2, activ_relu->negativeSlope);
1264             }
1265
1266             Ptr<ChannelsPReLULayer> activ_chprelu = activ.dynamicCast<ChannelsPReLULayer>();
1267             if( !activ_chprelu.empty() )
1268             {
1269                 const Mat& m = activ_chprelu->blobs[0];
1270                 CV_Assert(m.isContinuous() && m.type() == CV_32F && (int)m.total() == outCn);
1271                 const float* mdata = m.ptr<float>();
1272                 reluslope.resize(outCn+2);
1273                 std::copy(mdata, mdata + outCn, reluslope.begin());
1274                 reluslope[outCn] = reluslope[outCn+1] = reluslope[outCn-1];
1275             }
1276         }
1277
1278         int nstripes = std::max(getNumThreads(), 1);
1279
1280         ParallelConv::run(inputs[0], outputs[0], weightsMat, biasvec, reluslope,
1281                           kernel_size, strides, pads_begin, pads_end, dilations, activ.get(), ngroups, nstripes);
1282     }
1283
1284     virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
1285                            const std::vector<MatShape> &outputs) const CV_OVERRIDE
1286     {
1287         CV_Assert(inputs.size() == outputs.size());
1288
1289         int64 flops = 0;
1290         int karea = std::accumulate(kernel_size.begin(), kernel_size.end(), 1, std::multiplies<size_t>());
1291         for (int i = 0; i < inputs.size(); i++)
1292         {
1293             flops += total(outputs[i])*(CV_BIG_INT(2)*karea*inputs[i][1] + 1);
1294         }
1295
1296         return flops;
1297     }
1298 };
1299
1300 class DeConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl
1301 {
1302 public:
1303     Mat weightsMat, biasesMat;
1304     UMat umat_weights;
1305     UMat umat_biases;
1306
1307     DeConvolutionLayerImpl(const LayerParams& params) : BaseConvolutionLayerImpl(params) {}
1308
1309     MatShape computeColRowShape(const MatShape &inpShape, const MatShape &outShape) const CV_OVERRIDE
1310     {
1311         int dims = inpShape.size();
1312         int inpCn = inpShape[1];
1313         int inpD = dims == 5 ? inpShape[2] : 1;
1314         int inpH = inpShape[dims - 2];
1315         int inpW = inpShape.back();
1316         int outCn = outShape[1];
1317         int ngroups = inpCn / blobs[0].size[0];
1318         int outGroupCn = outCn / ngroups;
1319         int ksize = outGroupCn * std::accumulate(kernel_size.begin(), kernel_size.end(),
1320                                                  1, std::multiplies<size_t>());
1321         return shape(ksize, inpD * inpH * inpW);
1322     }
1323
1324     virtual bool supportBackend(int backendId) CV_OVERRIDE
1325     {
1326 #ifdef HAVE_INF_ENGINE
1327         const int outGroupCn = blobs[0].size[1];  // Weights are in IOHW or IODHW layout
1328         const int group = numOutput / outGroupCn;
1329
1330         if (backendId == DNN_BACKEND_INFERENCE_ENGINE)
1331         {
1332             if (kernel_size.size() == 3 && preferableTarget != DNN_TARGET_CPU) {
1333                 return false;
1334             }
1335
1336             if (std::accumulate(adjust_pads.begin(), adjust_pads.end(), 0, std::plus<size_t>()) > 0)
1337             {
1338                 if (padMode.empty())
1339                 {
1340                     if (preferableTarget != DNN_TARGET_CPU && group != 1)
1341                     {
1342                         for (int i = 0; i < adjust_pads.size(); i++) {
1343                             if (adjust_pads[i] && pads_begin[i])
1344                                 return false;
1345                         }
1346                     }
1347                     for (int i = 0; i < adjust_pads.size(); i++) {
1348                         if (pads_end[i] < adjust_pads[i])
1349                             return false;
1350                     }
1351                     return true;
1352                 }
1353                 else if (padMode == "SAME")
1354                 {
1355                     for (int i = 0; i < adjust_pads.size(); i++) {
1356                         if (kernel_size[i] < pads_begin[i] + 1 + adjust_pads[i])
1357                             return false;
1358                     }
1359                     return true;
1360                 }
1361                 else if (padMode == "VALID")
1362                     return false;
1363             }
1364
1365             if (group != 1)
1366             {
1367                 return preferableTarget == DNN_TARGET_CPU;
1368             }
1369             if (preferableTarget == DNN_TARGET_OPENCL || preferableTarget == DNN_TARGET_OPENCL_FP16)
1370                 return std::accumulate(dilations.begin(), dilations.end(), 1, std::multiplies<size_t>()) == 1;
1371             return true;
1372         }
1373         else
1374 #endif  // HAVE_INF_ENGINE
1375             return kernel_size.size() == 2 && (backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE);
1376     }
1377
1378     bool getMemoryShapes(const std::vector<MatShape> &inputs,
1379                          const int requiredOutputs,
1380                          std::vector<MatShape> &outputs,
1381                          std::vector<MatShape> &internals) const CV_OVERRIDE
1382     {
1383         CV_Assert(!hasBias() || blobs[1].total() == (size_t)numOutput);
1384         CV_Assert(inputs.size() != 0);
1385
1386         int outCn = numOutput;
1387         std::vector<int> outShape;
1388         outShape.push_back(inputs[0][0]);  // batch
1389         outShape.push_back(outCn);
1390         if (padMode.empty())
1391         {
1392             for (int i = 0; i < kernel_size.size(); i++)
1393                 outShape.push_back(strides[i] * (inputs[0][2 + i] - 1) + kernel_size[i] - pads_begin[i] - pads_end[i] + adjust_pads[i]);
1394         }
1395         else if (padMode == "VALID")
1396         {
1397             for (int i = 0; i < kernel_size.size(); i++)
1398                 outShape.push_back(strides[i] * (inputs[0][2 + i] - 1) + kernel_size[i] + adjust_pads[i]);
1399         }
1400         else if (padMode == "SAME")
1401         {
1402             for (int i = 0; i < kernel_size.size(); i++)
1403                 outShape.push_back(strides[i] * (inputs[0][2 + i] - 1) + 1 + adjust_pads[i]);
1404         }
1405         else
1406             CV_Error(Error::StsError, "Unsupported padding mode " + padMode);
1407
1408         CV_Assert(outCn % blobs[0].size[1] == 0);
1409         int ngroups = outCn / blobs[0].size[1];
1410
1411         int inpCn = inputs[0][1];
1412         CV_Assert(inpCn % ngroups == 0 && outCn % ngroups == 0);
1413         CV_Assert(blobs[0].size[0] == inpCn);
1414
1415         outputs.resize(1, outShape);
1416
1417         if (!is1x1())
1418             internals.push_back(computeColRowShape(inputs[0], outputs[0]));
1419
1420         return false;
1421     }
1422
1423     void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE
1424     {
1425         BaseConvolutionLayerImpl::finalize(inputs_arr, outputs_arr);
1426
1427         std::vector<Mat> inputs, outputs;
1428         inputs_arr.getMatVector(inputs);
1429         outputs_arr.getMatVector(outputs);
1430
1431         std::vector<int> inpShape;
1432         std::vector<int> outShape;
1433         for (int i = 2; i < inputs[0].dims; i++) {
1434             inpShape.push_back(inputs[0].size[i]);
1435             outShape.push_back(outputs[0].size[i]);
1436         }
1437         getConvPoolPaddings(outShape, kernel_size, strides, padMode, pads_begin, pads_end);
1438         if (pads_begin.size() == 2) {
1439             for (int i = 0; i < pads_begin.size(); i++) {
1440                 if (pads_begin[i] != pads_end[i])
1441                     CV_Error(Error::StsNotImplemented, "Unsupported asymmetric padding in deconvolution layer");
1442             }
1443             pad = Size(pads_begin[1], pads_begin[0]);
1444         }
1445
1446         weightsMultipliers.assign(numOutput, 1.0);
1447         if (weightsMat.empty())
1448         {
1449             transpose(blobs[0].reshape(1, blobs[0].size[0]), weightsMat);
1450             biasesMat = hasBias() ? blobs[1].reshape(1, numOutput)
1451                                   : Mat::zeros(numOutput, 1, CV_32F);
1452         }
1453     }
1454
1455     void fuseWeights(const Mat& w_, const Mat& b_) CV_OVERRIDE
1456     {
1457         Mat w = w_.total() == 1 ? Mat(1, numOutput, CV_32F, Scalar(w_.at<float>(0))) : w_;
1458         Mat b = b_.total() == 1 ? Mat(1, numOutput, CV_32F, Scalar(b_.at<float>(0))) : b_;
1459
1460         CV_Assert_N(!weightsMat.empty(),
1461                      w.empty() || numOutput == w.total(),
1462                      b.empty() || numOutput == b.total());
1463
1464         if (!w.empty())
1465         {
1466             transpose(blobs[0].reshape(1, blobs[0].size[0]), weightsMat);
1467             weightsMat = weightsMat.reshape(1, numOutput);
1468             for (int i = 0; i < numOutput; ++i)
1469             {
1470                 double wi = w.at<float>(i);
1471                 weightsMultipliers[i] *= wi;
1472                 cv::multiply(weightsMat.row(i), weightsMultipliers[i], weightsMat.row(i));
1473                 biasesMat.at<float>(i) *= wi;
1474             }
1475             weightsMat = weightsMat.reshape(1, weightsMat.total() / blobs[0].size[0]);
1476         }
1477
1478         if (!b.empty())
1479         {
1480             cv::add(biasesMat, b.reshape(1, numOutput), biasesMat);
1481         }
1482     }
1483
1484     class MatMulInvoker : public ParallelLoopBody
1485     {
1486     public:
1487         MatMulInvoker(const Mat& a, const Mat& b, Mat& c, int nstripes)
1488         {
1489             a_ = &a;
1490             b_ = &b;
1491             c_ = &c;
1492             nstripes_ = nstripes;
1493             useAVX = checkHardwareSupport(CPU_AVX);
1494             useAVX2 = checkHardwareSupport(CPU_AVX2);
1495             useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX;
1496         }
1497
1498         void operator()(const Range& range_) const CV_OVERRIDE
1499         {
1500             int stripeSize = (int)alignSize((b_->cols + nstripes_ - 1)/nstripes_, 16);
1501             Range range(range_.start*stripeSize, std::min(range_.end*stripeSize, b_->cols));
1502             int mmax = a_->rows;
1503             int nmax = range.end - range.start;
1504             int kmax = a_->cols;
1505             int m, n, k;
1506             const float* aptr = a_->ptr<float>();
1507             const float* bptr = b_->ptr<float>() + range.start;
1508             float* cptr = c_->ptr<float>() + range.start;
1509             size_t astep = a_->step1();
1510             size_t bstep = b_->step1();
1511             size_t cstep = c_->step1();
1512
1513         #if CV_TRY_AVX512_SKX
1514             if( useAVX512 )
1515                 opt_AVX512_SKX::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax );
1516             else
1517         #endif
1518         #if CV_TRY_AVX2
1519             if( useAVX2 )
1520                 opt_AVX2::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax );
1521             else
1522         #endif
1523         #if CV_TRY_AVX
1524             if( useAVX )
1525                 opt_AVX::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax );
1526             else
1527         #endif
1528             for( m = 0; m < mmax; m += 2 )
1529             {
1530                 float* dst0 = cptr + cstep*m;
1531                 float* dst1 = cptr + cstep*std::min(m+1, mmax-1);
1532                 const float* aptr0 = aptr + astep*m;
1533                 const float* aptr1 = aptr + astep*std::min(m+1, mmax-1);
1534
1535                 for( n = 0; n < nmax; n++ )
1536                 {
1537                     dst0[n] = 0.f;
1538                     dst1[n] = 0.f;
1539                 }
1540
1541                 for( k = 0; k < kmax; k += 4 )
1542                 {
1543                     float alpha00 = aptr0[k];
1544                     float alpha01 = aptr1[k];
1545                     float alpha10 = 0.f, alpha11 = 0.f;
1546                     float alpha20 = 0.f, alpha21 = 0.f;
1547                     float alpha30 = 0.f, alpha31 = 0.f;
1548                     const float* bptr0 = bptr + k*bstep;
1549                     const float* bptr1 = bptr0;
1550                     const float* bptr2 = bptr0;
1551                     const float* bptr3 = bptr0;
1552
1553                     if( k+1 < kmax )
1554                     {
1555                         alpha10 = aptr0[k+1];
1556                         alpha11 = aptr1[k+1];
1557                         bptr1 = bptr0 + bstep;
1558                         if( k+2 < kmax )
1559                         {
1560                             alpha20 = aptr0[k+2];
1561                             alpha21 = aptr1[k+2];
1562                             bptr2 = bptr1 + bstep;
1563                             if( k+3 < kmax )
1564                             {
1565                                 alpha30 = aptr0[k+3];
1566                                 alpha31 = aptr1[k+3];
1567                                 bptr3 = bptr2 + bstep;
1568                             }
1569                         }
1570                     }
1571                     n = 0;
1572
1573                 #if CV_SIMD128
1574                     v_float32x4 a00 = v_setall_f32(alpha00);
1575                     v_float32x4 a01 = v_setall_f32(alpha01);
1576                     v_float32x4 a10 = v_setall_f32(alpha10);
1577                     v_float32x4 a11 = v_setall_f32(alpha11);
1578                     v_float32x4 a20 = v_setall_f32(alpha20);
1579                     v_float32x4 a21 = v_setall_f32(alpha21);
1580                     v_float32x4 a30 = v_setall_f32(alpha30);
1581                     v_float32x4 a31 = v_setall_f32(alpha31);
1582
1583                     for( ; n <= nmax - 4; n += 4 )
1584                     {
1585                         v_float32x4 b0 = v_load(bptr0 + n);
1586                         v_float32x4 b1 = v_load(bptr1 + n);
1587                         v_float32x4 b2 = v_load(bptr2 + n);
1588                         v_float32x4 b3 = v_load(bptr3 + n);
1589                         v_float32x4 d0 = v_load(dst0 + n);
1590                         v_float32x4 d1 = v_load(dst1 + n);
1591                         d0 += b0*a00;
1592                         d1 += b0*a01;
1593                         d0 += b1*a10;
1594                         d1 += b1*a11;
1595                         d0 += b2*a20;
1596                         d1 += b2*a21;
1597                         d0 += b3*a30;
1598                         d1 += b3*a31;
1599                         v_store(dst0 + n, d0);
1600                         v_store(dst1 + n, d1);
1601                     }
1602                 #endif
1603
1604                     for( ; n < nmax; n++ )
1605                     {
1606                         float b0 = bptr0[n], b1 = bptr1[n];
1607                         float b2 = bptr2[n], b3 = bptr3[n];
1608                         float d0 = dst0[n] + alpha00*b0 + alpha10*b1 + alpha20*b2 + alpha30*b3;
1609                         float d1 = dst1[n] + alpha01*b0 + alpha11*b1 + alpha21*b2 + alpha31*b3;
1610                         dst0[n] = d0;
1611                         dst1[n] = d1;
1612                     }
1613                 }
1614             }
1615         }
1616
1617         const Mat *a_, *b_;
1618         Mat* c_;
1619         int nstripes_;
1620         bool useAVX;
1621         bool useAVX2;
1622         bool useAVX512;
1623     };
1624
1625     class Col2ImInvoker : public cv::ParallelLoopBody
1626     {
1627     public:
1628         const float* data_col;
1629         const float* biasvec;
1630         int channels, height, width;
1631         int kernel_h, kernel_w;
1632         int pad_h, pad_w;
1633         int stride_h, stride_w;
1634         float* data_im;
1635         int height_col, width_col;
1636         int nstripes;
1637         bool is1x1;
1638
1639         Col2ImInvoker()
1640             : data_col(0), biasvec(0), channels(0), height(0), width(0),
1641               kernel_h(0), kernel_w(0), pad_h(0), pad_w(0), stride_h(0), stride_w(0), data_im(0),
1642               height_col(0), width_col(0), nstripes(0), is1x1(0)
1643         {}
1644
1645         static void run(const float* data_col,
1646                         int channels, int height, int width,
1647                         int kernel_h, int kernel_w,
1648                         int pad_h, int pad_w,
1649                         int stride_h, int stride_w,
1650                         int height_col, int width_col,
1651                         float* data_im,
1652                         const float* biasvec,
1653                         bool is1x1)
1654         {
1655             const int nstripes = getNumThreads();
1656
1657             Col2ImInvoker t;
1658             t.data_col = data_col;
1659             t.data_im = data_im;
1660             t.channels = channels; t.height = height; t.width = width;
1661             t.kernel_h = kernel_h; t.kernel_w = kernel_w;
1662             t.pad_h = pad_h; t.pad_w = pad_w;
1663             t.stride_h = stride_h; t.stride_w = stride_w;
1664             t.height_col = height_col;
1665             t.width_col = width_col;
1666             t.nstripes = nstripes;
1667             t.is1x1 = is1x1;
1668             t.biasvec = biasvec;
1669
1670             parallel_for_(Range(0, nstripes), t, nstripes);
1671         }
1672
1673         virtual void operator ()(const Range &r) const CV_OVERRIDE
1674         {
1675             const float* data_col_ = data_col;
1676             float* data_im_ = data_im;
1677             int coeff_h = (1 - stride_h * kernel_w * height_col) * width_col;
1678             int coeff_w = (1 - stride_w * height_col * width_col);
1679             size_t total = (size_t)channels * height * width;
1680             size_t stripeSize = (total + nstripes - 1)/nstripes;
1681             size_t startIndex = r.start*stripeSize;
1682             size_t endIndex = std::min(r.end*stripeSize, total);
1683             int w = (int)(startIndex % width + pad_w);
1684             int h = (int)((startIndex / width) % height + pad_h);
1685             int c = (int)(startIndex / (width * height));
1686             int h_col_start = (h < kernel_h) ? 0 : (h - kernel_h) / stride_h + 1;
1687             int h_col_end = std::min(h / stride_h + 1, height_col);
1688             int plane_size_col = height_col * width_col;
1689             int offset = (c * kernel_h * kernel_w + h * kernel_w + w) * plane_size_col;
1690             bool is1x1_ = is1x1;
1691             const float* biasvec_ = biasvec;
1692
1693             for (size_t index = startIndex; index < endIndex; index++)
1694             {
1695                 // compute the start and end of the output
1696                 int w_col_start = (w < kernel_w) ? 0 : (w - kernel_w) / stride_w + 1;
1697                 int w_col_end = std::min(w / stride_w + 1, width_col);
1698                 float val;
1699
1700                 if( is1x1_ )
1701                     val = data_im_[index];
1702                 else
1703                 {
1704                     val = 0.f;
1705                     for (int h_col = h_col_start; h_col < h_col_end; ++h_col) {
1706                         for (int w_col = w_col_start; w_col < w_col_end; ++w_col) {
1707                             val += data_col_[offset + h_col * coeff_h + w_col * coeff_w];
1708                         }
1709                     }
1710                 }
1711                 data_im_[index] = val + biasvec_[c];
1712
1713                 offset += plane_size_col;
1714                 if( ++w >= width + pad_w )
1715                 {
1716                     w = (int)((index + 1)% width + pad_w);
1717                     h = (int)(((index + 1) / width) % height + pad_h);
1718                     c = (int)((index + 1) / (width * height));
1719                     h_col_start = (h < kernel_h) ? 0 : (h - kernel_h) / stride_h + 1;
1720                     h_col_end = std::min(h / stride_h + 1, height_col);
1721                     offset = (c * kernel_h * kernel_w + h * kernel_w + w) * plane_size_col;
1722                 }
1723             }
1724         }
1725     };
1726
1727 #ifdef HAVE_OPENCL
1728     bool forward_ocl(InputArrayOfArrays inputs_, OutputArrayOfArrays outputs_, OutputArrayOfArrays internals_)
1729     {
1730         std::vector<UMat> inputs;
1731         std::vector<UMat> outputs;
1732         std::vector<UMat> internals;
1733
1734         if (inputs_.depth() == CV_16S)
1735             return false;
1736
1737         inputs_.getUMatVector(inputs);
1738         outputs_.getUMatVector(outputs);
1739         internals_.getUMatVector(internals);
1740
1741         int outCn = numOutput;
1742         int inpCn = inputs[0].size[1];
1743
1744         if (is1x1())
1745             return false;
1746
1747         if (umat_weights.empty())
1748         {
1749             if (fusedWeights)
1750                 weightsMat.copyTo(umat_weights);
1751             else
1752                 transpose(blobs[0].reshape(1, inpCn), umat_weights);
1753
1754             if (fusedBias)
1755                 biasesMat.copyTo(umat_biases);
1756             else
1757             {
1758                 if (hasBias())
1759                     blobs[1].reshape(1, outCn).copyTo(umat_biases);
1760                 else
1761                     umat_biases = UMat::zeros(outCn, 1, CV_32F);
1762             }
1763         }
1764
1765         String buildopt = format("-DT=%s ", ocl::typeToStr(inputs[0].type()));
1766         buildopt += format("-DPAD_H=%d -DPAD_W=%d -DKERNEL_H=%d -DKERNEL_W=%d -DSTRIDE_H=%d -DSTRIDE_W=%d ",
1767                            pad.height, pad.width, kernel.height, kernel.width, stride.height, stride.width);
1768
1769         for (size_t ii = 0; ii < outputs.size(); ii++)
1770         {
1771             int ngroups = outCn / blobs[0].size[1];
1772             int inpGroupCn = inpCn / ngroups;
1773             int outGroupCn = blobs[0].size[1];
1774             const UMat& inp = inputs[ii];
1775             UMat& out = outputs[ii];
1776             int numImg = inp.size[0];
1777             int inpH = inp.size[2], inpW = inp.size[3];
1778             int outH = out.size[2], outW = out.size[3];
1779
1780             MatShape inpshape = shape(numImg*inpCn, inpH*inpW);
1781             MatShape outshape = shape(numImg*outCn, outH*outW);
1782             UMat convBlob = inputs[ii].reshape(1, inpshape.size(), &inpshape[0]);
1783             UMat decnBlob = out.reshape(1, outshape.size(), &outshape[0]);
1784             int rows = internals[0].rows / ngroups;
1785
1786             for (int n = 0; n < numImg; n++)
1787             {
1788                 for (int g = 0; g < ngroups; g++)
1789                 {
1790                     UMat colMat = internals[0].rowRange(_Range(g * rows, rows));
1791                     UMat convMat = convBlob.rowRange(_Range((g + n * ngroups) * inpGroupCn, inpGroupCn));
1792                     UMat wghtMat = umat_weights.colRange(_Range(g * inpGroupCn, inpGroupCn));
1793                     gemm(wghtMat, convMat, 1, noArray(), 0, colMat, 0);
1794                 }
1795
1796                 for (int g = 0; g < ngroups; g++)
1797                 {
1798                     int total = outGroupCn * decnBlob.cols;
1799                     int index = 0;
1800                     int height_col = inpH;
1801                     int width_col = inpW;
1802                     int coeff_h = (1 - stride.height * kernel.width * height_col) * width_col;
1803                     int coeff_w = (1 - stride.width * height_col * width_col);
1804
1805                     ocl::Kernel k("col2im", ocl::dnn::col2im_oclsrc, buildopt);
1806                     k.set(index++, total);
1807                     k.set(index++, ocl::KernelArg::PtrReadOnly(internals[0]));
1808                     k.set(index++, (int)(g * rows * internals[0].cols));
1809                     k.set(index++, outGroupCn);
1810                     k.set(index++, outH);
1811                     k.set(index++, outW);
1812                     k.set(index++, height_col);
1813                     k.set(index++, width_col);
1814                     k.set(index++, coeff_h);
1815                     k.set(index++, coeff_w);
1816                     k.set(index++, ocl::KernelArg::PtrReadOnly(umat_biases));
1817                     k.set(index++, (int)(g * outGroupCn * umat_biases.cols));
1818                     k.set(index++, ocl::KernelArg::PtrWriteOnly(decnBlob));
1819                     k.set(index++, (int)((g + n * ngroups) * outGroupCn * decnBlob.cols));
1820
1821                     size_t global[] = { (size_t)total };
1822                     bool ret = k.run(1, global, NULL, false);
1823                     if (!ret)
1824                         return false;
1825                 }
1826             }
1827         }
1828
1829         return true;
1830     }
1831 #endif
1832
1833     void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE
1834     {
1835         CV_TRACE_FUNCTION();
1836         CV_TRACE_ARG_VALUE(name, "name", name.c_str());
1837
1838         CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget),
1839                    forward_ocl(inputs_arr, outputs_arr, internals_arr));
1840
1841         if (inputs_arr.depth() == CV_16S)
1842         {
1843             forward_fallback(inputs_arr, outputs_arr, internals_arr);
1844             return;
1845         }
1846
1847         std::vector<Mat> inputs, outputs, internals;
1848         inputs_arr.getMatVector(inputs);
1849         outputs_arr.getMatVector(outputs);
1850         internals_arr.getMatVector(internals);
1851
1852         int outCn = numOutput;
1853         int inpCn = inputs[0].size[1];
1854         bool is1x1flag = is1x1();
1855         int nstripes = getNumThreads();
1856
1857         if( weightsMat.empty() )
1858         {
1859             transpose(blobs[0].reshape(1, inpCn), weightsMat);
1860             biasesMat = hasBias() ? blobs[1].reshape(1, outCn) : Mat::zeros(outCn, 1, CV_32F);
1861         }
1862
1863         for (size_t ii = 0; ii < outputs.size(); ii++)
1864         {
1865             int ngroups = outCn / blobs[0].size[1];
1866             int inpGroupCn = inpCn / ngroups;
1867             int outGroupCn = blobs[0].size[1];
1868             const Mat& inp = inputs[ii];
1869             Mat& out = outputs[ii];
1870             int numImg = inp.size[0];
1871             int inpH = inp.size[2], inpW = inp.size[3];
1872             int outH = out.size[2], outW = out.size[3];
1873
1874             Mat convBlob = inputs[ii].reshape(1, numImg*inpCn);
1875             Mat decnBlob = out.reshape(1, numImg*outCn);
1876
1877             for (int n = 0; n < numImg; n++)
1878             {
1879                 for (int g = 0; g < ngroups; g++)
1880                 {
1881                     Mat dstMat = decnBlob.rowRange(_Range((g + n * ngroups) * outGroupCn, outGroupCn));
1882                     Mat &colMat = is1x1flag ? dstMat : internals[0];
1883
1884                     Mat convMat = convBlob.rowRange(_Range((g + n * ngroups) * inpGroupCn, inpGroupCn));
1885                     Mat wghtMat = weightsMat.colRange(_Range(g * inpGroupCn, inpGroupCn));
1886                     Mat curBiasMat = biasesMat.rowRange(_Range(g * outGroupCn, outGroupCn));
1887
1888                     //gemm(wghtMat, convMat, 1, colMat, 0, colMat, 0);
1889                     MatMulInvoker mminvoker(wghtMat, convMat, colMat, nstripes);
1890                     parallel_for_(Range(0, nstripes), mminvoker, nstripes);
1891
1892                     Col2ImInvoker::run(colMat.ptr<float>(), outGroupCn, outH, outW,
1893                                        kernel.height, kernel.width, pad.height, pad.width,
1894                                        stride.height, stride.width, inpH, inpW, dstMat.ptr<float>(),
1895                                        curBiasMat.ptr<float>(), is1x1flag);
1896                 }
1897             }
1898         }
1899     }
1900
1901     virtual Ptr<BackendNode> initHalide(const std::vector<Ptr<BackendWrapper> > &inputs) CV_OVERRIDE
1902     {
1903 #ifdef HAVE_HALIDE
1904         Halide::Buffer<float> inputBuffer = halideBuffer(inputs[0]);
1905
1906         int inW, inH, inC, inN;
1907         getCanonicalSize(inputBuffer, &inW, &inH, &inC, &inN);
1908         const int outGroupCn = blobs[0].size[1];
1909         const int group = numOutput / outGroupCn;
1910         const int inpGroupCn = blobs[0].size[0] / group;
1911
1912         Halide::Var x("x"), y("y"), c("c"), n("n");
1913         Halide::Func top = (name.empty() ? Halide::Func() : Halide::Func(name));
1914         Halide::Func padded_input(name + "_constant_exterior");
1915         auto weights = wrapToHalideBuffer(blobs[0]);
1916
1917         Halide::Func dilated_input("dilated_input");
1918         dilated_input(x, y, c, n) = 0.0f;
1919         Halide::RDom r1(0, inW, 0, inH);
1920         dilated_input(r1.x * stride.width, r1.y * stride.height, c, n) =
1921               inputBuffer(r1.x, r1.y, c, n);
1922         dilated_input.compute_root();
1923
1924         Halide::Func bounded =
1925             Halide::BoundaryConditions::constant_exterior(dilated_input, 0,
1926                                                           0, (inW - 1) * stride.width + 1,
1927                                                           0, (inH - 1) * stride.height + 1,
1928                                                           0, inC, 0, inN);
1929         padded_input(x, y, c, n) = bounded(x, y, c, n);
1930
1931         Halide::RDom r(0, kernel.width, 0, kernel.height, 0, inpGroupCn);
1932         Halide::Expr kx = x + pad.width - r.x;
1933         Halide::Expr ky = y + pad.height - r.y;
1934         Halide::Expr kInC = r.z;
1935         Halide::Expr kOutC = c;
1936         for (int i = 1; i < group; ++i)
1937         {
1938             kInC = select(c < outGroupCn * i, kInC, inpGroupCn * i + r.z);
1939             kOutC = select(c < outGroupCn * i, kOutC, c - outGroupCn * i);
1940         }
1941         Halide::Expr topExpr = sum(padded_input(kx, ky, kInC, n) *
1942                                    weights(r.x, r.y, kOutC, kInC));
1943         if (hasBias())
1944         {
1945             auto bias = wrapToHalideBuffer(blobs[1], {numOutput});
1946             topExpr += bias(c);
1947         }
1948         top(x, y, c, n) = topExpr;
1949         return Ptr<BackendNode>(new HalideBackendNode({ padded_input, top }));
1950 #endif  // HAVE_HALIDE
1951         return Ptr<BackendNode>();
1952     }
1953
1954 #ifdef HAVE_INF_ENGINE
1955     virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> > &) CV_OVERRIDE
1956     {
1957         InferenceEngine::Layout layout = blobs[0].dims == 5? InferenceEngine::Layout::NCDHW :
1958                                                              InferenceEngine::Layout::OIHW;
1959
1960         auto ieWeights = wrapToInfEngineBlob(blobs[0], layout);
1961         if (fusedWeights)
1962         {
1963             ieWeights = InferenceEngine::make_shared_blob<float>({
1964                             InferenceEngine::Precision::FP32,
1965                             ieWeights->getTensorDesc().getDims(), layout
1966                         });
1967             ieWeights->allocate();
1968
1969             int inpCn = blobs[0].size[0];
1970             Mat newWeights = infEngineBlobToMat(ieWeights).reshape(1, inpCn);
1971             transpose(weightsMat, newWeights);
1972         }
1973
1974         const int outGroupCn = blobs[0].size[1];  // Weights are in IOHW or OIDHW layout
1975         const int group = numOutput / outGroupCn;
1976
1977         InferenceEngine::Builder::DeconvolutionLayer ieLayer(name);
1978
1979         ieLayer.setKernel(kernel_size);
1980         ieLayer.setStrides(strides);
1981         ieLayer.setDilation(dilations);
1982         ieLayer.setPaddingsBegin(pads_begin);
1983
1984         if (padMode.empty())
1985         {
1986             std::vector<size_t> paddings_end;
1987             for (int i = 0; i < pads_end.size(); i++) {
1988                 paddings_end.push_back(pads_end[i] - adjust_pads[i]);
1989             }
1990             ieLayer.setPaddingsEnd(paddings_end);
1991         }
1992         else if (padMode == "SAME")
1993         {
1994             std::vector<size_t> paddings_end;
1995             for (int i = 0; i < pads_begin.size(); i++) {
1996                 paddings_end.push_back(kernel_size[i] - pads_begin[i] - 1 - adjust_pads[i]);
1997             }
1998             ieLayer.setPaddingsEnd(paddings_end);
1999         }
2000         ieLayer.setGroup((size_t)group);
2001         ieLayer.setOutDepth((size_t)numOutput);
2002
2003         InferenceEngine::Builder::Layer l = ieLayer;
2004         addConstantData("weights", ieWeights, l);
2005         if (hasBias())
2006             addConstantData("biases", wrapToInfEngineBlob(biasesMat, {(size_t)numOutput}, InferenceEngine::Layout::C), l);
2007         return Ptr<BackendNode>(new InfEngineBackendNode(l));
2008     }
2009 #endif  // HAVE_INF_ENGINE
2010
2011     virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
2012                            const std::vector<MatShape> &outputs) const CV_OVERRIDE
2013     {
2014         CV_Assert(inputs.size() == outputs.size());
2015
2016         float flops = 0;
2017         int outChannels = blobs[0].size[0];
2018         size_t karea = std::accumulate(kernel_size.begin(), kernel_size.end(),
2019                                        1, std::multiplies<size_t>());
2020
2021         for (int i = 0; i < inputs.size(); i++)
2022         {
2023             flops += CV_BIG_INT(2)*outChannels*karea*total(inputs[i]);
2024         }
2025
2026         return flops;
2027     }
2028 };
2029
2030 Ptr<BaseConvolutionLayer> ConvolutionLayer::create(const LayerParams &params)
2031 {
2032     Ptr<ConvolutionLayerImpl> l(new ConvolutionLayerImpl(params));
2033     return l;
2034 }
2035
2036 Ptr<BaseConvolutionLayer> DeconvolutionLayer::create(const LayerParams &params)
2037 {
2038     return Ptr<BaseConvolutionLayer>(new DeConvolutionLayerImpl(params));
2039 }
2040
2041 }
2042 }